|
@@ -94,11 +94,20 @@ class Api
|
|
|
$col = $this->col($config['id']);
|
|
|
$set = $this->set($config['id']);
|
|
|
|
|
|
- if (strpos($config['site'], '{') !== false && strpos($config['site'], '}') !== false) {
|
|
|
- $this->preg($config, $col, $set);
|
|
|
+ $config['curl'] = array
|
|
|
+ (
|
|
|
+ 'request_type' => $config['request_type'],
|
|
|
+ 'content_type' => $config['content_type'],
|
|
|
+ 'header' => $config['header'],
|
|
|
+ 'param' => $config['param'],
|
|
|
+ );
|
|
|
+ if (strpos($config['site'], '{page=') !== false) {
|
|
|
+ $this->page($config['site'], 1, $config, $col, $set);
|
|
|
+ } if (strpos($config['param'], '{page=') !== false) {
|
|
|
+ $this->page($config['param'], 2, $config, $col, $set);
|
|
|
} else {
|
|
|
Dever::load('spider/lib/project')->set($config, 4, 1);
|
|
|
- $this->parse($config['site'], $config['id'], $config['collect_rule'], $col, $set, $config['push']);
|
|
|
+ $this->parse($config['site'], $config['id'], $config['collect_rule'], $config['curl'], $col, $set, $config['push']);
|
|
|
}
|
|
|
/*
|
|
|
Dever::task(function() use($config, $this)
|
|
@@ -120,34 +129,31 @@ class Api
|
|
|
return Dever::db('spider/set')->getList(['where_pid' => $project]);
|
|
|
}
|
|
|
|
|
|
- private function parse($url, $project, $rule, $col, $set, $push)
|
|
|
+ private function parse($url, $project, $rule, $param, $col, $set, $push)
|
|
|
{
|
|
|
- $parse = new Parse($url, $project, $rule, $col, $set, $push);
|
|
|
+ $parse = new Parse($url, $project, $rule, $param, $col, $set, $push);
|
|
|
return $parse->get();
|
|
|
}
|
|
|
|
|
|
- private function preg($config, $col, $set)
|
|
|
+ private function page($source, $type, $config, $col, $set)
|
|
|
{
|
|
|
- $pat = '/{(.*?)}/i';
|
|
|
- preg_match_all($pat, $config['site'], $match);
|
|
|
+ $pat = '/{page=(.*?)}/i';
|
|
|
+ preg_match_all($pat, $source, $match);
|
|
|
if (isset($match[1][0]) && $match[1][0]) {
|
|
|
if ($config['page_num'] <= 0) $config['page_num'] = 100;
|
|
|
- parse_str($match[1][0], $param);
|
|
|
- $this->page($param, $match[0][0], $config, $col, $set);
|
|
|
- }
|
|
|
- }
|
|
|
+ //parse_str($match[1][0], $param);
|
|
|
+ $page = $match[1][0];
|
|
|
|
|
|
- private function page($param, $replace, $config, $col, $set)
|
|
|
- {
|
|
|
- if (isset($param['page']) && $param['page']) {
|
|
|
- for ($i = $param['page']; $i <= $config['page_num']; $i++) {
|
|
|
- $url = str_replace($replace, $i, $config['site']);
|
|
|
+ for ($i = $page; $i <= $config['page_num']; $i++) {
|
|
|
+ $source = str_replace($match[0][0], $i, $source);
|
|
|
Dever::load('spider/lib/project')->set($config, 4, $i);
|
|
|
- $this->parse($url, $config['id'], $config['collect_rule'], $col, $set, $config['push']);
|
|
|
+ if ($type == 1) {
|
|
|
+ $config['site'] = $source;
|
|
|
+ } else {
|
|
|
+ $config['curl']['param'] = $source;
|
|
|
+ }
|
|
|
+ $this->parse($config['site'], $config['id'], $config['collect_rule'], $config['curl'], $col, $set, $config['push']);
|
|
|
}
|
|
|
- } else {
|
|
|
- Dever::load('spider/lib/project')->set($config, 4, 1);
|
|
|
- $this->parse(str_replace($replace, '', $value), $config['id'], $config['collect_rule'], $col, $set, $config['push']);
|
|
|
}
|
|
|
}
|
|
|
}
|