Parse.php 7.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311
  1. <?php
  2. namespace Spider\Lib;
  3. use Dever;
  4. use Spider\Lib\Doc\Dom;
  5. class Parse
  6. {
  7. private $url = '';
  8. private $host = '';
  9. private $log;
  10. private $doc = array();
  11. private $data = array();
  12. public function __construct($url, $project, $rule, $param, $col, $set, $push)
  13. {
  14. $doc = Doc::getInstance($url, $rule);
  15. $doc->log(new Log($project));
  16. $data = $doc->get($param);
  17. if ($data) {
  18. if (!is_array($data) && !is_object($data)) {
  19. $state = Dever::json_decode($data);
  20. if ($state) {
  21. $data = $state;
  22. }
  23. }
  24. if ($data) {
  25. if (is_array($data)) {
  26. $domain = parse_url($url);
  27. $host = $domain['scheme'] . '://' . $domain['host'] . '/';
  28. foreach ($data as $k => $v) {
  29. if (is_string($v) && !strstr($v, 'http')) {
  30. $v = $host . ltrim($v, '/');
  31. }
  32. $this->data[$k] = $this->load($doc, $k, $v, $col, $set, $push, $project, $v);
  33. }
  34. } else {
  35. $this->data = $this->load($doc, 0, $data, $col, $set, $push, $project, $url);
  36. }
  37. }
  38. }
  39. $doc->saveLog();
  40. }
  41. public function get()
  42. {
  43. return $this->data;
  44. }
  45. private function load($doc, $index, $data, $col, $set, $push, $project, $source)
  46. {
  47. if (!$col) {
  48. if (Dever::input('test') == 1) {
  49. $doc->outLog();
  50. echo 'error';die;
  51. }
  52. return false;
  53. }
  54. $result = $table = array();
  55. if (isset($col[1])) {
  56. $data = $doc->init($data);
  57. $this->getCol($doc, $col[1], $data, $result, $table, $source);
  58. }
  59. if (isset($col[2])) {
  60. $data = $doc->getCur();
  61. $this->getCol($doc, $col[2], $data, $result, $table, $source);
  62. }
  63. if ($set) {
  64. foreach ($set as $v) {
  65. $value = $this->set($index, $v, $project);
  66. $result[$v['key']] = $value;
  67. if (Dever::input('test') == 1) {
  68. $table[$v['name']] = $value;
  69. }
  70. }
  71. }
  72. if ($push) {
  73. $result['test'] = Dever::input('test');
  74. $this->push($push, $result, $project);
  75. }
  76. if (Dever::input('test') == 1) {
  77. $doc->outLog();
  78. echo Dever::table($table);die;
  79. }
  80. $this->update($result, $project, $source);
  81. return $result;
  82. }
  83. private function getCol($doc, $col, $data, &$result, &$table, $source = false)
  84. {
  85. foreach ($col as $v) {
  86. $callback = false;
  87. if (strpos($v['key'], '.') !== false) {
  88. $temp = explode('.', $v['key']);
  89. $v['key'] = $temp[1];
  90. $callback = $temp[0];
  91. }
  92. if ($source && strstr($v['collect_rule'], '{link}')) {
  93. $v['collect_rule'] = str_replace('{link}', $source, $v['collect_rule']);
  94. }
  95. $value = $doc->rule($data, $col, $v);
  96. if ($value == 'error') {
  97. break;
  98. }
  99. if ($callback) {
  100. if (function_exists($callback)) {
  101. $value = $callback($value);
  102. } else {
  103. $value = Dever::{$callback}($value);
  104. }
  105. }
  106. if ($v['local'] == 1) {
  107. $this->res = $v['res_key'];
  108. $value = $this->local($value, $v['type']);
  109. }
  110. if ($v['collect_filter_link'] == 1) {
  111. $value = $this->filter($value);
  112. }
  113. if ($value) {
  114. $result[$v['key']] = $value;
  115. if (Dever::input('test') == 1) {
  116. $table[$v['name']] = $value;
  117. }
  118. }
  119. }
  120. }
  121. private function push($push, $data, $project)
  122. {
  123. $push = Dever::split($push);
  124. $data['project_id'] = $project;
  125. foreach ($push as $k => $v) {
  126. if (strstr($v, 'http')) {
  127. Dever::curl($v, $data, 'post');
  128. } else {
  129. Dever::load($v, $data);
  130. }
  131. }
  132. }
  133. private function set($index, $data, $project)
  134. {
  135. if ($data['type'] == 1) {
  136. return $data['value'];
  137. } elseif ($data['type'] == 2) {
  138. $old = 0;
  139. $info = Dever::db('spider/data')->getOne(array('pid' => $project));
  140. if($info) {
  141. $value = json_decode($info['value'], true);
  142. if (isset($value[$data['key']])) {
  143. $old = $value[$data['key']];
  144. }
  145. }
  146. return $data['value'] + $index + $old;
  147. } elseif ($data['type'] == 3) {
  148. $eval = '$value = ' . $data['value'] . ';';
  149. eval($eval);
  150. return $value;
  151. } elseif ($data['type'] == 4) {
  152. $temp = Dever::split($data['value']);
  153. return mt_rand($temp[0], $temp[1]);
  154. } elseif ($data['type'] == 5) {
  155. $temp = Dever::split($data['value']);
  156. $temp[0] = Dever::maketime($temp[0]);
  157. $temp[1] = Dever::maketime($temp[1]);
  158. return mt_rand($temp[0], $temp[1]);
  159. }
  160. }
  161. private function filter($content)
  162. {
  163. $rule = '<(a).+href="(.*?)"(.*?)>(.*?)<\/a>';
  164. $content = preg_replace_callback('/' . $rule . '/i', array($this, 'filter_replace'), $content);
  165. return $content;
  166. }
  167. private function filter_replace($result)
  168. {
  169. if (isset($result[4]) && $result[4]) {
  170. return $result[4];
  171. }
  172. }
  173. private function local($content, $type = 1)
  174. {
  175. if ($type == 1) {
  176. $doc = Dom::init($content);
  177. $pic = Dom::find($doc, '$("img").each().attr("src")');
  178. if ($pic) {
  179. $content = $this->local_replace($pic, $content);
  180. }
  181. $video = Dom::find($doc, '$("video").each().attr("src")');
  182. if ($video) {
  183. $content = $this->local_replace($video, $content);
  184. }
  185. $audio = Dom::find($doc, '$("audio").each().attr("src")');
  186. if ($audio) {
  187. $content = $this->local_replace($audio, $content);
  188. }
  189. /*
  190. $rule = '<(img|video|audio).+src=\"?(.+\.(jpg|gif|bmp|bnp|png))\"?.+>';
  191. $content = preg_replace_callback('/' . $rule . '/i', array($this, 'local_rule_replace'), $content);
  192. */
  193. } else {
  194. $content = $this->copy($content);
  195. }
  196. return $content;
  197. }
  198. private function local_replace($file, $content)
  199. {
  200. if (is_string($file) && strstr($file, '[')) {
  201. $file = Dever::json_decode($file);
  202. }
  203. if (is_array($file)) {
  204. foreach ($file as $k => $v) {
  205. $content = $this->local_replace($v, $content);
  206. }
  207. } else {
  208. $result = $this->copy($file);
  209. if ($result) {
  210. $content = str_replace($file, $result, $content);
  211. }
  212. }
  213. return $content;
  214. }
  215. private function local_rule_replace($result)
  216. {
  217. if (isset($result[2]) && $result[2]) {
  218. $file = $this->copy($result[2]);
  219. if ($file) {
  220. $result[0] = str_replace($result[2], $file, $result[0]);
  221. return $result[0];
  222. }
  223. }
  224. }
  225. private function copy($file)
  226. {
  227. if (is_string($file) && strstr($file, '[')) {
  228. $temp = Dever::json_decode($file);
  229. if ($temp) {
  230. $file = array();
  231. foreach($temp as $k => $v) {
  232. $f = $this->copy($v);
  233. if ($f) {
  234. $file[] = $f;
  235. }
  236. }
  237. $file = implode(',', $file);
  238. return $file;
  239. }
  240. }
  241. if (strstr($file, ',')) {
  242. $temp = explode(',', $file);
  243. $file = array();
  244. foreach($temp as $k => $v) {
  245. $f = $this->copy($v);
  246. if ($f) {
  247. $file[] = $f;
  248. }
  249. }
  250. $file = implode(',', $file);
  251. return $file;
  252. }
  253. if (!strstr($file, 'http') && strstr($file, '//')) {
  254. $file = 'https:' . $file;
  255. }
  256. $data = Dever::load('upload/save.copy?file=' . $file . '&key=' . $this->res . '&state=1');
  257. if (isset($data['status']) && $data['status'] == -1) {
  258. return '';
  259. } elseif (isset($data['url'])) {
  260. return $data['url'];
  261. } else {
  262. return '';
  263. }
  264. }
  265. private function update($data, $project, $source)
  266. {
  267. if (!$data) {
  268. return;
  269. }
  270. $param['pid'] = $project;
  271. $param['source'] = $source;
  272. $info = Dever::db('spider/data')->one($param);
  273. $param['value'] = json_encode($data, JSON_UNESCAPED_UNICODE);
  274. if ($info) {
  275. $update = $param;
  276. $id = $update['where_id'] = $info['id'];
  277. Dever::db('spider/data')->update($update);
  278. } else {
  279. $update = $param;
  280. $id = Dever::db('spider/data')->insert($update);
  281. }
  282. echo $id;
  283. echo "\r\n";
  284. }
  285. }