Doc.php 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. <?php
  2. namespace Spider\Lib;
  3. use Dever;
  4. use Spider\Lib\Doc\Json;
  5. use Spider\Lib\Doc\Dom;
  6. class Doc
  7. {
  8. private $url;
  9. private $host;
  10. private $path;
  11. private $rule;
  12. public $type = 'dom';
  13. private $log = false;
  14. private $cur = '';
  15. static protected $instance;
  16. static public function getInstance($url, $rule = '')
  17. {
  18. $key = $url . md5($rule);
  19. if (empty(self::$instance[$key])) {
  20. # 此处开协程:Dever::coroutine(self::$instance[$key]);
  21. self::$instance[$key] = new self($url, $rule);
  22. }
  23. return self::$instance[$key];
  24. }
  25. public function __construct($url, $rule = '')
  26. {
  27. $this->url($url);
  28. $this->rule = $rule;
  29. if (strpos($this->rule, '$json') !== false) {
  30. $this->type = 'json';
  31. } else {
  32. $this->type = 'dom';
  33. }
  34. }
  35. private function url($url = false)
  36. {
  37. if (!$url) {
  38. return;
  39. }
  40. $this->url = $url;
  41. $value = parse_url($this->url);
  42. $this->path = $this->host = $value['scheme'] . '://' . $value['host'];
  43. if (isset($value['path']) && $value['path']) {
  44. $temp = explode('/', $value['path']);
  45. unset($temp[count($temp)-1]);
  46. $this->path .= implode('/', $temp);
  47. }
  48. $this->path .= '/';
  49. }
  50. public function get($param = array())
  51. {
  52. $doc = $this->doc(false, $param);
  53. if (!$this->cur) {
  54. $this->cur = $doc;
  55. }
  56. if ($this->rule) {
  57. $doc = $this->find($doc, $this->rule);
  58. }
  59. return $doc;
  60. }
  61. public function doc($url = false, $param = array())
  62. {
  63. $this->url($url);
  64. $html = $this->download($this->url, $param);
  65. return ($this->getClass())::init($html);
  66. }
  67. public function getCur()
  68. {
  69. return $this->cur;
  70. }
  71. private function download($url, $header = '', $param = '')
  72. {
  73. $this->addLog($url . '下载中...');
  74. $download = new Download($url, $header, $param);
  75. $this->addLog($url . '下载完成');
  76. return $download->get($this->type);
  77. }
  78. private function collect($data, $include, $exclude, $filter)
  79. {
  80. if ($include) {
  81. $include = explode("\n", str_replace("\r", '', $include));
  82. foreach ($include as $k => $v) {
  83. $state = preg_match('/' . $v . '/i', $data);
  84. if ($state) {
  85. break;
  86. }
  87. }
  88. if (!$state) {
  89. return 'error';
  90. }
  91. }
  92. if ($exclude) {
  93. $exclude = explode("\n", str_replace("\r", '', $exclude));
  94. foreach ($exclude as $k => $v) {
  95. $state = preg_match('/' . $v . '/i', $data);
  96. if (!$state) {
  97. return 'error';
  98. }
  99. }
  100. }
  101. if ($filter) {
  102. $filter = explode("\n", str_replace("\r", '', $filter));
  103. foreach ($filter as $k => $v) {
  104. $s = '';
  105. if (strstr($v, '=>')) {
  106. $temp = explode('=>', $v);
  107. $v = $temp[0];
  108. $s = $temp[1];
  109. }
  110. $data = preg_replace('/' . $v . '/i', $s, $data);
  111. }
  112. }
  113. return $data;
  114. }
  115. private function getClass()
  116. {
  117. return ucfirst($this->type);
  118. }
  119. public function find($doc, $rule)
  120. {
  121. return ($this->getClass())::find($doc, $rule);
  122. }
  123. public function init($data)
  124. {
  125. if (is_string($data) && filter_var($data, FILTER_VALIDATE_URL) !== false) {
  126. $data = $this->doc($data);
  127. } else {
  128. $data = ($this->getClass())::init($data);
  129. }
  130. return $data;
  131. }
  132. public function rule($data, $col, $config)
  133. {
  134. $name = '字段[' . $config['name'] . '('.$config['key'].')]' . '"';
  135. $this->addLog($name . '正在按照规则['.$config['collect_rule'].']进行解析');
  136. $method = 'rule_' . $this->type;
  137. $result = $this->getRule($data, $col, $config['collect_rule'], $config['key']);
  138. if (isset($config['collect_url']) && $config['collect_url']) {
  139. $collect_url = explode("\n", str_replace("\r", '', $config['collect_url']));
  140. if (!isset($collect_url[1])) {
  141. $collect_url[1] = '';
  142. }
  143. $temp = array();
  144. $temp[] = $result;
  145. $this->getNext($temp, $data, $col, $collect_url[0], $collect_url[1], $config['collect_rule'], $config['key']);
  146. $result = implode(',', $temp);
  147. }
  148. $this->addLog($name . '解析完成');
  149. return $this->collect($result, $config['collect_include'], $config['collect_exclude'], $config['collect_filter']);
  150. }
  151. public function getNext(&$result, $data, $col, $collect_url, $collect_include, $collect_rule, $key)
  152. {
  153. $url = $this->getUrl($data, $col, $collect_url, $key);
  154. if ($url) {
  155. if ($collect_include && !strstr($url, $collect_include)) {
  156. return;
  157. }
  158. $data = $this->init($url);
  159. if ($data) {
  160. $temp = $this->getRule($data, $col, $collect_rule, $key);
  161. if ($temp) {
  162. $result[] = $temp;
  163. $this->getNext($result, $data, $col, $collect_url, $collect_include, $collect_rule, $key);
  164. }
  165. }
  166. }
  167. }
  168. public function getUrl($data, $col, $collect_rule, $key)
  169. {
  170. $url = $this->getRule($data, $col, $collect_rule, $key);
  171. if (!$url) {
  172. return '';
  173. }
  174. if (strpos($url, 'http') === false) {
  175. if ($url[0] == '/') {
  176. $url = $this->host . $url;
  177. } elseif (strstr($url, '.')) {
  178. $url = $this->path . $url;
  179. } else {
  180. $url = $this->url . $url;
  181. }
  182. }
  183. return $url;
  184. }
  185. public function getRule($data, $col, $collect_rule, $key)
  186. {
  187. return ($this->getClass())::rule($this, $data, $col, $collect_rule, $key);
  188. }
  189. public function addLog($string)
  190. {
  191. if ($this->log) {
  192. $this->log->add($string);
  193. }
  194. }
  195. public function saveLog()
  196. {
  197. if ($this->log) {
  198. $this->log->save();
  199. }
  200. }
  201. public function outLog()
  202. {
  203. if ($this->log) {
  204. $this->log->out();
  205. }
  206. }
  207. public function log(Log $log)
  208. {
  209. $this->log = $log;
  210. }
  211. }