init & add agent & add job

2025-12-13 09:06:43 +00:00 · 2019-01-15 10:02:28 +08:00
parent 71f1f10e2c
commit d0a4b891b5
321 changed files with 24657 additions and 1 deletions
--- a/Spider.class.php
+++ b/Spider.class.php
@@ -0,0 +1,129 @@
+<?php
+
+require_once('util4p/CRObject.class.php');
+
+class Spider
+{
+	private $userAgent = 'Spider';
+	private $accept = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8';
+	private $acceptEncoding = 'gzip, deflate, br';
+	private $acceptLanguage = 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7';
+	private $cookie = '';
+	private $referer = '';
+
+	private $timeout = 15;
+	private $headers = array();
+	private $body = '';
+
+	private $info = array();
+
+
+	public function configure(CRObject $config)
+	{
+		$this->userAgent = $config->get('User-Agent', $this->userAgent);
+		$this->accept = $config->get('Accept', $this->accept);
+		$this->acceptEncoding = $config->get('Accept-Encoding', $this->acceptEncoding);
+		$this->acceptLanguage = $config->get('Accept-Encoding', $this->acceptLanguage);
+		$this->cookie = $config->get('Cookie', $this->cookie);
+		$this->referer = $config->get('Referer', $this->referer);
+		$this->timeout = $config->get('timeout', $this->timeout);
+	}
+
+	public function doGet($url)
+	{
+		$ch = curl_init();
+		curl_setopt($ch, CURLOPT_URL, $url);
+		curl_setopt($ch, CURLOPT_TIMEOUT, $this->timeout);
+		curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $this->timeout - 2);
+		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($ch, CURLOPT_HTTPHEADER, array('Expect:')); //避免data数据过长问题
+		curl_setopt($ch, CURLOPT_POST, false);
+		curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+		curl_setopt($ch, CURLOPT_HEADER, 1);
+		$ret = curl_exec($ch);
+		$err = curl_error($ch);
+		if (!$err) {
+			$header_size = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
+			$header = substr($ret, 0, $header_size);
+			$headers = array();
+			// Split the string on every "double" new line.
+			$arrRequests = explode("\r\n\r\n", $header);
+			// Loop of response headers. The "count() -1" is to avoid an empty row for the extra line break before the body of the response.
+			for ($index = 0; $index < count($arrRequests) - 1; $index++) {
+				foreach (explode("\r\n", $arrRequests[$index]) as $i => $line) {
+					if ($i === 0)
+						$headers[$index]['http_code'] = $line;
+					else {
+						list ($key, $value) = explode(': ', $line);
+						$headers[$index][$key] = $value;
+					}
+				}
+			}
+			$this->headers = $headers[max(0, count($headers) - 1)];
+			$this->body = substr($ret, $header_size);
+			$this->info = curl_getinfo($ch);
+		}
+		return !$err;
+	}
+
+	/*
+	 *
+	 * @param $url string
+	 * @param $post_data array('key' => 'value')
+	 *
+	 * */
+	public function doPost($url, $post_data)
+	{
+		$fields_string = http_build_query($post_data);
+		//open connection
+		$ch = curl_init();
+		//set the url, number of POST vars, POST data
+		curl_setopt($ch, CURLOPT_URL, $url);
+		curl_setopt($ch, CURLOPT_TIMEOUT, $this->timeout);
+		curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $this->timeout - 2);
+		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($ch, CURLOPT_POST, 1);
+		curl_setopt($ch, CURLOPT_POSTFIELDS, $fields_string);
+		curl_setopt($ch, CURLOPT_HEADER, 1);
+
+		$ret = curl_exec($ch);
+		$err = curl_error($ch);
+		if (!$err) {
+			$header_size = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
+			$header = substr($ret, 0, $header_size);
+			$headers = array();
+			// Split the string on every "double" new line.
+			$arrRequests = explode("\r\n\r\n", $header);
+			// Loop of response headers. The "count() -1" is to avoid an empty row for the extra line break before the body of the response.
+			for ($index = 0; $index < count($arrRequests) - 1; $index++) {
+				foreach (explode("\r\n", $arrRequests[$index]) as $i => $line) {
+					if ($i === 0)
+						$headers[$index]['http_code'] = $line;
+					else {
+						list ($key, $value) = explode(': ', $line);
+						$headers[$index][$key] = $value;
+					}
+				}
+			}
+			$this->headers = $headers[max(0, count($headers) - 1)];
+			$this->body = substr($ret, $header_size);
+			$this->info = curl_getinfo($ch);
+		}
+		return !$err;
+	}
+
+	public function getHeader($key)
+	{
+		return $key;
+	}
+
+	public function getBody()
+	{
+		return $this->body;
+	}
+
+	public function getStatusCode()
+	{
+		return $this->info['http_code'];
+	}
+}