forked from meteor1993/python-learning
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathCrawlProxy.py
More file actions
52 lines (46 loc) · 1.9 KB
/
Copy pathCrawlProxy.py
File metadata and controls
52 lines (46 loc) · 1.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
import requests
from pyquery import PyQuery
from MysqlClient import MysqlClient
from VerifyProxy import VerifyProxy
class CrawlProxy(object):
def __init__(self):
self.mysql = MysqlClient()
self.verify = VerifyProxy()
def get_page(self, url, charset):
response = requests.get(url)
response.encoding = charset
return response.text
def crawl_ip3366(self, page_num = 3):
"""
获取代理 ip3366
:param page_num:
:return:
"""
start_url = 'http://www.ip3366.net/?stype=1&page={}'
urls = [start_url.format(page) for page in range(1, page_num + 1)]
for url in urls:
print('crawl:', url)
html = self.get_page(url, 'gb2312')
if html:
d = PyQuery(html)
trs = d('.table-bordered tbody tr').items()
for tr in trs:
scheme = tr.find('td:nth-child(4)').text().lower()
ip = tr.find('td:nth-child(1)').text()
port = tr.find('td:nth-child(2)').text()
verify_result = self.verify.verify_proxy(scheme, ip, port)
if verify_result["status"] == '1':
proxy = {
"scheme": scheme,
"ip": ip,
"port": port,
"status": verify_result["status"],
"response_time": verify_result["response_time"],
}
# 存入数据库
self.mysql.add_proxy(proxy)
print('代理', ip, '连通测试已通过，已保存 Mysql')
else:
print('代理', ip, '连通测试未通过')
if __name__ == '__main__':
CrawlProxy().crawl_ip3366()