-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathqueue.py
More file actions
45 lines (33 loc) · 1.05 KB
/
queue.py
File metadata and controls
45 lines (33 loc) · 1.05 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
# !/usr/bin/env python
# -*- coding: utf-8 -*-
import Queue
import threading
import urllib2
import time
hosts = ["http://www.baidu.com", "http://www.163.com", "http://www.weibo.com",
"http://www.sina.com", "http://www.oschina.net"]
queue = Queue.Queue()
class ThreadUrl(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while True:
host = self.queue.get()
# 设置头信息
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
headers = {'Connection': 'keep-alive', 'User-Agent': user_agent}
req = urllib2.Request(host, None, headers=headers)
res = urllib2.urlopen(req)
print res.read(1024)+"\n-----------------"
self.queue.task_done()
def start():
for i in range(len(hosts)):
t = ThreadUrl(queue)
t.setDaemon(True)
t.start()
for host in hosts:
queue.put(host)
queue.join()
if __name__ == '__main__':
start()