Use aiohttp.ClientSession in crawler.

ajdavis · ajdavis · commit e144e27de3ae · 2015-09-05T19:10:14.000-04:00
Beginning with aiohttp 0.16, ClientSession is preferred to using
TCPConnector directly, for connection pooling HTTP keep-alive.
diff --git a/crawler/code/crawling.py b/crawler/code/crawling.py
@@ -61,7 +61,7 @@ def __init__(self, roots,
         self.q = Queue(loop=self.loop)
         self.seen_urls = set()
         self.done = []
-        self.connector = aiohttp.TCPConnector(loop=self.loop)
+        self.session = aiohttp.ClientSession(loop=self.loop)
         self.root_domains = set()
         for root in roots:
             parts = urllib.parse.urlparse(root)
@@ -83,7 +83,7 @@ def __init__(self, roots,
 
     def close(self):
         """Close resources."""
-        self.connector.close()
+        self.session.close()
 
     def host_okay(self, host):
         """Check if a host should be crawled.
@@ -172,11 +172,9 @@ def fetch(self, url, max_redirect):
         exception = None
         while tries < self.max_tries:
             try:
-                response = yield from aiohttp.request(
-                    'get', url,
-                    connector=self.connector,
-                    allow_redirects=False,
-                    loop=self.loop)
+                response = yield from self.session.get(
+                    url, allow_redirects=False)
+
                 if tries > 1:
                     LOGGER.info('try %r for %r success', tries, url)
 
@@ -229,6 +227,8 @@ def fetch(self, url, max_redirect):
                 self.q.put_nowait((link, self.max_redirect))
             self.seen_urls.update(links)
 
+        yield from response.release()
+
     @asyncio.coroutine
     def work(self):
         """Process queue items forever."""
diff --git a/crawler/code/requirements-py33.txt b/crawler/code/requirements-py33.txt
@@ -7,4 +7,4 @@
 # Then install these packages with "pip install -r requirements-py33.txt".
 
 asyncio
-aiohttp
+aiohttp>=0.16
diff --git a/crawler/code/requirements-py34.txt b/crawler/code/requirements-py34.txt
@@ -6,4 +6,4 @@
 #
 # Then install this package with "pip install -r requirements-py34.txt".
 
-aiohttp
+aiohttp>=0.16
diff --git a/crawler/crawler.markdown b/crawler/crawler.markdown
@@ -298,7 +298,7 @@ We entice you with a promise. It is possible to write asynchronous code that com
 ```python
     @asyncio.coroutine
     def fetch(self, url):
-        response = yield from aiohttp.request('get', url)
+        response = yield from self.session.get(url)
         body = yield from response.read()
 ```
 
@@ -806,6 +806,10 @@ class Crawler:
         self.q = Queue()
         self.seen_urls = set()
         
+        # aiohttp's ClientSession does connection pooling and
+        # HTTP keep-alives for us.
+        self.session = aiohttp.ClientSession(loop=self.loop)
+        
         # Put (URL, max_redirect) in the queue.
         self.q.put((root_url, self.max_redirect))
 ```
@@ -935,8 +939,8 @@ The crawler fetches "foo" and sees it redirects to "baz", so it adds "baz" to th
     @asyncio.coroutine
     def fetch(self, url, max_redirect):
         # Handle redirects ourselves.
-        response = yield from aiohttp.request(
-            'get', url, allow_redirects=False)
+        response = yield from self.session.get(
+            url, allow_redirects=False)
 
         if is_redirect(response):
             if max_redirect > 0:

Original file line number	Diff line number	Diff line change
`@@ -6,4 +6,4 @@`
`6`	`6`	`#`
`7`	`7`	`# Then install this package with "pip install -r requirements-py34.txt".`
`8`	`8`
`9`		`-aiohttp`
	`9`	`+aiohttp>=0.16`