1 parent ad751e8 commit e23e97aCopy full SHA for e23e97a
1 file changed
crawler/crawling.py
@@ -42,27 +42,6 @@ def is_redirect(response):
42
'num_new_urls'])
43
44
45
-class ExampleQueue(Queue):
46
- """Simplified joinable queue that doesn't use Event.
47
-
48
- Written to test out code examples in chapter.md. Not actually used below.
49
- """
50
- def __init__(self, maxsize=0, *, loop=None):
51
- super().__init__(maxsize, loop=loop)
52
- self._join_future = Future(loop=self._loop)
53
54
- def task_done(self):
55
- if self._unfinished_tasks <= 0:
56
- raise ValueError('task_done() called too many times')
57
- self._unfinished_tasks -= 1
58
- if self._unfinished_tasks == 0:
59
- self._join_future.set_result(None)
60
61
- def join(self):
62
- if self._unfinished_tasks > 0:
63
- yield from self._join_future
64
65
66
class Crawler:
67
"""Crawl a set of URLs.
68
0 commit comments