forked from jaraco/cssutils
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy path_fetchgae.py
More file actions
78 lines (63 loc) · 2.4 KB
/
_fetchgae.py
File metadata and controls
78 lines (63 loc) · 2.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
"""GAE specific URL reading functions"""
__all__ = ['_defaultFetcher']
import email.message
from google.appengine.api import urlfetch
from . import errorhandler
log = errorhandler.ErrorHandler()
def _parse_header(content_type):
msg = email.message.EmailMessage()
msg['content-type'] = content_type
return msg.get_content_type(), msg['content-type'].params
def _defaultFetcher(url):
"""
uses GoogleAppEngine (GAE)
fetch(url, payload=None, method=GET, headers={}, allow_truncated=False)
Response
content
The body content of the response.
content_was_truncated
True if the allow_truncated parameter to fetch() was True and
the response exceeded the maximum response size. In this case,
the content attribute contains the truncated response.
status_code
The HTTP status code.
headers
The HTTP response headers, as a mapping of names to values.
Exceptions
exception InvalidURLError()
The URL of the request was not a valid URL, or it used an
unsupported method. Only http and https URLs are supported.
exception DownloadError()
There was an error retrieving the data.
This exception is not raised if the server returns an HTTP
error code: In that case, the response data comes back intact,
including the error code.
exception ResponseTooLargeError()
The response data exceeded the maximum allowed size, and the
allow_truncated parameter passed to fetch() was False.
"""
try:
r = urlfetch.fetch(url, method=urlfetch.GET)
except urlfetch.Error as e:
log.warn(f'Error opening url={url!r}: {e}', error=IOError)
return
if r.status_code != 200:
# TODO: 301 etc
log.warn(
f'Error opening url={url!r}: HTTP status {r.status_code}',
error=IOError,
)
return
# find mimetype and encoding
try:
mimetype, params = _parse_header(r.headers['content-type'])
encoding = params['charset']
except KeyError:
mimetype = 'application/octet-stream'
encoding = None
if mimetype != 'text/css':
log.error(
f'Expected "text/css" mime type for url {url!r} but found: {mimetype!r}',
error=ValueError,
)
return encoding, r.content