# HG changeset patch
# Parent 8c33152fd75c3a0c8516ed048b70d56a61815e1f
Issue #24363: Bypass parser to avoid parsing HTTP headers as email body
Now http.client.parse_headers() calls the internal FeedParser._parse_
headers() method directly, bypassing the possibility of the main parsing
logic interpreting invalid HTTP header fields as an email body.
diff -r 8c33152fd75c Doc/library/email.errors.rst
--- a/Doc/library/email.errors.rst Fri Aug 12 12:05:48 2016 +0000
+++ b/Doc/library/email.errors.rst Fri Aug 12 12:20:07 2016 +0000
@@ -91,17 +91,15 @@
middle of a header block.
* :class:`MissingHeaderBodySeparatorDefect` - A line was found while parsing
- headers that had no leading white space but contained no ':'. Parsing
- continues assuming that the line represents the first line of the body.
+ headers that had no leading white space but contained no ':'. If parsing
+ continues assuming that the line represents the first line of the body,
+ this defect is used.
.. versionadded:: 3.3
* :class:`MalformedHeaderDefect` -- A header was found that was missing a colon,
or was otherwise malformed.
- .. deprecated:: 3.3
- This defect has not been used for several Python versions.
-
* :class:`MultipartInvariantViolationDefect` -- A message claimed to be a
:mimetype:`multipart`, but no subparts were found. Note that when a message
has this defect, its :meth:`~email.message.Message.is_multipart` method may
diff -r 8c33152fd75c Lib/email/errors.py
--- a/Lib/email/errors.py Fri Aug 12 12:05:48 2016 +0000
+++ b/Lib/email/errors.py Fri Aug 12 12:20:07 2016 +0000
@@ -55,8 +55,9 @@
class MissingHeaderBodySeparatorDefect(MessageDefect):
"""Found line with no leading whitespace and no colon before blank line."""
-# XXX: backward compatibility, just in case (it was never emitted).
-MalformedHeaderDefect = MissingHeaderBodySeparatorDefect
+
+class MalformedHeaderDefect(MessageDefect):
+ """An ordinary header line did not match the expected format."""
class MultipartInvariantViolationDefect(MessageDefect):
"""A message claimed to be a multipart but no subparts were found."""
diff -r 8c33152fd75c Lib/email/feedparser.py
--- a/Lib/email/feedparser.py Fri Aug 12 12:05:48 2016 +0000
+++ b/Lib/email/feedparser.py Fri Aug 12 12:20:07 2016 +0000
@@ -237,6 +237,10 @@
self._input.unreadline(line)
break
headers.append(line)
+ if len(headers) > 1 and headers[-1].startswith('From '):
+ # Something looking like a unix-from at the end - it's
+ # probably the first line of the body, so push the line back.
+ self._input.unreadline(headers.pop())
# Done with the headers, so parse them and figure out what we're
# supposed to see in the body of the message.
self._parse_headers(headers)
@@ -321,11 +325,7 @@
lines.append(line)
self._cur.set_payload(EMPTYSTRING.join(lines))
return
- # Make sure a valid content type was specified per RFC 2045:6.4.
- if (self._cur.get('content-transfer-encoding', '8bit').lower()
- not in ('7bit', '8bit', 'binary')):
- defect = errors.InvalidMultipartContentTransferEncodingDefect()
- self.policy.handle_defect(self._cur, defect)
+ self._validate_composite_cte()
# Create a line match predicate which matches the inter-part
# boundary as well as the end-of-multipart boundary. Don't push
# this onto the input stream until we've scanned past the
@@ -472,6 +472,7 @@
def _parse_headers(self, lines):
# Passed a list of lines that make up the headers for the current msg
+ # This method is also called directly by http.client.parse_headers()
lastheader = ''
lastvalue = []
for lineno, line in enumerate(lines):
@@ -497,23 +498,18 @@
if mo:
line = line[:-len(mo.group(0))]
self._cur.set_unixfrom(line)
- continue
- elif lineno == len(lines) - 1:
- # Something looking like a unix-from at the end - it's
- # probably the first line of the body, so push back the
- # line and stop.
- self._input.unreadline(line)
- return
else:
# Weirdly placed unix-from line. Note this as a defect
# and ignore it.
defect = errors.MisplacedEnvelopeHeaderDefect(line)
self._cur.defects.append(defect)
- continue
+ continue
+
# Split the line on the colon separating field name from value.
- # There will always be a colon, because if there wasn't the part of
- # the parser that calls us would have started parsing the body.
- i = line.find(':')
+ # There will always be a colon, because if there wasn't, it would
+ # have been picked up by a headerRE check in the caller, or the
+ # continuation or envelope checks above.
+ i = line.index(':')
# If the colon is on the start of the line the header is clearly
# malformed, but we might be able to salvage the rest of the
@@ -523,13 +519,20 @@
self._cur.defects.append(defect)
continue
- assert i>0, "_parse_headers fed line with no : and no leading WS"
lastheader = line[:i]
lastvalue = [line]
# Done with all the lines, so handle the last header.
if lastheader:
self._cur.set_raw(*self.policy.header_source_parse(lastvalue))
+ def _validate_composite_cte(self):
+ # Make sure a valid Content-Transfer-Encoding was specified per
+ # RFC 2045:6.4.
+ if (self._cur.get('content-transfer-encoding', '8bit').lower()
+ not in ('7bit', '8bit', 'binary')):
+ defect = errors.InvalidMultipartContentTransferEncodingDefect()
+ self.policy.handle_defect(self._cur, defect)
+
class BytesFeedParser(FeedParser):
"""Like FeedParser, but feed accepts bytes."""
diff -r 8c33152fd75c Lib/http/client.py
--- a/Lib/http/client.py Fri Aug 12 12:05:48 2016 +0000
+++ b/Lib/http/client.py Fri Aug 12 12:20:07 2016 +0000
@@ -68,7 +68,8 @@
Req-sent-unread-response _CS_REQ_SENT