Skip to content

Match tokenizer token metadata with CPython 3.15 #9445

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 2 commits into from
Oct 29, 2026
Merged

Match tokenizer token metadata with CPython 3.15 #9445

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c178eb6
Match tokenizer token metadata with CPython 3.15
codex Oct 29, 2026
f8db381
Fix tokenizer continuation, numeric and UTF-8 EOF metadata
codex Oct 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions Lib/test/test_tokenize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1921,7 +1921,6 @@ def test_comment_at_the_end_of_the_source_without_newline(self):
tokens = list(tokenize.tokenize(BytesIO(source.encode('utf-8')).readline))
self.assertEqual(tokens, expected_tokens)

@unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 869 characters long. Set self.maxDiff to None to see it.
def test_newline_and_space_at_the_end_of_the_source_without_newline(self):
# See https://github.com/python/cpython/issues/105435
source = 'a\n '
Expand Down Expand Up @@ -2240,7 +2239,6 @@ def test_number_followed_by_name(self):
tokens = list(tokenize.generate_tokens(StringIO(source).readline))
self.assertEqual(tokens, expected_tokens)

@unittest.expectedFailure # TODO: RUSTPYTHON; Diff is 855 characters long. Set self.maxDiff to None to see it.
def test_number_starting_with_zero(self):
source = "01234"
expected_tokens = [
Expand Down Expand Up @@ -2881,7 +2879,6 @@ def test_tabs(self):
NAME 'pass' (2, 14) (2, 18)
""")

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_async(self):

self.check_tokenize('async = 1', """\
Expand Down Expand Up @@ -3230,7 +3227,6 @@ def generate_source(indents):
IndentationError, compile, invalid, "<string>", "exec"
)

@unittest.expectedFailure # TODO: RUSTPYTHON; (0, '')]
def test_continuation_lines_indentation(self):
def get_tokens(string):
the_string = StringIO(string)
Expand Down
196 changes: 165 additions & 31 deletions crates/stdlib/src/_tokenize.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,56 @@ mod _tokenize {

let source_len = TextSize::from(source.len() as u32);

// Ruff has no NL token for a final indentation-only line without a LF.
if extra_tokens
&& *need_implicit_nl
&& !source.is_empty()
&& errors.is_empty()
&& tokens[*index..]
.iter()
.all(|t| matches!(t.kind(), TokenKind::Dedent | TokenKind::EndOfFile))
{
let final_line = last_physical_line(source);
if !final_line.is_empty()
&& final_line
.bytes()
.all(|b| matches!(b, b' ' | b'\t' | b'\x0c'))
{
*need_implicit_nl = false;
let end_lc = line_index.line_column(source_len, source);
let line = end_lc.line.get();
let column = end_lc.column.to_zero_indexed() as isize;
return Ok(PyIterReturn::Return(make_token_tuple(
vm,
TOKEN_NL,
"",
line,
column,
line,
column + 1,
final_line,
)));
}
}

// A comment's implicit NL precedes the terminal DEDENT sequence.
if extra_tokens
&& *need_implicit_nl
&& tokens
.get(*index)
.is_some_and(|t| t.kind() == TokenKind::Dedent)
&& tokens[*index..]
.iter()
.all(|t| matches!(t.kind(), TokenKind::Dedent | TokenKind::EndOfFile))
&& let Some(comment) =
last_non_dedent_token(tokens).filter(|t| t.kind() == TokenKind::Comment)
{
*need_implicit_nl = false;
return Ok(PyIterReturn::Return(implicit_comment_newline(
vm, comment, source, line_index,
)));
}

while *index < tokens.len() {
let token = tokens[*index];
*index += 1;
Expand Down Expand Up @@ -244,7 +294,21 @@ mod _tokenize {
}

let raw_type = token_kind_value(kind);
let token_type = if extra_tokens
let leading_zero_number = extra_tokens
&& kind == TokenKind::Unknown
&& errors.iter().any(|err| {
matches!(
err.error,
ParseErrorType::Lexical(LexicalErrorType::LeadingZerosInDecimalInteger)
) && err.location.start() == range.start()
&& err.location.end() > range.start()
&& err.location.end() < range.end()
});
// Ruff diagnoses the zero prefix but retains the complete numeric token.
// CPython accepts that token as NUMBER only in extra-token mode.
let token_type = if leading_zero_number {
token_kind_value(TokenKind::Int)
} else if extra_tokens
&& (kind == TokenKind::Unknown || (raw_type > TOKEN_DEDENT && raw_type < TOKEN_OP))
{
TOKEN_OP
Expand All @@ -253,15 +317,30 @@ mod _tokenize {
};

let (token_str, start_line, start_col, end_line, end_col, line_str) =
if kind == TokenKind::Dedent {
if !extra_tokens && kind == TokenKind::Indent {
// Explicit continuations can span the indentation range.
// CPython reports the line reached after consuming it.
let line = line_index.line_column(range.end(), source).line.get();
("", line, 0, line, 0, source.full_line_str(range.end()))
} else if kind == TokenKind::Dedent {
let last_line = source.lines().count();
let default_pos = if extra_tokens {
(last_line + 1, 0)
} else {
(last_line, 0)
};
let (pos, dedent_line) =
next_non_dedent_info(tokens, *index, source, line_index, default_pos);
let default_line = if extra_tokens {
""
} else {
non_extra_eof_line(tokens, source, line_index)
};
let (pos, dedent_line) = next_non_dedent_info(
tokens,
*index,
source,
line_index,
(default_pos, default_line),
);
("", pos.0, pos.1, pos.0, pos.1, dedent_line)
} else {
let start_lc = line_index.line_column(range.start(), source);
Expand Down Expand Up @@ -293,7 +372,11 @@ mod _tokenize {
let s = if in_source { &source[range] } else { "" };
(s, end_lc.line.get(), end_lc.column.to_zero_indexed())
};
let line_str = source.full_line_str(range.start());
let line_str = if kind == TokenKind::String && in_source {
source.full_lines_str(range)
} else {
source.full_line_str(range.start())
};
(s, start_line, start_col, el, ec, line_str)
};

Expand Down Expand Up @@ -334,15 +417,16 @@ mod _tokenize {
Some((mid_type, end_line, end_col, line_str.to_string()));
}

// The non-extra tokenizer leaves INDENT/DEDENT token pointers unset.
let (token_str, start_col, end_col) =
if !extra_tokens && matches!(kind, TokenKind::Indent | TokenKind::Dedent) {
("", -1, -1)
} else {
(token_str, start_col as isize, end_col as isize)
};

return Ok(PyIterReturn::Return(make_token_tuple(
vm,
token_type,
token_str,
start_line,
start_col as isize,
end_line,
end_col as isize,
line_str,
vm, token_type, token_str, start_line, start_col, end_line, end_col, line_str,
)));
}

Expand All @@ -354,18 +438,8 @@ mod _tokenize {
.rev()
.find(|t| t.kind() != TokenKind::EndOfFile);
if let Some(last) = last_tok.filter(|t| t.kind() == TokenKind::Comment) {
let end_lc = line_index.line_column(last.range().end(), source);
let nl_line = end_lc.line.get();
let nl_col = end_lc.column.to_zero_indexed();
return Ok(PyIterReturn::Return(make_token_tuple(
vm,
TOKEN_NL,
"",
nl_line,
nl_col as isize,
nl_line,
nl_col as isize + 1,
source.full_line_str(last.range().start()),
return Ok(PyIterReturn::Return(implicit_comment_newline(
vm, last, source, line_index,
)));
}
}
Expand Down Expand Up @@ -404,9 +478,11 @@ mod _tokenize {
let (em_line, em_col, em_line_str): (usize, isize, &str) = if extra_tokens {
(last_line + 1, 0, "")
} else {
let last_line_text =
source.full_line_str(TextSize::from(source.len().saturating_sub(1) as u32));
(last_line, -1, last_line_text)
(
last_line,
-1,
non_extra_eof_line(tokens, source, line_index),
)
};

let result = make_token_tuple(
Expand Down Expand Up @@ -448,27 +524,85 @@ mod _tokenize {
60 // default to FSTRING_MIDDLE
}

fn last_physical_line(source: &str) -> &str {
source.char_indices().next_back().map_or("", |(offset, _)| {
source.full_line_str(TextSize::from(offset as u32))
})
}

fn last_non_dedent_token(tokens: &[Token]) -> Option<&Token> {
tokens
.iter()
.rev()
.find(|t| !matches!(t.kind(), TokenKind::Dedent | TokenKind::EndOfFile))
}

fn non_extra_eof_line<'a>(
tokens: &[Token],
source: &'a str,
line_index: &LineIndex,
) -> &'a str {
let Some(last) = last_non_dedent_token(tokens) else {
return "";
};
if matches!(
last.kind(),
TokenKind::Comment | TokenKind::NonLogicalNewline
) || line_index
.line_column(last.range().start(), source)
.line
.get()
!= source.lines().count()
{
// Ignored trailing lines never populated the non-extra line cache.
""
} else {
last_physical_line(source)
}
}

fn implicit_comment_newline(
vm: &VirtualMachine,
comment: &Token,
source: &str,
line_index: &LineIndex,
) -> PyObjectRef {
let end_lc = line_index.line_column(comment.range().end(), source);
let line = end_lc.line.get();
let column = end_lc.column.to_zero_indexed() as isize;
make_token_tuple(
vm,
TOKEN_NL,
"",
line,
column,
line,
column + 1,
source.full_line_str(comment.range().start()),
)
}

/// Find the next non-DEDENT token's position and source line.
/// Returns ((line, col), line_str).
fn next_non_dedent_info<'a>(
tokens: &[Token],
index: usize,
source: &'a str,
line_index: &LineIndex,
default_pos: (usize, usize),
default_info: ((usize, usize), &'a str),
) -> ((usize, usize), &'a str) {
for future in &tokens[index..] {
match future.kind() {
TokenKind::Dedent => continue,
TokenKind::EndOfFile => return (default_pos, ""),
TokenKind::EndOfFile => return default_info,
_ => {
let flc = line_index.line_column(future.range().start(), source);
let pos = (flc.line.get(), flc.column.to_zero_indexed());
return (pos, source.full_line_str(future.range().start()));
}
}
}
(default_pos, "")
default_info
}

/// Raise a SyntaxError with the given message and position.
Expand Down
Loading