Skip to content

Respect single-input boundaries when emitting parser warnings #9304

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 1 commit into from
Oct 23, 2026
Merged

Respect single-input boundaries when emitting parser warnings #9304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 34 additions & 5 deletions crates/compiler/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1279,6 +1279,35 @@ fn first_parse_error(
Err(error)
}

/// The first logical input accepted by the single-input grammar.
#[derive(Clone, Copy)]
pub enum SingleModeStatementBoundary {
Simple { newline_index: usize, end: TextSize },
Compound { end: TextSize },
}

/// Classify a logical input without changing parser diagnostics or recovery.
#[must_use]
pub fn single_mode_statement_boundary(
parsed: &parser::Parsed<ast::Mod>,
) -> Option<SingleModeStatementBoundary> {
let ast::Mod::Module(module) = parsed.syntax() else {
return None;
};
let first = module.body.first()?;
if is_compound_stmt(first) {
return Some(SingleModeStatementBoundary::Compound { end: first.end() });
}
let newline_index = parsed
.tokens()
.iter()
.position(|token| token.kind() == TokenKind::Newline)?;
Some(SingleModeStatementBoundary::Simple {
newline_index,
end: parsed.tokens()[newline_index].end(),
})
}

/// Single mode reads only the first statement when it is a simple statement, so any code after
/// its line is reported as multiple statements, ahead of errors the rest of the source has.
#[must_use]
Expand All @@ -1289,8 +1318,8 @@ pub fn single_mode_multiple_statements_error(
let ast::Mod::Module(module) = parsed.syntax() else {
return None;
};
let first = module.body.first()?;
if is_compound_stmt(first) {
let boundary = single_mode_statement_boundary(parsed)?;
if matches!(boundary, SingleModeStatementBoundary::Compound { .. }) {
// The interactive grammar ends a compound statement with a NEWLINE,
// so the first token of a following statement is invalid syntax.
let next = module.body.get(1)?;
Expand All @@ -1315,9 +1344,9 @@ pub fn single_mode_multiple_statements_error(
));
}
let tokens = parsed.tokens();
let newline_index = tokens
.iter()
.position(|token| token.kind() == TokenKind::Newline)?;
let SingleModeStatementBoundary::Simple { newline_index, .. } = boundary else {
unreachable!()
};
let newline = &tokens[newline_index];
if parsed
.errors()
Expand Down
62 changes: 57 additions & 5 deletions crates/vm/src/vm/compile.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1738,16 +1738,19 @@ mod escape_warnings {
warnings: Vec<ParserWarning>,
tokenizer_recovery: bool,
has_valid_syntax: bool,
single_boundary: Option<compiler::SingleModeStatementBoundary>,
}

fn collect_parser_warnings(
source: &str,
filename: &str,
mode: Option<ruff_python_parser::Mode>,
inherited_barry_as_flufl: bool,
single_mode: bool,
) -> CollectedParserWarnings {
let mut ast_tokenizer_recovery = true;
let mut has_valid_syntax = false;
let mut single_boundary = None;
let mut warnings = Vec::new();
collect_numeric_literal_warnings(source, &mut warnings);
// The compile that follows rejects this source; parsing it here
Expand All @@ -1765,6 +1768,9 @@ mod escape_warnings {
let parsed = ruff_python_parser::parse_unchecked(barry_source.source(), options);
if parsed.has_valid_syntax() {
has_valid_syntax = true;
if single_mode {
single_boundary = compiler::single_mode_statement_boundary(&parsed);
}
let mut visitor = EscapeWarningVisitor {
source,
warnings: &mut warnings,
Expand Down Expand Up @@ -1837,6 +1843,7 @@ mod escape_warnings {
warnings,
tokenizer_recovery: ast_tokenizer_recovery,
has_valid_syntax,
single_boundary,
}
}

Expand All @@ -1861,14 +1868,20 @@ mod escape_warnings {
inherited_barry_as_flufl: bool,
emitted_escapes: &[usize],
) -> Result<(), CompileWarningError> {
let is_single = matches!(mode, compiler::Mode::Single);
let mode = match mode {
compiler::Mode::Eval => ruff_python_parser::Mode::Expression,
compiler::Mode::Exec | compiler::Mode::Single | compiler::Mode::BlockExpr => {
ruff_python_parser::Mode::Module
}
};
let collected =
collect_parser_warnings(source, filename, Some(mode), inherited_barry_as_flufl);
let collected = collect_parser_warnings(
source,
filename,
Some(mode),
inherited_barry_as_flufl,
is_single,
);
if !collected.has_valid_syntax || !collected.tokenizer_recovery {
// Invalid source and interpolated tokenizer warnings need a
// separate recovery contract. Preserve their existing behavior.
Expand All @@ -1880,9 +1893,10 @@ mod escape_warnings {
filename,
module,
collected.warnings,
collected.single_boundary,
WarningEmission::Standard {
emitted_escapes,
tokenizer_recovery: collected.tokenizer_recovery,
tokenizer_recovery: true,
},
)
.map(|_| ())
Expand All @@ -1902,6 +1916,7 @@ mod escape_warnings {
filename,
module,
warnings,
None,
WarningEmission::Standard {
emitted_escapes: &[],
tokenizer_recovery: false,
Expand Down Expand Up @@ -1954,6 +1969,7 @@ mod escape_warnings {
filename,
module,
warnings,
None,
WarningEmission::Standard {
emitted_escapes,
tokenizer_recovery: false,
Expand All @@ -1975,13 +1991,26 @@ mod escape_warnings {
CompileStart::Eval => Some(ruff_python_parser::Mode::Expression),
CompileStart::FuncType => None,
};
let collected =
collect_parser_warnings(source, filename, mode, inherited_barry_as_flufl);
let collected = collect_parser_warnings(
source,
filename,
mode,
inherited_barry_as_flufl,
matches!(start, CompileStart::Single),
);
self.emit_parser_warning_batch(
source,
filename,
module.map(AsObject::as_object),
collected.warnings,
if matches!(start, CompileStart::Single)
&& collected.has_valid_syntax
&& collected.tokenizer_recovery
{
collected.single_boundary
} else {
None
},
WarningEmission::Ast {
tokenizer_recovery: collected.tokenizer_recovery
&& matches!(start, CompileStart::File | CompileStart::Eval),
Expand All @@ -2003,12 +2032,14 @@ mod escape_warnings {
filename,
Some(ruff_python_parser::Mode::Module),
inherited_barry_as_flufl,
false,
);
self.emit_parser_warning_batch(
source,
filename,
module.map(AsObject::as_object),
collected.warnings,
None,
WarningEmission::Interactive(emitted),
)
.map(|_| ())
Expand All @@ -2020,6 +2051,7 @@ mod escape_warnings {
filename: &str,
module: Option<&PyObject>,
warnings: Vec<ParserWarning>,
single_boundary: Option<compiler::SingleModeStatementBoundary>,
emission: WarningEmission<'_>,
) -> Result<Vec<usize>, CompileWarningError> {
let (emitted_escapes, ast_tokenizer_recovery, mut interactive) = match emission {
Expand All @@ -2033,6 +2065,26 @@ mod escape_warnings {
let mut considered_escapes = Vec::new();
let mut parser_error = None;
for warning in warnings {
// A successful single input does not parse later statements.
// Keep those numeric events for recovery after an earlier
// parser warning escalates, rather than truncating the source.
if parser_error.is_none()
&& let Some(boundary) = single_boundary
{
let end = match (boundary, &warning) {
(compiler::SingleModeStatementBoundary::Simple { end, .. }, _) => Some(end),
(
compiler::SingleModeStatementBoundary::Compound { end },
ParserWarning::Escape(_, EscapeOrigin::Parser, _),
) => Some(end),
// Compound input requests another token and can emit
// tokenizer warnings while rejecting the next statement.
_ => None,
};
if end.is_some_and(|end| warning.offset() >= end.to_usize()) {
continue;
}
}
// CPython resumes tokenization after a parser-origin SyntaxError.
// Preserve ordinary warning order and never replay prior warnings.
if parser_error.is_some() && !matches!(warning, ParserWarning::Numeric { .. }) {
Expand Down
Loading