Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 65 additions & 5 deletions crates/vm/src/stdlib/_ast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1233,6 +1233,44 @@ fn invalid_type_comment_syntax_error(
None
}

/// The parser phase where a rejected TYPE_COMMENT stops grammar actions.
/// Use the validator's token-backed location, not a textual comment search.
pub(crate) fn invalid_type_comment_warning_boundary(
source_file: &SourceFile,
parsed: &parser::Parsed<ast::Mod>,
interactive: bool,
) -> Option<usize> {
let comments = TypeCommentSource::new(source_file.source_text(), parsed.tokens());
match invalid_type_comment_syntax_error(source_file, &comments)? {
CompileError::Parse(error) => {
let offset = error.raw_location.start().to_usize();
(!interactive || type_comment_precedes_single_boundary(offset, source_file, parsed))
.then_some(offset)
}
CompileError::Codegen(_) => None,
}
}

fn type_comment_precedes_single_boundary(
offset: usize,
source_file: &SourceFile,
parsed: &parser::Parsed<ast::Mod>,
) -> bool {
use rustpython_compiler::SingleModeStatementBoundary;
match rustpython_compiler::single_mode_statement_boundary(parsed) {
Some(SingleModeStatementBoundary::Simple { end, .. }) => offset < end.to_usize(),
Some(SingleModeStatementBoundary::Compound { .. }) => {
// Compound input still requests NEWLINE after the statement. A
// rejected comment may be that next token, before the next AST node.
match rustpython_compiler::single_mode_multiple_statements_error(source_file, parsed) {
Some(CompileError::Parse(error)) => offset < error.raw_location.start().to_usize(),
_ => true,
}
}
None => true,
}
}

fn feature_version_syntax_error(
source: &str,
source_file: &SourceFile,
Expand Down Expand Up @@ -1769,6 +1807,7 @@ pub(crate) fn parse<E: From<CompileError>>(
interactive: bool,
explicit_future_features: crate::bytecode::CodeFlags,
dont_imply_dedent: bool,
type_comment_error_offset: Option<&mut Option<usize>>,
mut emit_warning: impl FnMut(usize, char) -> Result<(), E>,
) -> Result<PyObjectRef, E> {
let source_file = SourceFileBuilder::new(filename.to_owned(), source.to_owned()).finish();
Expand All @@ -1785,17 +1824,38 @@ pub(crate) fn parse<E: From<CompileError>>(
);
rustpython_compiler::pre_parse_source_error(&source_file)?;
let parsed = parser::parse_unchecked(barry_source.source(), options);
let type_comment_source =
type_comments.then(|| TypeCommentSource::new(source, parsed.tokens()));
let mut type_comment_error = type_comment_source
.as_ref()
.and_then(|lines| invalid_type_comment_syntax_error(&source_file, lines));
if interactive && parsed.has_valid_syntax() {
// A rejected comment before the first logical-input boundary prevents
// Simple input from succeeding. Compound input still needs NEWLINE,
// so compare its actual next-token error rather than its AST end.
type_comment_error = type_comment_error.filter(|error| match error {
CompileError::Parse(error) => type_comment_precedes_single_boundary(
error.raw_location.start().to_usize(),
&source_file,
&parsed,
),
CompileError::Codegen(_) => false,
});
}
if interactive
&& !(parsed.has_valid_syntax() && type_comment_error.is_some())
&& let Some(error) =
rustpython_compiler::single_mode_multiple_statements_error(&source_file, &parsed)
{
return Err(error.into());
}
let type_comment_source =
type_comments.then(|| TypeCommentSource::new(source, parsed.tokens()));
if let Some(lines) = &type_comment_source
&& let Some(error) = invalid_type_comment_syntax_error(&source_file, lines)
{
if let Some(error) = type_comment_error {
if parsed.has_valid_syntax()
&& let Some(offset) = type_comment_error_offset
&& let CompileError::Parse(error) = &error
{
*offset = Some(error.raw_location.start().to_usize());
}
return Err(error.into());
}
if let Err(errors) = parsed.as_result() {
Expand Down
3 changes: 3 additions & 0 deletions crates/vm/src/stdlib/_ast/string.rs
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,9 @@ fn warn_invalid_escape_sequences_in_format_spec<E>(
}
true
}
// CPython's string-parser prepass protects a backslash before a
// non-ASCII codepoint before invoking the escape decoder.
ch if !ch.is_ascii() => true,
_ => false,
};
if !valid {
Expand Down
Loading
Loading