Str(ast::StringLiteral),

Bytes(ast::BytesLiteral),

FString(ast::FString),

TString(ast::TString),

fn range(&self) -> TextRange {

match self {

Self::Str(node) => node.range(),

Self::Bytes(node) => node.range(),

Self::FString(node) => node.range(),

Self::TString(node) => node.range(),

}

}

fn from(string: StringType) -> Self {

match string {

StringType::Str(node) => Expr::from(node),

StringType::Bytes(node) => Expr::from(node),

StringType::FString(node) => Expr::from(node),

StringType::TString(node) => Expr::from(node),

}

}

#[inline]

pub(crate) const fn start_token(self) -> TokenKind {

match self {

InterpolatedStringKind::FString => TokenKind::FStringStart,

InterpolatedStringKind::TString => TokenKind::TStringStart,

}

}

#[inline]

pub(crate) const fn middle_token(self) -> TokenKind {

match self {

InterpolatedStringKind::FString => TokenKind::FStringMiddle,

InterpolatedStringKind::TString => TokenKind::TStringMiddle,

}

}

#[inline]

pub(crate) const fn end_token(self) -> TokenKind {

match self {

InterpolatedStringKind::FString => TokenKind::FStringEnd,

InterpolatedStringKind::TString => TokenKind::TStringEnd,

}

}

fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

match self {

InterpolatedStringKind::FString => f.write_str("f-string"),

InterpolatedStringKind::TString => f.write_str("t-string"),

}

}

/// The raw content of the string e.g., the `foo` part in `"foo"`.

source: &'src str,

/// Current position of the parser in the source.

cursor: usize,

/// Flags that can be used to query information about the string.

flags: AnyStringFlags,

/// The location of the first character in the source from the start of the file.

offset: TextSize,

/// The range of the string literal.

range: TextRange,

fn new(source: &'src str, flags: AnyStringFlags, offset: TextSize, range: TextRange) -> Self {

Self {

source,

cursor: 0,

flags,

offset,

range,

}

}

#[inline]

fn skip_bytes(&mut self, bytes: usize) -> &str {

let skipped_str = &self.source[self.cursor..self.cursor + bytes];

self.cursor += bytes;

skipped_str

}

/// Returns the current position of the parser considering the offset.

#[inline]

fn position(&self) -> TextSize {

self.compute_position(self.cursor)

}

/// Computes the position of the cursor considering the offset.

#[inline]

fn compute_position(&self, cursor: usize) -> TextSize {

self.offset + TextSize::try_from(cursor).unwrap()

}

/// Returns the next byte in the string, if there is one.

///

/// # Panics

///

/// When the next byte is a part of a multi-byte character.

#[inline]

fn next_byte(&mut self) -> Option<u8> {

self.source.as_bytes()[self.cursor..].first().map(|&byte| {

self.cursor += 1;

byte

})

}

#[inline]

fn next_char(&mut self) -> Option<char> {

self.source[self.cursor..].chars().next().inspect(|c| {

self.cursor += c.len_utf8();

})

}

#[inline]

fn peek_byte(&self) -> Option<u8> {

self.source.as_bytes()[self.cursor..].first().copied()

}

#[inline]

fn peek_char(&self) -> Option<char> {

self.source[self.cursor..].chars().next()

}

/// Returns the position of the byte at `index` as the unicode escape decoder sees the

/// content: every non-ASCII character is replaced by its `\UXXXXXXXX` escape, and a

/// backslash that starts a pair before a non-ASCII character or at the end gains `u005c`.

fn escape_position(&self, index: usize) -> u32 {

let width = |c: char| if c.is_ascii() { 1 } else { 10 };

let mut position = 0;

let mut chars = self.source.char_indices().peekable();

while let Some((offset, c)) = chars.next() {

if offset >= index {

break;

}

if c != '\\' {

position += width(c);

continue;

}

position += 1;

let Some(&(next_offset, next)) = chars.peek() else {

position += 5;

break;

};

if !next.is_ascii() {

position += 5;

}

if next_offset >= index {

break;

}

chars.next();

position += width(next);

}

position

}

/// Returns the error for the escape sequence starting at the backslash at `start`, where the

/// decoder stopped before the byte at `end`.

fn escape_error(&self, kind: UnicodeEscapeErrorKind, start: usize, end: usize) -> LexicalError {

let error = if self.flags.is_byte_string() {

LexicalErrorType::BytesEscapeError {

position: u32::try_from(start).unwrap(),

}

} else {

LexicalErrorType::UnicodeEscapeError {

kind,

start: self.escape_position(start),

end: self.escape_position(end) - 1,

}

};

LexicalError::new(error, self.range)

}

fn parse_unicode_literal(

&mut self,

literal_number: usize,

escape_start: usize,

) -> Result<char, LexicalError> {

let kind = match literal_number {

2 => UnicodeEscapeErrorKind::TruncatedHexByte,

4 => UnicodeEscapeErrorKind::TruncatedShortUnicode,

_ => UnicodeEscapeErrorKind::TruncatedLongUnicode,

};

let mut p: u32 = 0u32;

for i in 1..=literal_number {

let Some(d) = self.peek_char().and_then(|c| c.to_digit(16)) else {

return Err(self.escape_error(kind, escape_start, self.cursor));

};

self.cursor += 1;

p += d << ((literal_number - i) * 4);

}

match p {

0xD800..=0xDFFF => Ok(std::char::REPLACEMENT_CHARACTER),

_ => std::char::from_u32(p).ok_or_else(|| {

self.escape_error(

UnicodeEscapeErrorKind::IllegalCharacter,

escape_start,

self.cursor,

)

}),

}

}

fn parse_octet(&mut self, o: u8) -> char {

let mut radix_bytes = [o, 0, 0];

let mut len = 1;

while len < 3 {

let Some(b'0'..=b'7') = self.peek_byte() else {

break;

};

radix_bytes[len] = self.next_byte().unwrap();

len += 1;

}

// OK because radix_bytes is always going to be in the ASCII range.

let radix_str = std::str::from_utf8(&radix_bytes[..len]).expect("ASCII bytes");

let value = u32::from_str_radix(radix_str, 8).unwrap();

char::from_u32(value).unwrap()

}

fn parse_unicode_name(&mut self, escape_start: usize) -> Result<char, LexicalError> {

if self.peek_char() != Some('{') {

return Err(self.escape_error(

UnicodeEscapeErrorKind::MalformedName,

escape_start,

self.cursor,

));

}

self.cursor += 1;

let Some(close_idx) = self.source[self.cursor..].find('}') else {

return Err(self.escape_error(

UnicodeEscapeErrorKind::MalformedName,

escape_start,

self.source.len(),

));

};

if close_idx == 0 {

return Err(self.escape_error(

UnicodeEscapeErrorKind::MalformedName,

escape_start,

self.cursor,

));

}

let name_and_ending = self.skip_bytes(close_idx + 1);

let name = &name_and_ending[..name_and_ending.len() - 1];

unicode_names2::character(name).ok_or_else(|| {

self.escape_error(

UnicodeEscapeErrorKind::UnknownName,

escape_start,

self.cursor,

)

})

}

/// Parse an escaped character, returning the new character.

fn parse_escaped_char(&mut self) -> Result<Option<EscapedChar>, LexicalError> {

let escape_start = self.cursor - 1;

let Some(first_char) = self.next_char() else {

// TODO: check when this error case happens

return Err(LexicalError::new(

LexicalErrorType::StringError,

TextRange::empty(self.position()),

));

};

let new_char = match first_char {

'\\' => '\\',

'\'' => '\'',

'\"' => '"',

'a' => '\x07',

'b' => '\x08',

'f' => '\x0c',

'n' => '\n',

'r' => '\r',

't' => '\t',

'v' => '\x0b',

o @ '0'..='7' => self.parse_octet(o as u8),

'x' => self.parse_unicode_literal(2, escape_start)?,

'u' if !self.flags.is_byte_string() => self.parse_unicode_literal(4, escape_start)?,

'U' if !self.flags.is_byte_string() => self.parse_unicode_literal(8, escape_start)?,

'N' if !self.flags.is_byte_string() => self.parse_unicode_name(escape_start)?,

// Special cases where the escape sequence is not a single character

'\n' => return Ok(None),

'\r' => {

if self.peek_byte() == Some(b'\n') {

self.next_byte();

}

return Ok(None);

}

_ => return Ok(Some(EscapedChar::Escape(first_char))),

};

Ok(Some(EscapedChar::Literal(new_char)))

}

fn parse_interpolated_string_middle(

mut self,

) -> Result<ast::InterpolatedStringLiteralElement, LexicalError> {

// Fast-path: if the f-string or t-string doesn't contain any escape sequences, return the literal.

let Some(mut index) = memchr::memchr3(b'{', b'}', b'\\', self.source.as_bytes()) else {

return Ok(ast::InterpolatedStringLiteralElement {

value: self.source.into(),

range: self.range,

node_index: AtomicNodeIndex::NONE,

});

};

let mut value = String::with_capacity(self.source.len());

loop {

// Add the characters before the escape sequence (or curly brace) to the string.

let before_with_slash_or_brace = self.skip_bytes(index + 1);

let before = &before_with_slash_or_brace[..before_with_slash_or_brace.len() - 1];

value.push_str(before);

// Add the escaped character to the string.

match self.source.as_bytes()[self.cursor - 1] {

// If there are any curly braces inside a `F/TStringMiddle` token,

// then they were escaped (i.e. `{{` or `}}`). This means that

// the raw source contains a doubled brace, but the literal value only

// contains one brace.

brace @ (b'{' | b'}') => {

if self.peek_byte() == Some(brace) {

self.next_byte();

}

value.push(char::from(brace));

}

// We can encounter a `\` as the last character in a `F/TStringMiddle`

// token which is valid in this context. For example,

//

// ```python

// f"\{foo} \{bar:\}"

// # ^ ^^ ^

// ```

//

// Here, the `F/TStringMiddle` token content will be "\" and " \"

// which is invalid if we look at the content in isolation:

//

// ```python

// "\"

// ```

//

// However, the content is syntactically valid in the context of

// the f/t-string because it's a substring of the entire f/t-string.

// This is still an invalid escape sequence, but we don't want to

// raise a syntax error as is done by the CPython parser. It might

// be supported in the future, refer to point 3: https://peps.python.org/pep-0701/#rejected-ideas

b'\\' => {

if !self.flags.is_raw_string() && self.peek_byte().is_some() {

if let Some(brace @ (b'{' | b'}')) = self.peek_byte()

&& self.source.as_bytes().get(self.cursor + 1).copied() == Some(brace)

{

// Leave the doubled brace for the next iteration to collapse.

value.push('\\');

} else {

match self.parse_escaped_char()? {

None => {}

Some(EscapedChar::Literal(c)) => value.push(c),

Some(EscapedChar::Escape(c)) => {

value.push('\\');

value.push(c);

}

}

}

} else {

value.push('\\');

}

}

ch => {

unreachable!("Expected '{{', '}}', or '\\' but got {:?}", ch);

}

}

let Some(next_index) =

memchr::memchr3(b'{', b'}', b'\\', &self.source.as_bytes()[self.cursor..])

else {

// Add the rest of the string to the value.

let rest = &self.source[self.cursor..];

value.push_str(rest);

break;

};

index = next_index;

}

Ok(ast::InterpolatedStringLiteralElement {

value: value.into_boxed_str(),

range: self.range,

node_index: AtomicNodeIndex::NONE,

})

}

fn parse_bytes(mut self) -> Result<StringType, LexicalError> {

if self.source.as_bytes().find_non_ascii_byte().is_some() {

return Err(LexicalError::new(

LexicalErrorType::InvalidByteLiteral,

self.range,

));

}

if self.flags.is_raw_string() {

// For raw strings, no escaping is necessary.

return Ok(StringType::Bytes(ast::BytesLiteral {

value: self.source.as_bytes().into(),

range: self.range,

flags: self.flags.into(),

node_index: AtomicNodeIndex::NONE,

}));

}

let Some(mut escape) = memchr::memchr(b'\\', self.source.as_bytes()) else {

// If the string doesn't contain any escape sequences, return the owned string.

return Ok(StringType::Bytes(ast::BytesLiteral {

value: self.source.as_bytes().into(),

range: self.range,

flags: self.flags.into(),

node_index: AtomicNodeIndex::NONE,

}));

};

// If the string contains escape sequences, we need to parse them.

let mut value = Vec::with_capacity(self.source.len());

loop {

// Add the characters before the escape sequence to the string.

let before_with_slash = self.skip_bytes(escape + 1);

let before = &before_with_slash[..before_with_slash.len() - 1];

value.extend_from_slice(before.as_bytes());

// Add the escaped character to the string.

match self.parse_escaped_char()? {

None => {}

Some(EscapedChar::Literal(c)) => value.push(c as u8),

Some(EscapedChar::Escape(c)) => {

value.push(b'\\');

value.push(c as u8);

}

}

let Some(next_escape) = memchr::memchr(b'\\', &self.source.as_bytes()[self.cursor..])

else {

// Add the rest of the string to the value.

let rest = &self.source[self.cursor..];

value.extend_from_slice(rest.as_bytes());

break;

};

// Update the position of the next escape sequence.

escape = next_escape;

}

Ok(StringType::Bytes(ast::BytesLiteral {

value: value.into_boxed_slice(),

range: self.range,

flags: self.flags.into(),

node_index: AtomicNodeIndex::NONE,

}))

}

fn parse_string(mut self) -> Result<StringType, LexicalError> {

if self.flags.is_raw_string() {

// For raw strings, no escaping is necessary.

return Ok(StringType::Str(ast::StringLiteral {

value: self.source.into(),

range: self.range,

flags: self.flags.into(),

node_index: AtomicNodeIndex::NONE,

}));

}

let Some(mut escape) = memchr::memchr(b'\\', self.source.as_bytes()) else {

// If the string doesn't contain any escape sequences, return the owned string.

return Ok(StringType::Str(ast::StringLiteral {

value: self.source.into(),

range: self.range,

flags: self.flags.into(),

node_index: AtomicNodeIndex::NONE,

}));

};

// If the string contains escape sequences, we need to parse them.

let mut value = String::with_capacity(self.source.len());

loop {

// Add the characters before the escape sequence to the string.

let before_with_slash = self.skip_bytes(escape + 1);

let before = &before_with_slash[..before_with_slash.len() - 1];

value.push_str(before);

// Add the escaped character to the string.

match self.parse_escaped_char()? {

None => {}

Some(EscapedChar::Literal(c)) => value.push(c),

Some(EscapedChar::Escape(c)) => {

value.push('\\');

value.push(c);

}

}

let Some(next_escape) = self.source[self.cursor..].find('\\') else {

// Add the rest of the string to the value.

let rest = &self.source[self.cursor..];

value.push_str(rest);

break;

};

// Update the position of the next escape sequence.

escape = next_escape;

}

Ok(StringType::Str(ast::StringLiteral {

value: value.into_boxed_str(),

range: self.range,

flags: self.flags.into(),

node_index: AtomicNodeIndex::NONE,

}))

}

fn parse(self) -> Result<StringType, LexicalError> {

if self.flags.is_byte_string() {

self.parse_bytes()

} else {

self.parse_string()

}

}

source: &str,

flags: AnyStringFlags,

range: TextRange,

) -> Result<StringType, LexicalError> {

StringParser::new(source, flags, range.start() + flags.opener_len(), range).parse()

source: &str,

flags: AnyStringFlags,

range: TextRange,

) -> Result<ast::InterpolatedStringLiteralElement, LexicalError> {

StringParser::new(source, flags, range.start(), range).parse_interpolated_string_middle()

use ruff_python_ast::Suite;

use crate::error::LexicalErrorType;

use crate::{InterpolatedStringErrorType, ParseError, ParseErrorType, Parsed, parse_module};

const WINDOWS_EOL: &str = "\r\n";

const MAC_EOL: &str = "\r";

const UNIX_EOL: &str = "\n";

fn parse_suite(source: &str) -> Result<Suite, ParseError> {

parse_module(source).map(Parsed::into_suite)

}

fn nested_format_spec(prefix: char, depth: usize) -> String {

let mut replacement_field = String::from("{spec}");

for _ in 0..depth {

replacement_field = format!("{{foo:{replacement_field}}}");

}

format!(r#"{prefix}"{replacement_field}""#)

}

fn string_parser_escaped_eol(eol: &str) -> Suite {

let source = format!(r"'text \{eol}more text'");

parse_suite(&source).unwrap()

}

#[test]

fn test_string_parser_escaped_unix_eol() {

let suite = string_parser_escaped_eol(UNIX_EOL);

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_string_parser_escaped_mac_eol() {

let suite = string_parser_escaped_eol(MAC_EOL);

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_string_parser_escaped_windows_eol() {

let suite = string_parser_escaped_eol(WINDOWS_EOL);

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_fstring() {

let source = r#"f"{a}{ b }{{foo}}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_fstring_nested_spec() {

let source = r#"f"{foo:{spec}}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn parse_fstring_nested_spec_grows_stack() {

let error = parse_suite(&nested_format_spec('f', 200)).unwrap_err();

assert_eq!(

error.error,

ParseErrorType::Lexical(LexicalErrorType::TooDeeplyNestedBrackets)

);

}

#[test]

fn test_parse_fstring_not_nested_spec() {

let source = r#"f"{foo:spec}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_empty_fstring() {

let source = r#"f"""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_fstring_parse_self_documenting_base() {

let source = r#"f"{user=}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_fstring_parse_self_documenting_base_more() {

let source = r#"f"mix {user=} with text and {second=}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_fstring_parse_self_documenting_format() {

let source = r#"f"{user=:>10}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

fn parse_fstring_error(source: &str) -> InterpolatedStringErrorType {

parse_suite(source)

.map_err(|e| match e.error {

ParseErrorType::Lexical(LexicalErrorType::FStringError(e)) => e,

ParseErrorType::FStringError(e) => e,

e => unreachable!("Expected FStringError: {:?}", e),

})

.expect_err("Expected error")

}

#[test]

fn test_parse_invalid_fstring() {

use InterpolatedStringErrorType::{InvalidConversionFlag, LambdaWithoutParentheses};

assert_eq!(parse_fstring_error(r#"f"{5!1}""#), InvalidConversionFlag);

assert_eq!(

parse_fstring_error("f'{lambda x:{x}}'"),

LambdaWithoutParentheses

);

// NOTE: The parser produces the `LambdaWithoutParentheses` for this case, but

// since the parser only return the first error to maintain compatibility with

// the rest of the codebase, this test case fails. The `LambdaWithoutParentheses`

// error appears after the unexpected `FStringMiddle` token, which is between the

// `:` and the `{`.

// assert_eq!(parse_fstring_error("f'{lambda x: {x}}'"), LambdaWithoutParentheses);

assert!(parse_suite(r#"f"{class}""#).is_err());

}

#[test]

fn test_parse_fstring_not_equals() {

let source = r#"f"{1 != 2}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_fstring_equals() {

let source = r#"f"{42 == 42}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_fstring_self_doc_prec_space() {

let source = r#"f"{x =}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_fstring_self_doc_trailing_space() {

let source = r#"f"{x= }""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_fstring_yield_expr() {

let source = r#"f"{yield}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_tstring() {

let source = r#"t"{a}{ b }{{foo}}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_tstring_nested_spec() {

let source = r#"t"{foo:{spec}}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn parse_tstring_nested_spec_grows_stack() {

let error = parse_suite(&nested_format_spec('t', 200)).unwrap_err();

assert_eq!(

error.error,

ParseErrorType::Lexical(LexicalErrorType::TooDeeplyNestedBrackets)

);

}

#[test]

fn test_parse_tstring_not_nested_spec() {

let source = r#"t"{foo:spec}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_empty_tstring() {

let source = r#"t"""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_tstring_parse_self_documenting_base() {

let source = r#"t"{user=}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_tstring_parse_self_documenting_base_more() {

let source = r#"t"mix {user=} with text and {second=}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_tstring_parse_self_documenting_format() {

let source = r#"t"{user=:>10}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

fn parse_tstring_error(source: &str) -> InterpolatedStringErrorType {

parse_suite(source)

.map_err(|e| match e.error {

ParseErrorType::Lexical(LexicalErrorType::TStringError(e)) => e,

ParseErrorType::TStringError(e) => e,

e => unreachable!("Expected TStringError: {:?}", e),

})

.expect_err("Expected error")

}

#[test]

fn test_parse_invalid_tstring() {

use InterpolatedStringErrorType::{InvalidConversionFlag, LambdaWithoutParentheses};

assert_eq!(parse_tstring_error(r#"t"{5!1}""#), InvalidConversionFlag);

assert_eq!(

parse_tstring_error("t'{lambda x:{x}}'"),

LambdaWithoutParentheses

);

// NOTE: The parser produces the `LambdaWithoutParentheses` for this case, but

// since the parser only return the first error to maintain compatibility with

// the rest of the codebase, this test case fails. The `LambdaWithoutParentheses`

// error appears after the unexpected `tStringMiddle` token, which is between the

// `:` and the `{`.

// assert_eq!(parse_tstring_error("f'{lambda x: {x}}'"), LambdaWithoutParentheses);

assert!(parse_suite(r#"t"{class}""#).is_err());

}

#[test]

fn test_parse_tstring_not_equals() {

let source = r#"t"{1 != 2}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_tstring_equals() {

let source = r#"t"{42 == 42}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_tstring_self_doc_prec_space() {

let source = r#"t"{x =}""#;

let suite = parse_suite(source).unwrap();

insta::assert_debug_snapshot!(suite);

}

#[test]

fn test_parse_tstring_self_doc_trailing_space() {