Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 45 additions & 8 deletions packages/compiler/src/expression_parser/lexer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -612,23 +612,60 @@ class _Scanner {
): string | (Token & {type: TokenType.Error}) {
buffer += this.input.substring(marker, this.index);
let unescapedCode: number;
let unicodeEscapeForError: string | null = null;
this.advance();
if (this.peek === chars.$u) {
// 4 character hex code for unicode character.
const hex: string = this.input.substring(this.index + 1, this.index + 5);
if (/^[0-9a-f]+$/i.test(hex)) {
// Unicode escape: \uXXXX or \u{XXXX}
this.advance(); // skip 'u'
if (this.input.charCodeAt(this.index) === chars.$LBRACE) {
// Braced unicode escape: \u{X} to \u{XXXXXX}
this.advance(); // skip '{'
const hexStart = this.index;
while (this.index < this.length) {
const ch = this.input.charCodeAt(this.index);
if (ch === chars.$RBRACE) break;
this.advance();
}

const hasClosingBrace = this.input.charCodeAt(this.index) === chars.$RBRACE;
const hex = this.input.substring(hexStart, this.index);
unicodeEscapeForError = hasClosingBrace ? `\\u{${hex}}` : `\\u{${hex}`;

if (hasClosingBrace) {
this.advance(); // skip '}'
}

if (!hasClosingBrace || !/^[0-9a-f]{1,6}$/i.test(hex)) {
return this.error(`Invalid unicode escape [${unicodeEscapeForError}]`, 0);
}

unescapedCode = parseInt(hex, 16);
if (unescapedCode > 0x10ffff) {
return this.error(`Invalid unicode escape [${unicodeEscapeForError}]`, 0);
}
} else {
return this.error(`Invalid unicode escape [\\u${hex}]`, 0);
}
for (let i = 0; i < 5; i++) {
this.advance();
// 4 character hex code for unicode character: \uXXXX
const hex: string = this.input.substring(this.index, this.index + 4);
unicodeEscapeForError = `\\u${hex}`;
if (/^[0-9a-f]{4}$/i.test(hex)) {
unescapedCode = parseInt(hex, 16);
} else {
return this.error(`Invalid unicode escape [${unicodeEscapeForError}]`, 0);
}
for (let i = 0; i < 4; i++) {
this.advance();
}
}
} else {
unescapedCode = unescape(this.peek);
this.advance();
}
buffer += String.fromCharCode(unescapedCode);
if (unescapedCode > 0xffff) {
// Encode as surrogate pair for code points above BMP.
buffer += String.fromCodePoint(unescapedCode);
} else {
buffer += String.fromCharCode(unescapedCode);
}
return buffer;
}

Expand Down
9 changes: 6 additions & 3 deletions packages/compiler/src/expression_parser/parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1815,9 +1815,12 @@ class _ParseAST {
* `this.skip` for more details on token skipping.
*/
private error(message: string, index = this.index) {
this.errors.push(
getParseError(message, this.input, this.getErrorLocationText(index), this.parseSourceSpan),
);
const duplicateOfLexerError = this.next.isError() && message.startsWith('Unexpected token ');
if (!duplicateOfLexerError) {
this.errors.push(
getParseError(message, this.input, this.getErrorLocationText(index), this.parseSourceSpan),
);
}
this.skip();
}

Expand Down
39 changes: 36 additions & 3 deletions packages/compiler/test/expression_parser/lexer_spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,18 @@ describe('lexer', () => {
expect(tokens[0].toString()).toEqual('\u00a0');
});

it('should tokenize braced unicode', () => {
const tokens: Token[] = lex('"\\u{4f60}"');
expect(tokens.length).toEqual(1);
expect(tokens[0].toString()).toEqual('你');
});

it('should tokenize braced unicode above BMP', () => {
const tokens: Token[] = lex('"\\u{1F680}"');
expect(tokens.length).toEqual(1);
expect(tokens[0].toString()).toEqual('🚀');
});

it('should tokenize relation', () => {
const tokens: Token[] = lex('! == != < > <= >= === !==');
expectOperatorToken(tokens[0], 0, 1, '!');
Expand Down Expand Up @@ -367,12 +379,27 @@ describe('lexer', () => {
it('should throw error on invalid unicode', () => {
expectErrorToken(
lex("'\\u1''bla'")[0],
2,
2,
"Lexer Error: Invalid unicode escape [\\u1''b] at column 2 in expression ['\\u1''bla']",
3,
3,
"Lexer Error: Invalid unicode escape [\\u1''b] at column 3 in expression ['\\u1''bla']",
);
});

it('should throw error on unterminated braced unicode', () => {
expectErrorToken(
lex('"\\u{1F600"')[0],
10,
10,
'Lexer Error: Invalid unicode escape [\\u{1F600"] at column 10 in expression ["\\u{1F600"]',
);
});

it('should throw error on out of range braced unicode', () => {
const token = lex('"\\u{110000}"')[0];
expect(token.isError()).toBe(true);
expect(token.toString()).toContain('Invalid unicode escape [\\u{110000}]');
});

it('should tokenize ?. as operator', () => {
expectOperatorToken(lex('?.')[0], 0, 2, '?.');
});
Expand Down Expand Up @@ -491,6 +518,12 @@ describe('lexer', () => {
expect(tokens[0].toString()).toBe('\u00a0');
});

it('should tokenize braced unicode inside a template string', () => {
const tokens: Token[] = lex('`\\u{1F680}`');
expect(tokens.length).toBe(1);
expect(tokens[0].toString()).toBe('🚀');
});

it('should tokenize template literal with an interpolation in the end', () => {
const tokens: Token[] = lex('`hello ${name}`');
expect(tokens.length).toBe(5);
Expand Down
26 changes: 26 additions & 0 deletions packages/compiler/test/expression_parser/parser_spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1024,6 +1024,32 @@ describe('parser', () => {
expect((map.keys[0] as LiteralMapPropertyKey).isShorthandInitialized).toBe(true);
});

describe('unicode braced escapes', () => {
it('should parse braced unicode escape in strings', () => {
checkBinding('"\\u{4f60}"', '"你"');
});

it('should parse braced unicode escape for emoji', () => {
checkBinding('"\\u{1F600}"', '"😀"');
});

it('should still parse 4-digit unicode escapes', () => {
checkBinding('"\\u4f60"', '"你"');
});

it('should parse multiple braced unicode escapes in sequence', () => {
checkBinding('"\\u{48}\\u{65}\\u{6C}\\u{6C}\\u{6F}"', '"Hello"');
});

it('should error for unterminated braced unicode escape', () => {
expectBindingError('"\\u{1F600"', 'Invalid unicode escape [\\u{1F600');
});

it('should error for out of range braced unicode escape', () => {
expectBindingError('"\\u{110000}"', 'Invalid unicode escape [\\u{110000}]');
});
});

describe('arrow functions', () => {
it('should parse a single-parameter arrow function', () => {
checkBinding('a => a');
Expand Down