-
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathLexer.php
More file actions
131 lines (114 loc) · 4.89 KB
/
Copy pathLexer.php
File metadata and controls
131 lines (114 loc) · 4.89 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
<?php
declare(strict_types=1);
namespace PhpScript\Core;
use PhpScript\Contracts\LexerInterface;
use PhpScript\Exceptions\LexerException;
final readonly class Lexer implements LexerInterface
{
/**
* An associative array defining token patterns for a lexical analyzer or tokenizer.
* Each key corresponds to a specific token type, and its value is the regex pattern
* to recognize that type in the input string.
*
* Token types include:
* - Whitespace and comments
* - Numbers and strings
* - Programming language keywords
* - Identifiers (e.g., variables and functions)
* - Common operators and symbols (e.g., `=`, `+`, `{`, `}`)
*
* @var array<string, string>
*/
private array $tokenPatterns;
public function __construct()
{
// initialize token patterns in defined order
$this->tokenPatterns = [
TokenType::T_WHITESPACE->value => '[\s\x{00A0}]+',
TokenType::T_COMMENT->value => '\/\/[^\n]*',
TokenType::T_NUMBER->value => '\b\d+(\.\d+)?\b',
TokenType::T_STRING->value => '\"(.*?)(?<!\\\\)\"|\'(.*?)(?<!\\\\)\'',
// Keywords (have to be before T_IDENTIFIER)
TokenType::T_IF->value => '\bif\b',
TokenType::T_ELSE->value => '\belse\b',
TokenType::T_FOR->value => '\bfor\b',
TokenType::T_FOREACH->value => '\bforeach\b',
TokenType::T_AS->value => '\bas\b',
TokenType::T_ECHO->value => '\becho\b',
TokenType::T_RETURN->value => '\breturn\b',
TokenType::T_TRUE->value => '\btrue\b',
TokenType::T_FALSE->value => '\bfalse\b',
TokenType::T_NULL->value => '\bnull\b',
TokenType::T_BREAK->value => '\bbreak\b',
TokenType::T_CONTINUE->value => '\bcontinue\b',
TokenType::T_LINEBREAK->value => '\bLINEBREAK\b',
// Identifiers
TokenType::T_IDENTIFIER->value => '\b[a-zA-Z_]\w*\b',
// Operators
TokenType::T_INCREMENT->value => '\+\+',
TokenType::T_DECREMENT->value => '--',
TokenType::T_COMPARE_EQUALS->value => '==(=)?',
TokenType::T_COMPARE_UNEQUALS->value => '!=(=)?',
TokenType::T_EQUALS->value => '=',
TokenType::T_BANG->value => '!',
TokenType::T_DOT->value => '\.',
TokenType::T_LEFT_PARENTHESIS->value => '\(',
TokenType::T_RIGHT_PARENTHESIS->value => '\)',
TokenType::T_LEFT_BRACE->value => '\{',
TokenType::T_RIGHT_BRACE->value => '\}',
TokenType::T_LEFT_BRACKET->value => '\[',
TokenType::T_RIGHT_BRACKET->value => '\]',
TokenType::T_SEMICOLON->value => ';',
TokenType::T_COMMA->value => ',',
TokenType::T_PLUS->value => '\+',
TokenType::T_MINUS->value => '\-',
TokenType::T_MULTIPLY->value => '\*',
TokenType::T_DIVIDE->value => '\/',
TokenType::T_GREATER_THAN->value => '>',
TokenType::T_LESS_THAN->value => '<',
];
}
/**
* Tokenizes the given script string into an array of tokens based on predefined patterns.
*
* @return list<\PhpScript\Core\Token>
*
* @throws LexerException If an unknown character or syntax error is encountered during tokenization.
*/
public function tokenize(string $script): array
{
$tokens = [];
$offset = 0;
$line = 1;
$lineOffset = 0;
$length = strlen($script);
while ($offset < $length) {
$remaining = substr($script, $offset);
$matchFound = false;
foreach ($this->tokenPatterns as $type => $pattern) {
if (preg_match('/^(' . $pattern . ')/u', $remaining, $matches)) {
$tokenTypeEnum = TokenType::from($type);
$value = $matches[1];
if ($tokenTypeEnum === TokenType::T_STRING) {
$content = isset($matches[3]) && $matches[3] !== '' ? $matches[3] : ($matches[2] ?? '');
$value = stripcslashes($content);
}
$column = $offset - $lineOffset + 1;
$tokens[] = new Token($tokenTypeEnum, $value, $line, $column, $offset);
if (str_contains($matches[0], "\n")) {
$line += substr_count($matches[0], "\n");
$lineOffset = $offset + (int) strrpos($matches[0], "\n") + 1;
}
$offset += strlen($matches[0]);
$matchFound = true;
break;
}
}
if (! $matchFound) {
$column = $offset - $lineOffset + 1;
throw LexerException::unknownCharOrSyntaxError($remaining, $line, $column, $offset);
}
}
return $tokens;
}
}