Repository navigation
Expand file tree
/
Copy pathtokenizer.py
More file actions
151 lines (125 loc) · 5.19 KB
/
Copy pathtokenizer.py
File metadata and controls
151 lines (125 loc) · 5.19 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
"""
Tokenizer for mathematical expression parser.
Converts input string into a sequence of Tokens.
"""
from enum import Enum, auto
from typing import List, Optional, Any
from dataclasses import dataclass
class TokenType(Enum):
NUMBER = auto()
PLUS = auto()
MINUS = auto()
MULTIPLY = auto()
DIVIDE = auto()
MODULO = auto()
POWER = auto()
LPAREN = auto()
RPAREN = auto()
COMMA = auto()
IDENTIFIER = auto()
EOF = auto()
@dataclass
class Token:
type: TokenType
value: Any
position: int
def __repr__(self) -> str:
return f"Token({self.type.name}, {repr(self.value)}, pos={self.position})"
class TokenizerError(Exception):
def __init__(self, message: str, position: int):
super().__init__(f"Tokenizer error at position {position}: {message}")
self.position = position
class Tokenizer:
FUNCTIONS = {"sin", "cos", "tan", "sqrt", "abs", "log", "exp", "floor", "ceil", "min", "max", "asin", "acos", "atan"}
CONSTANTS = {"pi", "e"}
def __init__(self, expression: str):
self.expression = expression
self.pos = 0
self.length = len(expression)
def _peek(self, offset: int = 0) -> Optional[str]:
idx = self.pos + offset
if idx < self.length:
return self.expression[idx]
return None
def tokenize(self) -> List[Token]:
tokens: List[Token] = []
while self.pos < self.length:
ch = self.expression[self.pos]
if ch.isspace():
self.pos += 1
continue
start_pos = self.pos
# Numbers (integers & floats)
if ch.isdigit() or (ch == '.' and self._peek(1) and self._peek(1).isdigit()):
num_str = ""
has_decimal = False
while self.pos < self.length:
curr = self.expression[self.pos]
if curr.isdigit():
num_str += curr
self.pos += 1
elif curr == '.' and not has_decimal:
has_decimal = True
num_str += curr
self.pos += 1
else:
break
val = float(num_str) if has_decimal else int(num_str)
tokens.append(Token(TokenType.NUMBER, val, start_pos))
continue
# Identifiers (Functions, Constants, Variables)
if ch.isalpha() or ch == '_':
ident = ""
while self.pos < self.length and (self.expression[self.pos].isalnum() or self.expression[self.pos] == '_'):
ident += self.expression[self.pos]
self.pos += 1
tokens.append(Token(TokenType.IDENTIFIER, ident, start_pos))
continue
# Single & Multi-character operators
if ch == '+':
tokens.append(Token(TokenType.PLUS, '+', start_pos))
self.pos += 1
elif ch == '-':
tokens.append(Token(TokenType.MINUS, '-', start_pos))
self.pos += 1
elif ch == '*':
if self._peek(1) == '*':
tokens.append(Token(TokenType.POWER, '**', start_pos))
self.pos += 2
else:
tokens.append(Token(TokenType.MULTIPLY, '*', start_pos))
self.pos += 1
elif ch == '/':
tokens.append(Token(TokenType.DIVIDE, '/', start_pos))
self.pos += 1
elif ch == '%':
tokens.append(Token(TokenType.MODULO, '%', start_pos))
self.pos += 1
elif ch == '^':
tokens.append(Token(TokenType.POWER, '^', start_pos))
self.pos += 1
elif ch == '(':
tokens.append(Token(TokenType.LPAREN, '(', start_pos))
self.pos += 1
elif ch == ')':
tokens.append(Token(TokenType.RPAREN, ')', start_pos))
self.pos += 1
elif ch == ',':
tokens.append(Token(TokenType.COMMA, ',', start_pos))
self.pos += 1
else:
raise TokenizerError(f"Unexpected character '{ch}'", start_pos)
# Insert implicit multiplication tokens where appropriate (e.g. 2(3), 2x, (a)(b))
processed_tokens: List[Token] = []
for i, tok in enumerate(tokens):
processed_tokens.append(tok)
if i + 1 < len(tokens):
next_tok = tokens[i + 1]
# Number followed by LPAREN, IDENTIFIER
if tok.type == TokenType.NUMBER and next_tok.type in (TokenType.LPAREN, TokenType.IDENTIFIER):
processed_tokens.append(Token(TokenType.MULTIPLY, '*', next_tok.position))
# RPAREN followed by LPAREN, NUMBER, IDENTIFIER
elif tok.type == TokenType.RPAREN and next_tok.type in (TokenType.LPAREN, TokenType.NUMBER, TokenType.IDENTIFIER):
processed_tokens.append(Token(TokenType.MULTIPLY, '*', next_tok.position))
processed_tokens.append(Token(TokenType.EOF, "", self.length))
return processed_tokens