88 lines
1.8 KiB
Python
88 lines
1.8 KiB
Python
|
from enum import Enum, auto
|
||
|
|
from typing import NamedTuple
|
||
|
|
from position import Span
|
||
|
|
|
||
|
|
|
||
|
|
class TokenType(Enum):
|
||
|
|
Eof = auto()
|
||
|
|
InvalidChar = auto()
|
||
|
|
MalformedComment = auto()
|
||
|
|
MalformedChar = auto()
|
||
|
|
MalformedString = auto()
|
||
|
|
Id = auto()
|
||
|
|
Int = auto()
|
||
|
|
Char = auto()
|
||
|
|
String = auto()
|
||
|
|
LParen = auto()
|
||
|
|
RParen = auto()
|
||
|
|
LBrace = auto()
|
||
|
|
RBrace = auto()
|
||
|
|
LBracket = auto()
|
||
|
|
RBracket = auto()
|
||
|
|
Underscore = auto()
|
||
|
|
Dot = auto()
|
||
|
|
DotDot = auto()
|
||
|
|
DotDotDot = auto()
|
||
|
|
DotDotEqual = auto()
|
||
|
|
Comma = auto()
|
||
|
|
Colon = auto()
|
||
|
|
ColonColon = auto()
|
||
|
|
ColonColonLT = auto()
|
||
|
|
Semicolon = auto()
|
||
|
|
Ampersand = auto()
|
||
|
|
Plus = auto()
|
||
|
|
PlusEqual = auto()
|
||
|
|
Minus = auto()
|
||
|
|
MinusEqual = auto()
|
||
|
|
MinusLT = auto()
|
||
|
|
Asterisk = auto()
|
||
|
|
AsteriskEqual = auto()
|
||
|
|
AsteriskAsterisk = auto()
|
||
|
|
Slash = auto()
|
||
|
|
SlashEqual = auto()
|
||
|
|
Percent = auto()
|
||
|
|
PercentEqual = auto()
|
||
|
|
Equal = auto()
|
||
|
|
EqualEqual = auto()
|
||
|
|
EqualLT = auto()
|
||
|
|
Exclamation = auto()
|
||
|
|
ExclamationEqual = auto()
|
||
|
|
LT = auto()
|
||
|
|
LTEqual = auto()
|
||
|
|
GT = auto()
|
||
|
|
GTEqual = auto()
|
||
|
|
KwFalse = auto()
|
||
|
|
KwTrue = auto()
|
||
|
|
KwNot = auto()
|
||
|
|
KwIn = auto()
|
||
|
|
KwAnd = auto()
|
||
|
|
KwOr = auto()
|
||
|
|
KwXor = auto()
|
||
|
|
KwLet = auto()
|
||
|
|
KwMut = auto()
|
||
|
|
KwIf = auto()
|
||
|
|
KwMatch = auto()
|
||
|
|
KwElse = auto()
|
||
|
|
KwLoop = auto()
|
||
|
|
KwWhile = auto()
|
||
|
|
KwFor = auto()
|
||
|
|
KwBreak = auto()
|
||
|
|
KwContinue = auto()
|
||
|
|
KwFn = auto()
|
||
|
|
KwReturn = auto()
|
||
|
|
|
||
|
|
|
||
|
|
class Token(NamedTuple):
|
||
|
|
token_type: TokenType
|
||
|
|
index: int
|
||
|
|
length: int
|
||
|
|
span: Span
|
||
|
|
|
||
|
|
def text_slice(self, text: str) -> str:
|
||
|
|
return text[self.index:self.index + self.length]
|
||
|
|
|
||
|
|
|
||
|
|
class TokenIterator:
|
||
|
|
def next(self) -> Token:
|
||
|
|
raise NotImplementedError()
|