sfja.skp/web-stack-project
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

implement unimplemented tokens

Browse Source
This commit is contained in:
Theis Pieter Hollebeek 2023-02-07 14:25:08 +01:00
parent 6f347a5744
commit ee0eba066a
1 changed files with 20 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/bong/lexer.rs
View File

@ -1,5 +1,3 @@
#![allow(dead_code)]
const NO_MUT_PEEK_NEXT_MESSAGE: &str = "should not mutate between peek & next";
use std::iter::Peekable;
@ -282,7 +280,8 @@ fn make_comment<T: Iterator<Item = char>>(
}
}
fn lexer(code: &str) -> Vec<Token> {
#[allow(dead_code)]
pub fn lexer(code: &str) -> Vec<Token> {
let mut tokens = Vec::new();
let mut iter = code.chars().peekable();
let mut col = 0;
@ -319,6 +318,24 @@ fn lexer(code: &str) -> Vec<Token> {
'}' => {
tokens.push(single_token(&mut iter, Token::RBrace, &mut col));
}
'[' => {
tokens.push(single_token(&mut iter, Token::LBracket, &mut col));
}
']' => {
tokens.push(single_token(&mut iter, Token::RBracket, &mut col));
}
':' => {
tokens.push(single_token(&mut iter, Token::Colon, &mut col));
}
';' => {
tokens.push(single_token(&mut iter, Token::SemiColon, &mut col));
}
',' => {
tokens.push(single_token(&mut iter, Token::Comma, &mut col));
}
'=' => {
tokens.push(single_token(&mut iter, Token::Equal, &mut col));
}
'#' | '.' => {
tokens.push(make_id_or_class(&mut iter, &mut col));
}