diff --git a/src/bong/lexer.rs b/src/bong/lexer.rs index ccf1f64..d7a10ba 100644 --- a/src/bong/lexer.rs +++ b/src/bong/lexer.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - const NO_MUT_PEEK_NEXT_MESSAGE: &str = "should not mutate between peek & next"; use std::iter::Peekable; @@ -282,7 +280,8 @@ fn make_comment>( } } -fn lexer(code: &str) -> Vec { +#[allow(dead_code)] +pub fn lexer(code: &str) -> Vec { let mut tokens = Vec::new(); let mut iter = code.chars().peekable(); let mut col = 0; @@ -319,6 +318,24 @@ fn lexer(code: &str) -> Vec { '}' => { tokens.push(single_token(&mut iter, Token::RBrace, &mut col)); } + '[' => { + tokens.push(single_token(&mut iter, Token::LBracket, &mut col)); + } + ']' => { + tokens.push(single_token(&mut iter, Token::RBracket, &mut col)); + } + ':' => { + tokens.push(single_token(&mut iter, Token::Colon, &mut col)); + } + ';' => { + tokens.push(single_token(&mut iter, Token::SemiColon, &mut col)); + } + ',' => { + tokens.push(single_token(&mut iter, Token::Comma, &mut col)); + } + '=' => { + tokens.push(single_token(&mut iter, Token::Equal, &mut col)); + } '#' | '.' => { tokens.push(make_id_or_class(&mut iter, &mut col)); }