use crate::frontend::source_registry::{SourceId, SourcePosition, SourceSlice}; use crate::frontend::tokens::{LiteralValue, Token, TokenType}; use crate::result::{LoxError, LoxResult}; pub struct Lexer { input: String, start_char: usize, current_char: usize, start_pos: SourcePosition, end_pos: SourcePosition, source_id: SourceId, } fn get_keyword_token(word: &str) -> Option { match word { "and" => Some(TokenType::And), "class" => Some(TokenType::Class), "do" => Some(TokenType::StartBlock), "end" => Some(TokenType::EndBlock), "false" => Some(TokenType::False), "for" => Some(TokenType::For), "fun" => Some(TokenType::Fun), "if" => Some(TokenType::If), "elif" => Some(TokenType::Elif), "else" => Some(TokenType::Else), "or" => Some(TokenType::Or), "print" => Some(TokenType::Print), "return" => Some(TokenType::Return), "super" => Some(TokenType::Super), "this" => Some(TokenType::This), "true" => Some(TokenType::True), "var" => Some(TokenType::Var), "val" => Some(TokenType::Val), "while" => Some(TokenType::While), "True" => Some(TokenType::True), "False" => Some(TokenType::False), "Nil" => Some(TokenType::Nil), _ => None, } } impl Lexer { pub fn new(input: String, source_id: SourceId) -> Lexer { Lexer { input, start_char: 0, current_char: 0, start_pos: SourcePosition::default(), end_pos: SourcePosition::default(), source_id, } } fn advance_column(&mut self) { self.current_char += 1; self.end_pos.column += 1; } fn advance_line(&mut self) { self.end_pos.line += 1; self.end_pos.column = 0; } pub fn scans_tokens(&mut self) -> LoxResult> { let mut tokens = Vec::new(); while !self.is_at_end() { self.start_char = self.current_char; self.start_pos = self.end_pos.clone(); match self.scan_token() { Ok(Some(token)) => tokens.push(token), Ok(None) => {} Err(err) => return Err(err), } } tokens.push(self.make_token(TokenType::Eof)); println!("Tokens:"); tokens.iter().for_each(|val| println!("{}", val)); Ok(tokens) } fn is_at_end(&self) -> bool { self.current_char >= self.input.len() } fn advance(&mut self) -> char { self.advance_column(); self.input.chars().nth(self.current_char - 1).unwrap() } fn peek(&self) -> char { if self.is_at_end() { '\0' } else { self.input.chars().nth(self.current_char).unwrap() } } fn peek_next(&self) -> char { if self.current_char + 1 >= self.input.len() { '\0' } else { self.input.chars().nth(self.current_char + 1).unwrap() } } fn make_token(&self, token_type: TokenType) -> Token { let text = self.input[self.start_char..self.current_char].to_string(); Token::new( token_type, text, SourceSlice::from_positions( self.source_id.clone(), self.start_pos.clone(), self.end_pos.clone(), ), ) } fn make_token_with_literal(&self, token_type: TokenType, literal: LiteralValue) -> Token { let text = self.input[self.start_char..self.current_char].to_string(); Token::new_complete( token_type, text, Some(literal), SourceSlice::from_positions( self.source_id.clone(), self.start_pos.clone(), self.end_pos.clone(), ), ) } fn scan_token(&mut self) -> LoxResult> { let c = self.advance(); match (c, self.peek()) { ('(', _) => Ok(Some(self.make_token(TokenType::LeftParen))), (')', _) => Ok(Some(self.make_token(TokenType::RightParen))), ('{', _) => Ok(Some(self.make_token(TokenType::LeftBrace))), ('}', _) => Ok(Some(self.make_token(TokenType::RightBrace))), ('[', _) => Ok(Some(self.make_token(TokenType::LeftBracket))), (']', _) => Ok(Some(self.make_token(TokenType::RightBracket))), (',', _) => Ok(Some(self.make_token(TokenType::Comma))), ('.', _) => Ok(Some(self.make_token(TokenType::Dot))), ('-', _) => Ok(Some(self.make_token(TokenType::Minus))), ('+', _) => Ok(Some(self.make_token(TokenType::Plus))), (';', _) => Ok(Some(self.make_token(TokenType::Semicolon))), ('*', _) => Ok(Some(self.make_token(TokenType::Star))), ('!', '=') => { self.advance(); // consuma il '=' Ok(Some(self.make_token(TokenType::BangEqual))) } ('!', _) => Ok(Some(self.make_token(TokenType::Bang))), ('=', '=') => { self.advance(); // consuma il '=' Ok(Some(self.make_token(TokenType::EqualEqual))) } ('=', _) => Ok(Some(self.make_token(TokenType::Equal))), ('<', '=') => { self.advance(); // consuma il '=' Ok(Some(self.make_token(TokenType::LessEqual))) } ('<', _) => Ok(Some(self.make_token(TokenType::Less))), ('>', '=') => { self.advance(); // consuma il '=' Ok(Some(self.make_token(TokenType::GreaterEqual))) } ('>', _) => Ok(Some(self.make_token(TokenType::Greater))), ('/', '/') => { // Commento single-line while self.peek() != '\n' && !self.is_at_end() { self.advance(); } Ok(None) } ('/', '*') => { // Commento multi-line while (self.peek() != '*' || self.peek_next() != '/') && !self.is_at_end() { if self.peek() == '\n' { self.advance_line(); } self.advance(); } if self.is_at_end() { return Err(LoxError::LexicalError { source_slice: SourceSlice::from_positions( self.source_id.clone(), self.start_pos.clone(), self.end_pos.clone(), ), message: "Unterminated comment".to_string(), }); } else { self.advance(); // consuma '*' self.advance(); // consuma '/' } Ok(None) } ('/', _) => Ok(Some(self.make_token(TokenType::Slash))), (' ', _) | ('\r', _) | ('\t', _) => Ok(None), ('\n', _) => { self.advance_line(); Ok(None) } ('"', _) => self.string(), (c, _) if c.is_digit(10) => self.number(), (c, _) if c.is_alphanumeric() || c == '_' => self.identifier(), _ => Err(LoxError::LexicalError { source_slice: SourceSlice::from_positions( self.source_id.clone(), self.start_pos.clone(), self.end_pos.clone(), ), message: format!("Unexpected character: {}", c), }), } } fn string(&mut self) -> LoxResult> { while self.peek() != '"' && !self.is_at_end() { self.advance(); } if self.is_at_end() { return Err(LoxError::LexicalError { source_slice: SourceSlice::from_positions( self.source_id.clone(), self.start_pos.clone(), self.end_pos.clone(), ), message: "Unterminated string".to_string(), }); } self.advance(); Ok(Some(self.make_token_with_literal( TokenType::String, LiteralValue::String(self.input[self.start_char..self.current_char].to_string()), ))) } fn number(&mut self) -> LoxResult> { while self.peek().is_digit(10) { self.advance(); } if self.peek() == '.' && self.peek_next().is_digit(10) { self.advance(); while self.peek().is_digit(10) { self.advance(); } } Ok(Some( self.make_token_with_literal( TokenType::Number, LiteralValue::Number( self.input[self.start_char..self.current_char] .parse() .unwrap(), ), ), )) } fn identifier(&mut self) -> LoxResult> { while self.peek().is_alphanumeric() { self.advance(); } let text = self.input[self.start_char..self.current_char].to_string(); match get_keyword_token(&text) { Some(TokenType::True) => Ok(Some( self.make_token_with_literal(TokenType::True, LiteralValue::Boolean(true)), )), Some(TokenType::False) => Ok(Some( self.make_token_with_literal(TokenType::False, LiteralValue::Boolean(false)), )), Some(TokenType::Nil) => Ok(Some( self.make_token_with_literal(TokenType::Nil, LiteralValue::Nil), )), Some(token_type) => Ok(Some(self.make_token(token_type))), None => Ok(Some(self.make_token_with_literal( TokenType::Identifier, LiteralValue::String(text.clone()), ))), } } }