use crate::common::base_value::{BaseValue, Number}; use crate::common::lox_result::{lexical_error, LoxError, LoxResult}; use crate::frontend::source_registry::{SourceId, SourcePosition, SourceSlice}; use crate::frontend::tokens::{Token, TokenType}; pub struct Lexer { input: String, // Byte offsets into `input` (not char counts) so slicing is UTF-8 correct. start: usize, current: usize, start_pos: SourcePosition, end_pos: SourcePosition, source_id: SourceId, } fn get_keyword_token(word: &str) -> Option { match word { "and" => Some(TokenType::And), "class" => Some(TokenType::Class), "do" => Some(TokenType::StartBlock), "end" => Some(TokenType::EndBlock), "false" => Some(TokenType::False), "for" => Some(TokenType::For), "fun" => Some(TokenType::Fun), "if" => Some(TokenType::If), "then" => Some(TokenType::Then), "elif" => Some(TokenType::Elif), "else" => Some(TokenType::Else), "or" => Some(TokenType::Or), "in" => Some(TokenType::In), "print" => Some(TokenType::Print), "return" => Some(TokenType::Return), "break" => Some(TokenType::Break), "super" => Some(TokenType::Super), "this" => Some(TokenType::This), "true" => Some(TokenType::True), "var" => Some(TokenType::Var), "val" => Some(TokenType::Val), "while" => Some(TokenType::While), "True" => Some(TokenType::True), "False" => Some(TokenType::False), "Nil" => Some(TokenType::Nil), "fn" => Some(TokenType::Fn), _ => None, } } impl Lexer { pub fn new(input: String, source_id: SourceId) -> Lexer { Lexer { input, start: 0, current: 0, start_pos: SourcePosition::default(), end_pos: SourcePosition::default(), source_id, } } pub fn scans_tokens(&mut self) -> LoxResult> { let mut tokens = Vec::new(); while !self.is_at_end() { self.start = self.current; self.start_pos = self.end_pos.clone(); match self.scan_token() { Ok(Some(token)) => tokens.push(token), Ok(None) => {} Err(err) => return Err(err), } } tokens.push(self.make_token(TokenType::Eof)); Ok(tokens) } fn is_at_end(&self) -> bool { self.current >= self.input.len() } fn advance(&mut self) -> char { let c = self.input[self.current..].chars().next().unwrap(); self.current += c.len_utf8(); if c == '\n' { self.end_pos.line += 1; self.end_pos.column = 0; } else { self.end_pos.column += 1; } c } fn peek(&self) -> char { self.input[self.current..].chars().next().unwrap_or('\0') } fn peek_next(&self) -> char { self.input[self.current..].chars().nth(1).unwrap_or('\0') } fn make_token(&self, token_type: TokenType) -> Token { let text = self.input[self.start..self.current].to_string(); Token::new( token_type, text, SourceSlice::from_positions( self.source_id.clone(), self.start_pos.clone(), self.end_pos.clone(), ), ) } fn make_token_with_literal(&self, token_type: TokenType, literal: BaseValue) -> Token { let text = self.input[self.start..self.current].to_string(); Token::new_complete( token_type, text, Some(literal), SourceSlice::from_positions( self.source_id.clone(), self.start_pos.clone(), self.end_pos.clone(), ), ) } fn scan_token(&mut self) -> LoxResult> { let c = self.advance(); match (c, self.peek()) { ('(', _) => Ok(Some(self.make_token(TokenType::LeftParen))), (')', _) => Ok(Some(self.make_token(TokenType::RightParen))), ('{', _) => Ok(Some(self.make_token(TokenType::LeftBrace))), ('}', _) => Ok(Some(self.make_token(TokenType::RightBrace))), ('[', _) => Ok(Some(self.make_token(TokenType::LeftBracket))), (']', _) => Ok(Some(self.make_token(TokenType::RightBracket))), (',', _) => Ok(Some(self.make_token(TokenType::Comma))), ('.', _) => Ok(Some(self.make_token(TokenType::Dot))), ('-', _) => Ok(Some(self.make_token(TokenType::Minus))), ('+', _) => Ok(Some(self.make_token(TokenType::Plus))), (';', _) => Ok(Some(self.make_token(TokenType::Semicolon))), (':', _) => Ok(Some(self.make_token(TokenType::Colon))), ('*', _) => Ok(Some(self.make_token(TokenType::Star))), ('%', _) => Ok(Some(self.make_token(TokenType::Percent))), ('!', '=') => { self.advance(); // consuma il '=' Ok(Some(self.make_token(TokenType::BangEqual))) } ('!', _) => Ok(Some(self.make_token(TokenType::Bang))), ('=', '=') => { self.advance(); // consuma il '=' Ok(Some(self.make_token(TokenType::EqualEqual))) } ('=', _) => Ok(Some(self.make_token(TokenType::Equal))), ('<', '=') => { self.advance(); // consuma il '=' Ok(Some(self.make_token(TokenType::LessEqual))) } ('<', _) => Ok(Some(self.make_token(TokenType::Less))), ('>', '=') => { self.advance(); // consuma il '=' Ok(Some(self.make_token(TokenType::GreaterEqual))) } ('>', _) => Ok(Some(self.make_token(TokenType::Greater))), ('/', '/') => { // Commento single-line while self.peek() != '\n' && !self.is_at_end() { self.advance(); } Ok(None) } ('/', '*') => { // Commento multi-line while (self.peek() != '*' || self.peek_next() != '/') && !self.is_at_end() { self.advance(); } if self.is_at_end() { return lexical_error( SourceSlice::from_positions( self.source_id.clone(), self.start_pos.clone(), self.end_pos.clone(), ), "Unterminated comment".to_string(), ); } else { self.advance(); // consuma '*' self.advance(); // consuma '/' } Ok(None) } ('/', _) => Ok(Some(self.make_token(TokenType::Slash))), (' ', _) | ('\r', _) | ('\t', _) => Ok(None), ('\n', _) => Ok(None), ('"', _) => self.string(), (c, _) if c.is_digit(10) => self.number(), (c, _) if c.is_alphanumeric() || c == '_' => self.identifier(), _ => Err(LoxError::LexicalError { source_slice: SourceSlice::from_positions( self.source_id.clone(), self.start_pos.clone(), self.end_pos.clone(), ), message: format!("Unexpected character: {}", c), }), } } fn string(&mut self) -> LoxResult> { while self.peek() != '"' && !self.is_at_end() { self.advance(); } if self.is_at_end() { return Err(LoxError::LexicalError { source_slice: SourceSlice::from_positions( self.source_id.clone(), self.start_pos.clone(), self.end_pos.clone(), ), message: "Unterminated string".to_string(), }); } self.advance(); Ok(Some(self.make_token_with_literal( TokenType::String, BaseValue::String(self.input[self.start..self.current].to_string()), ))) } fn number(&mut self) -> LoxResult> { // Leggi la parte intera while self.peek().is_digit(10) { self.advance(); } // Controlla se c'è una parte decimale let has_decimal = if self.peek() == '.' && self.peek_next().is_digit(10) { self.advance(); // consuma il '.' while self.peek().is_digit(10) { self.advance(); } true } else { false }; // Controlla se c'è un suffisso (f o u) let suffix = if self.peek() == 'f' || self.peek() == 'u' { let s = self.peek(); self.advance(); Some(s) } else { None }; let num_str = &self.input[self.start..self.current]; let num_str_without_suffix = if suffix.is_some() { &num_str[..num_str.len() - 1] } else { num_str }; // Determina il tipo di numero basandosi sul contenuto e suffisso let number_value = match suffix { Some('f') => { // Float esplicito con suffisso 'f' Number::F64(num_str_without_suffix.parse().unwrap()) } Some('u') => { // Unsigned esplicito con suffisso 'u' Number::U128(num_str_without_suffix.parse().unwrap()) } _ if has_decimal => { // Float implicito (contiene un punto decimale) Number::F64(num_str.parse().unwrap()) } _ => { // Intero con segno di default Number::I32(num_str.parse().unwrap()) } }; Ok(Some(self.make_token_with_literal( TokenType::Number, BaseValue::Number(number_value), ))) } fn identifier(&mut self) -> LoxResult> { while self.peek().is_alphanumeric() || self.peek() == '_' { self.advance(); } let text = self.input[self.start..self.current].to_string(); match get_keyword_token(&text) { Some(TokenType::True) => Ok(Some( self.make_token_with_literal(TokenType::True, BaseValue::Boolean(true)), )), Some(TokenType::False) => Ok(Some( self.make_token_with_literal(TokenType::False, BaseValue::Boolean(false)), )), Some(TokenType::Nil) => Ok(Some( self.make_token_with_literal(TokenType::Nil, BaseValue::Nil), )), Some(token_type) => Ok(Some(self.make_token(token_type))), None => Ok(Some(self.make_token_with_literal( TokenType::Identifier, BaseValue::String(text.clone()), ))), } } } #[cfg(test)] mod tests { use std::f64; use super::*; /// Lex `input` and return the resulting tokens, panicking on lexical errors. fn lex(input: &str) -> Vec { Lexer::new(input.to_string(), 0) .scans_tokens() .expect("expected input to lex without errors") } /// Lex `input` and collect just the token types (including the trailing EOF). fn token_types(input: &str) -> Vec { lex(input).into_iter().map(|t| t.token_type).collect() } #[test] fn scans_single_character_tokens() { assert_eq!( token_types("(){}[],.-+;:*%"), vec![ TokenType::LeftParen, TokenType::RightParen, TokenType::LeftBrace, TokenType::RightBrace, TokenType::LeftBracket, TokenType::RightBracket, TokenType::Comma, TokenType::Dot, TokenType::Minus, TokenType::Plus, TokenType::Semicolon, TokenType::Colon, TokenType::Star, TokenType::Percent, TokenType::Eof, ] ); } #[test] fn scans_one_and_two_char_operators() { assert_eq!( token_types("! != = == < <= > >= /"), vec![ TokenType::Bang, TokenType::BangEqual, TokenType::Equal, TokenType::EqualEqual, TokenType::Less, TokenType::LessEqual, TokenType::Greater, TokenType::GreaterEqual, TokenType::Slash, TokenType::Eof, ] ); } #[test] fn keyword_lookup_matches_known_words() { assert_eq!(get_keyword_token("and"), Some(TokenType::And)); assert_eq!(get_keyword_token("if"), Some(TokenType::If)); assert_eq!(get_keyword_token("then"), Some(TokenType::Then)); assert_eq!(get_keyword_token("while"), Some(TokenType::While)); assert_eq!(get_keyword_token("do"), Some(TokenType::StartBlock)); assert_eq!(get_keyword_token("end"), Some(TokenType::EndBlock)); assert_eq!(get_keyword_token("not_a_keyword"), None); } #[test] fn scans_keywords_in_a_stream() { assert_eq!( token_types("if then else while print return"), vec![ TokenType::If, TokenType::Then, TokenType::Else, TokenType::While, TokenType::Print, TokenType::Return, TokenType::Eof, ] ); } #[test] fn scans_integer_number() { let tokens = lex("42"); assert_eq!(tokens[0].token_type, TokenType::Number); assert_eq!(tokens[0].literal, Some(BaseValue::Number(Number::I32(42)))); } #[test] fn scans_float_number() { let tokens = lex("3.141592653589793"); assert_eq!( tokens[0].literal, Some(BaseValue::Number(Number::F64(f64::consts::PI))) ); } #[test] fn scans_number_with_float_suffix() { let tokens = lex("5f"); assert_eq!(tokens[0].literal, Some(BaseValue::Number(Number::F64(5.0)))); } #[test] fn scans_number_with_unsigned_suffix() { let tokens = lex("7u"); assert_eq!(tokens[0].literal, Some(BaseValue::Number(Number::U128(7)))); } #[test] fn scans_string_literal_including_quotes() { // The lexer slices from the opening quote through the closing quote, // so the literal currently retains the surrounding quotes. let tokens = lex("\"hello\""); assert_eq!(tokens[0].token_type, TokenType::String); assert_eq!( tokens[0].literal, Some(BaseValue::String("\"hello\"".to_string())) ); } #[test] fn scans_identifier() { let tokens = lex("foo_bar"); assert_eq!(tokens[0].token_type, TokenType::Identifier); assert_eq!( tokens[0].literal, Some(BaseValue::String("foo_bar".to_string())) ); } #[test] fn scans_boolean_and_nil_literals() { let tokens = lex("true false Nil"); assert_eq!(tokens[0].literal, Some(BaseValue::Boolean(true))); assert_eq!(tokens[1].literal, Some(BaseValue::Boolean(false))); assert_eq!(tokens[2].literal, Some(BaseValue::Nil)); } #[test] fn ignores_line_comments() { assert_eq!( token_types("1 // a comment\n2"), vec![TokenType::Number, TokenType::Number, TokenType::Eof] ); } #[test] fn ignores_block_comments() { assert_eq!( token_types("1 /* multi\nline */ 2"), vec![TokenType::Number, TokenType::Number, TokenType::Eof] ); } #[test] fn always_appends_eof_even_for_empty_input() { let tokens = lex(""); assert_eq!(tokens.len(), 1); assert_eq!(tokens[0].token_type, TokenType::Eof); } #[test] fn unterminated_string_is_a_lexical_error() { let result = Lexer::new("\"oops".to_string(), 0).scans_tokens(); assert!(matches!(result, Err(LoxError::LexicalError { .. }))); } #[test] fn unterminated_block_comment_is_a_lexical_error() { let result = Lexer::new("/* never ends".to_string(), 0).scans_tokens(); assert!(matches!(result, Err(LoxError::LexicalError { .. }))); } #[test] fn unexpected_character_is_a_lexical_error() { let result = Lexer::new("@".to_string(), 0).scans_tokens(); assert!(result.is_err()); } #[test] fn handles_multibyte_identifier() { // `é` is two UTF-8 bytes; the old char-counted slicing would panic or // slice mid-codepoint here. Byte offsets make this correct. let tokens = lex("café"); assert_eq!(tokens[0].token_type, TokenType::Identifier); assert_eq!( tokens[0].literal, Some(BaseValue::String("café".to_string())) ); } #[test] fn tracks_line_and_column_across_newlines() { // "1\n22": the second token sits at the start of line 1. let tokens = lex("1\n22"); assert_eq!(tokens[1].lexeme, "22"); assert_eq!(tokens[1].source_slice.start_position.line, 1); assert_eq!(tokens[1].source_slice.start_position.column, 0); } }