Introduce NodeId for AST and synthetic slices

src/frontend/lexer.rs

@@ -5,8 +5,9 @@ use crate::frontend::tokens::{Token, TokenType};
 
 pub struct Lexer {
     input: String,
-    start_char: usize,
-    current_char: usize,
+    // Byte offsets into `input` (not char counts) so slicing is UTF-8 correct.
+    start: usize,
+    current: usize,
     start_pos: SourcePosition,
     end_pos: SourcePosition,
     source_id: SourceId,
@@ -48,28 +49,18 @@ impl Lexer {
     pub fn new(input: String, source_id: SourceId) -> Lexer {
         Lexer {
             input,
-            start_char: 0,
-            current_char: 0,
+            start: 0,
+            current: 0,
             start_pos: SourcePosition::default(),
             end_pos: SourcePosition::default(),
             source_id,
         }
     }
 
-    fn advance_column(&mut self) {
-        self.current_char += 1;
-        self.end_pos.column += 1;
-    }
-
-    fn advance_line(&mut self) {
-        self.end_pos.line += 1;
-        self.end_pos.column = 0;
-    }
-
     pub fn scans_tokens(&mut self) -> LoxResult<Vec<Token>> {
         let mut tokens = Vec::new();
         while !self.is_at_end() {
-            self.start_char = self.current_char;
+            self.start = self.current;
             self.start_pos = self.end_pos.clone();
             match self.scan_token() {
                 Ok(Some(token)) => tokens.push(token),
@@ -82,32 +73,31 @@ impl Lexer {
     }
 
     fn is_at_end(&self) -> bool {
-        self.current_char >= self.input.len()
+        self.current >= self.input.len()
     }
 
     fn advance(&mut self) -> char {
-        self.advance_column();
-        self.input.chars().nth(self.current_char - 1).unwrap()
+        let c = self.input[self.current..].chars().next().unwrap();
+        self.current += c.len_utf8();
+        if c == '\n' {
+            self.end_pos.line += 1;
+            self.end_pos.column = 0;
+        } else {
+            self.end_pos.column += 1;
+        }
+        c
     }
 
     fn peek(&self) -> char {
-        if self.is_at_end() {
-            '\0'
-        } else {
-            self.input.chars().nth(self.current_char).unwrap()
-        }
+        self.input[self.current..].chars().next().unwrap_or('\0')
     }
 
     fn peek_next(&self) -> char {
-        if self.current_char + 1 >= self.input.len() {
-            '\0'
-        } else {
-            self.input.chars().nth(self.current_char + 1).unwrap()
-        }
+        self.input[self.current..].chars().nth(1).unwrap_or('\0')
     }
 
     fn make_token(&self, token_type: TokenType) -> Token {
-        let text = self.input[self.start_char..self.current_char].to_string();
+        let text = self.input[self.start..self.current].to_string();
         Token::new(
             token_type,
             text,
@@ -119,7 +109,7 @@ impl Lexer {
         )
     }
     fn make_token_with_literal(&self, token_type: TokenType, literal: BaseValue) -> Token {
-        let text = self.input[self.start_char..self.current_char].to_string();
+        let text = self.input[self.start..self.current].to_string();
         Token::new_complete(
             token_type,
             text,
@@ -179,9 +169,6 @@ impl Lexer {
             ('/', '*') => {
                 // Commento multi-line
                 while (self.peek() != '*' || self.peek_next() != '/') && !self.is_at_end() {
-                    if self.peek() == '\n' {
-                        self.advance_line();
-                    }
                     self.advance();
                 }
                 if self.is_at_end() {
@@ -201,10 +188,7 @@ impl Lexer {
             }
             ('/', _) => Ok(Some(self.make_token(TokenType::Slash))),
             (' ', _) | ('\r', _) | ('\t', _) => Ok(None),
-            ('\n', _) => {
-                self.advance_line();
-                Ok(None)
-            }
+            ('\n', _) => Ok(None),
             ('"', _) => self.string(),
             (c, _) if c.is_digit(10) => self.number(),
             (c, _) if c.is_alphanumeric() || c == '_' => self.identifier(),
@@ -236,7 +220,7 @@ impl Lexer {
         self.advance();
         Ok(Some(self.make_token_with_literal(
             TokenType::String,
-            BaseValue::String(self.input[self.start_char..self.current_char].to_string()),
+            BaseValue::String(self.input[self.start..self.current].to_string()),
         )))
     }
 
@@ -266,7 +250,7 @@ impl Lexer {
             None
         };
 
-        let num_str = &self.input[self.start_char..self.current_char];
+        let num_str = &self.input[self.start..self.current];
         let num_str_without_suffix = if suffix.is_some() {
             &num_str[..num_str.len() - 1]
         } else {
@@ -303,7 +287,7 @@ impl Lexer {
         while self.peek().is_alphanumeric() || self.peek() == '_' {
             self.advance();
         }
-        let text = self.input[self.start_char..self.current_char].to_string();
+        let text = self.input[self.start..self.current].to_string();
         match get_keyword_token(&text) {
             Some(TokenType::True) => Ok(Some(
                 self.make_token_with_literal(TokenType::True, BaseValue::Boolean(true)),
@@ -509,4 +493,25 @@ mod tests {
         let result = Lexer::new("@".to_string(), 0).scans_tokens();
         assert!(result.is_err());
     }
+
+    #[test]
+    fn handles_multibyte_identifier() {
+        // `é` is two UTF-8 bytes; the old char-counted slicing would panic or
+        // slice mid-codepoint here. Byte offsets make this correct.
+        let tokens = lex("café");
+        assert_eq!(tokens[0].token_type, TokenType::Identifier);
+        assert_eq!(
+            tokens[0].literal,
+            Some(BaseValue::String("café".to_string()))
+        );
+    }
+
+    #[test]
+    fn tracks_line_and_column_across_newlines() {
+        // "1\n22": the second token sits at the start of line 1.
+        let tokens = lex("1\n22");
+        assert_eq!(tokens[1].lexeme, "22");
+        assert_eq!(tokens[1].source_slice.start_position.line, 1);
+        assert_eq!(tokens[1].source_slice.start_position.column, 0);
+    }
 }