ProjectValkay/rlox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce NodeId for AST and synthetic slices

Browse Source
This commit is contained in:
Giulio Agostini
2026-06-30 14:05:46 +02:00
parent ef8abda048
commit d40fe2a550
11 changed files with 504 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/frontend/lexer.rs
+45 -40
View File
@@ -5,8 +5,9 @@ use crate::frontend::tokens::{Token, TokenType};
pub struct Lexer {
input: String,
start_char: usize,
current_char: usize,
// Byte offsets into `input` (not char counts) so slicing is UTF-8 correct.
start: usize,
current: usize,
start_pos: SourcePosition,
end_pos: SourcePosition,
source_id: SourceId,
@@ -48,28 +49,18 @@ impl Lexer {
pub fn new(input: String, source_id: SourceId) -> Lexer {
Lexer {
input,
start_char: 0,
current_char: 0,
start: 0,
current: 0,
start_pos: SourcePosition::default(),
end_pos: SourcePosition::default(),
source_id,
}
}
fn advance_column(&mut self) {
self.current_char += 1;
self.end_pos.column += 1;
}
fn advance_line(&mut self) {
self.end_pos.line += 1;
self.end_pos.column = 0;
}
pub fn scans_tokens(&mut self) -> LoxResult<Vec<Token>> {
let mut tokens = Vec::new();
while !self.is_at_end() {
self.start_char = self.current_char;
self.start = self.current;
self.start_pos = self.end_pos.clone();
match self.scan_token() {
Ok(Some(token)) => tokens.push(token),
@@ -82,32 +73,31 @@ impl Lexer {
}
fn is_at_end(&self) -> bool {
self.current_char >= self.input.len()
self.current >= self.input.len()
}
fn advance(&mut self) -> char {
self.advance_column();
self.input.chars().nth(self.current_char - 1).unwrap()
let c = self.input[self.current..].chars().next().unwrap();
self.current += c.len_utf8();
if c == '\n' {
self.end_pos.line += 1;
self.end_pos.column = 0;
} else {
self.end_pos.column += 1;
}
c
}
fn peek(&self) -> char {
if self.is_at_end() {
'\0'
} else {
self.input.chars().nth(self.current_char).unwrap()
}
self.input[self.current..].chars().next().unwrap_or('\0')
}
fn peek_next(&self) -> char {
if self.current_char + 1 >= self.input.len() {
'\0'
} else {
self.input.chars().nth(self.current_char + 1).unwrap()
}
self.input[self.current..].chars().nth(1).unwrap_or('\0')
}
fn make_token(&self, token_type: TokenType) -> Token {
let text = self.input[self.start_char..self.current_char].to_string();
let text = self.input[self.start..self.current].to_string();
Token::new(
token_type,
text,
@@ -119,7 +109,7 @@ impl Lexer {
)
}
fn make_token_with_literal(&self, token_type: TokenType, literal: BaseValue) -> Token {
let text = self.input[self.start_char..self.current_char].to_string();
let text = self.input[self.start..self.current].to_string();
Token::new_complete(
token_type,
text,
@@ -179,9 +169,6 @@ impl Lexer {
('/', '*') => {
// Commento multi-line
while (self.peek() != '*' || self.peek_next() != '/') && !self.is_at_end() {
if self.peek() == '\n' {
self.advance_line();
}
self.advance();
}
if self.is_at_end() {
@@ -201,10 +188,7 @@ impl Lexer {
}
('/', _) => Ok(Some(self.make_token(TokenType::Slash))),
(' ', _) | ('\r', _) | ('\t', _) => Ok(None),
('\n', _) => {
self.advance_line();
Ok(None)
}
('\n', _) => Ok(None),
('"', _) => self.string(),
(c, _) if c.is_digit(10) => self.number(),
(c, _) if c.is_alphanumeric() || c == '_' => self.identifier(),
@@ -236,7 +220,7 @@ impl Lexer {
self.advance();
Ok(Some(self.make_token_with_literal(
TokenType::String,
BaseValue::String(self.input[self.start_char..self.current_char].to_string()),
BaseValue::String(self.input[self.start..self.current].to_string()),
)))
}
@@ -266,7 +250,7 @@ impl Lexer {
None
};
let num_str = &self.input[self.start_char..self.current_char];
let num_str = &self.input[self.start..self.current];
let num_str_without_suffix = if suffix.is_some() {
&num_str[..num_str.len() - 1]
} else {
@@ -303,7 +287,7 @@ impl Lexer {
while self.peek().is_alphanumeric() || self.peek() == '_' {
self.advance();
}
let text = self.input[self.start_char..self.current_char].to_string();
let text = self.input[self.start..self.current].to_string();
match get_keyword_token(&text) {
Some(TokenType::True) => Ok(Some(
self.make_token_with_literal(TokenType::True, BaseValue::Boolean(true)),
@@ -509,4 +493,25 @@ mod tests {
let result = Lexer::new("@".to_string(), 0).scans_tokens();
assert!(result.is_err());
}
#[test]
fn handles_multibyte_identifier() {
// `é` is two UTF-8 bytes; the old char-counted slicing would panic or
// slice mid-codepoint here. Byte offsets make this correct.
let tokens = lex("café");
assert_eq!(tokens[0].token_type, TokenType::Identifier);
assert_eq!(
tokens[0].literal,
Some(BaseValue::String("café".to_string()))
);
}
#[test]
fn tracks_line_and_column_across_newlines() {
// "1\n22": the second token sits at the start of line 1.
let tokens = lex("1\n22");
assert_eq!(tokens[1].lexeme, "22");
assert_eq!(tokens[1].source_slice.start_position.line, 1);
assert_eq!(tokens[1].source_slice.start_position.column, 0);
}
}
src/frontend/parser.rs
+46 -42
View File
@@ -53,7 +53,6 @@ impl Parser {
self.var_statement()
}
(TokenType::Identifier, TokenType::Colon) => self.var_statement(),
(TokenType::Identifier, TokenType::Equal) => self.assignment_statement(),
(TokenType::If, _) => self.if_statement(),
(TokenType::While, _) => self.while_statement(),
(TokenType::For, _) => self.for_statement(),
@@ -68,7 +67,7 @@ impl Parser {
let variable = self.var_statement()?;
let condition = self.expression()?;
self.consume(TokenType::Semicolon, "Expected ';' after for condition")?;
let increment = self.assignment_statement()?;
let increment = self.expression_statement()?;
let body = self.statement(Some("loop".to_string()))?;
let end_slice = self.previous().source_slice.clone();
@@ -260,8 +259,8 @@ impl Parser {
}
let body = self.statement(None)?;
let node = AstNode {
node: Expr::Literal {
let node = AstNode::new(
Expr::Literal {
value: BaseValue::Function(LoxFunction {
parameters,
return_type: None,
@@ -270,8 +269,8 @@ impl Parser {
guard,
}),
},
source_slice: combine_position.clone(),
};
combine_position.clone(),
);
Ok(AstNode::new(
Stmt::VarDeclaration {
name: name_lexeme,
@@ -294,12 +293,12 @@ impl Parser {
}
// todo: make type annotation
}
let mut value = AstNode {
node: Expr::Literal {
let mut value = AstNode::new(
Expr::Literal {
value: BaseValue::Nil,
},
source_slice: start_slice.clone(),
};
start_slice.clone(),
);
if self.peek().token_type == TokenType::Equal {
self.advance();
value = self.expression()?;
@@ -323,31 +322,6 @@ impl Parser {
combined_slice,
))
}
fn assignment_statement(&mut self) -> LoxResult<AstNode<Stmt>> {
let start_slice = self.peek().source_slice.clone();
let name = self.consume(TokenType::Identifier, "Expect variable name.")?;
let name_lexeme = name.lexeme.clone();
self.consume(TokenType::Equal, "Expect '=' after variable name.")?;
let value = self.expression()?;
let semicolon = self.consume(
TokenType::Semicolon,
"Expect ';' after variable declaration.",
)?;
let end_slice = semicolon.source_slice.clone();
let combined_slice = SourceSlice::from_positions(
start_slice.source_id,
start_slice.start_position,
end_slice.end_position,
);
Ok(AstNode::new(
Stmt::VarAssigment {
name: name_lexeme,
value: Box::new(value),
return_value: Box::new(BaseValue::Nil),
},
combined_slice,
))
}
fn print_statement(&mut self) -> LoxResult<AstNode<Stmt>> {
// consume the print keyword
@@ -441,12 +415,12 @@ impl Parser {
source_slice,
}) => {
if message == "Expect expression." {
AstNode {
node: Expr::Literal {
AstNode::new(
Expr::Literal {
value: BaseValue::Nil,
},
source_slice: start_slice.clone(),
}
start_slice.clone(),
)
} else {
return Err(LoxError::ParseError {
message,
@@ -479,7 +453,34 @@ impl Parser {
}
fn expression(&mut self) -> LoxResult<AstNode<Expr>> {
self.or_and()
self.assignment()
}
fn assignment(&mut self) -> LoxResult<AstNode<Expr>> {
let expr = self.or_and()?;
if self.peek().token_type == TokenType::Equal {
self.advance(); // consume '='
// Right-associative: `a = b = c` parses as `a = (b = c)`.
let value = self.assignment()?;
let combined_slice = SourceSlice::from_positions(
expr.source_slice.source_id,
expr.source_slice.start_position.clone(),
value.source_slice.end_position.clone(),
);
let target_slice = expr.source_slice.clone();
match expr.node {
Expr::Identifier { name } => Ok(AstNode::new(
Expr::Assign {
name,
value: Box::new(value),
},
combined_slice,
)),
_ => parse_error(target_slice, "Invalid assignment target."),
}
} else {
Ok(expr)
}
}
fn or_and(&mut self) -> LoxResult<AstNode<Expr>> {
@@ -978,8 +979,11 @@ mod tests {
fn parses_assignment_statement() {
let stmts = parse_ok("x = 5;");
match &stmts[0].node {
Stmt::VarAssigment { name, .. } => assert_eq!(name, "x"),
other => panic!("expected assignment statement, got {:?}", other),
Stmt::Expression { expression, .. } => match &expression.node {
Expr::Assign { name, .. } => assert_eq!(name, "x"),
other => panic!("expected assign expression, got {:?}", other),
},
other => panic!("expected expression statement, got {:?}", other),
}
}
src/frontend/source_registry.rs
+21 -2
View File
@@ -51,7 +51,7 @@ impl SourceFile {
}
}
#[derive(Debug, Clone, PartialEq, Default)]
#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
pub struct SourcePosition {
pub line: usize,
pub column: usize,
@@ -63,7 +63,7 @@ impl Display for SourcePosition {
}
}
#[derive(Clone, PartialEq, Default)]
#[derive(Clone, PartialEq, Eq, Hash)]
pub struct SourceSlice {
pub source_id: SourceId,
pub start_position: SourcePosition,
@@ -91,6 +91,25 @@ impl Display for SourceSlice {
}
impl SourceSlice {
/// An explicit, non-located placeholder span.
///
/// Use this only when a real span is genuinely unavailable. Unlike the old
/// `Default` impl, it is greppable and obviously intentional, so it can't be
/// produced by accident.
pub fn synthetic() -> Self {
Self {
source_id: 0,
start_position: SourcePosition::default(),
end_position: SourcePosition::default(),
}
}
/// Whether this span is the non-located placeholder produced by
/// [`SourceSlice::synthetic`].
pub fn is_synthetic(&self) -> bool {
*self == Self::synthetic()
}
pub fn from_positions(
source_id: SourceId,
start_position: SourcePosition,