DzLox

:)
git clone https://git.sr.ht/~ashymad/DzLox
Log | Files | Refs | Submodules | LICENSE

scanner.zig (7902B)

      1 const std = @import("std");
      2 
      3 pub const TokenType = enum {
      4     LEFT_PAREN,
      5     RIGHT_PAREN,
      6     LEFT_BRACE,
      7     RIGHT_BRACE,
      8     LEFT_BRACKET,
      9     RIGHT_BRACKET,
     10     COMMA,
     11     DOT,
     12     MINUS,
     13     PLUS,
     14     COLON,
     15     SEMICOLON,
     16     SLASH,
     17     STAR,
     18     QUESTION,
     19     // One or two character tokens.
     20     BANG,
     21     BANG_EQUAL,
     22     EQUAL,
     23     EQUAL_EQUAL,
     24     GREATER,
     25     GREATER_EQUAL,
     26     LESS,
     27     LESS_EQUAL,
     28     // Literals.
     29     IDENTIFIER,
     30     STRING,
     31     CHAR,
     32     NUMBER,
     33     // Keywords.
     34     AND,
     35     CLASS,
     36     ELSE,
     37     FALSE,
     38     FOR,
     39     FUN,
     40     IF,
     41     NIL,
     42     OR,
     43     PRINT,
     44     RETURN,
     45     SUPER,
     46     THIS,
     47     TRUE,
     48     VAR,
     49     CON,
     50     WHILE,
     51     SWITCH,
     52     CASE,
     53     DEFAULT,
     54 
     55     EOF,
     56 };
     57 
     58 pub const ScannerError = error{ UnexpectedCharacter, UnknownCharacter, UnterminatedString, UnterminatedChar, EmptyToken };
     59 
     60 pub fn ScannerErrorString(err: ScannerError) []const u8 {
     61     return switch (err) {
     62         ScannerError.UnknownCharacter => "Unknown Character",
     63         ScannerError.UnexpectedCharacter => "Unexpected Character",
     64         ScannerError.UnterminatedString => "Unterminated String",
     65         ScannerError.UnterminatedChar => "Unterminated Char",
     66         ScannerError.EmptyToken => "Empty Token",
     67     };
     68 }
     69 
     70 pub const Token = struct {
     71     type: ScannerError!TokenType,
     72     lexeme: []const u8,
     73     line: i32,
     74     column: usize,
     75 
     76     pub const Empty = @This(){ .type = ScannerError.EmptyToken, .lexeme = "", .line = -1, .column = 1 };
     77 };
     78 
     79 pub const Scanner = struct {
     80     const identifiers = std.StaticStringMap(TokenType).initComptime(.{
     81         .{ "and", TokenType.AND },
     82         .{ "class", TokenType.CLASS },
     83         .{ "else", TokenType.ELSE },
     84         .{ "false", TokenType.FALSE },
     85         .{ "for", TokenType.FOR },
     86         .{ "fun", TokenType.FUN },
     87         .{ "if", TokenType.IF },
     88         .{ "nil", TokenType.NIL },
     89         .{ "or", TokenType.OR },
     90         .{ "print", TokenType.PRINT },
     91         .{ "return", TokenType.RETURN },
     92         .{ "super", TokenType.SUPER },
     93         .{ "this", TokenType.THIS },
     94         .{ "true", TokenType.TRUE },
     95         .{ "var", TokenType.VAR },
     96         .{ "con", TokenType.CON },
     97         .{ "while", TokenType.WHILE },
     98         .{ "switch", TokenType.SWITCH },
     99         .{ "case", TokenType.CASE },
    100         .{ "default", TokenType.DEFAULT },
    101     });
    102 
    103     pub fn init(source: []const u8) !@This() {
    104         return @This(){ .start = source.ptr, .current = source.ptr, .end = source.ptr + source.len, .line_ptr = source.ptr, .line = 1 };
    105     }
    106 
    107     pub fn scanToken(self: *@This()) Token {
    108         self.skipWhitespace();
    109 
    110         self.start = self.current;
    111 
    112         if (self.isAtEnd()) return self.makeToken(TokenType.EOF);
    113 
    114         switch (self.advance()) {
    115             '(' => return self.makeToken(TokenType.LEFT_PAREN),
    116             ')' => return self.makeToken(TokenType.RIGHT_PAREN),
    117             '{' => return self.makeToken(TokenType.LEFT_BRACE),
    118             '}' => return self.makeToken(TokenType.RIGHT_BRACE),
    119             '[' => return self.makeToken(TokenType.LEFT_BRACKET),
    120             ']' => return self.makeToken(TokenType.RIGHT_BRACKET),
    121             ':' => return self.makeToken(TokenType.COLON),
    122             ';' => return self.makeToken(TokenType.SEMICOLON),
    123             ',' => return self.makeToken(TokenType.COMMA),
    124             '.' => return self.makeToken(TokenType.DOT),
    125             '-' => return self.makeToken(TokenType.MINUS),
    126             '+' => return self.makeToken(TokenType.PLUS),
    127             '/' => return self.makeToken(TokenType.SLASH),
    128             '*' => return self.makeToken(TokenType.STAR),
    129             '?' => return self.makeToken(TokenType.QUESTION),
    130             '!' => return self.makeToken(if (self.match('=')) TokenType.BANG_EQUAL else TokenType.BANG),
    131             '=' => return self.makeToken(if (self.match('=')) TokenType.EQUAL_EQUAL else TokenType.EQUAL),
    132             '<' => return self.makeToken(if (self.match('=')) TokenType.LESS_EQUAL else TokenType.LESS),
    133             '>' => return self.makeToken(if (self.match('=')) TokenType.GREATER_EQUAL else TokenType.GREATER),
    134             '"' => return self.string(),
    135             '\'' => return self.char(),
    136             '0'...'9' => return self.number(),
    137             'a'...'z', 'A'...'Z', '_' => return self.identifier(),
    138             else => return self.makeToken(ScannerError.UnknownCharacter),
    139         }
    140 
    141         return self.makeToken(ScannerError.UnexpectedCharacter);
    142     }
    143 
    144     fn string(self: *@This()) Token {
    145         while (self.peek() != '"' and !self.isAtEnd()) {
    146             if (self.peek() == '\n') self.line += 1;
    147             _ = self.advance();
    148         }
    149 
    150         if (self.isAtEnd()) return self.makeToken(ScannerError.UnterminatedString);
    151 
    152         _ = self.advance();
    153 
    154         return self.makeToken(TokenType.STRING);
    155     }
    156 
    157     fn char(self: *@This()) Token {
    158         if (self.isAtEnd()) return self.makeToken(ScannerError.UnterminatedChar);
    159         _ = self.advance();
    160         if (self.isAtEnd() or self.peek() != '\'') return self.makeToken(ScannerError.UnterminatedChar);
    161         _ = self.advance();
    162 
    163         return self.makeToken(TokenType.CHAR);
    164     }
    165 
    166     fn skipWhitespace(self: *@This()) void {
    167         while (true) {
    168             switch (self.peek()) {
    169                 ' ', '\r', '\t' => _ = self.advance(),
    170                 '\n' => {
    171                     self.line += 1;
    172                     _ = self.advance();
    173                     self.line_ptr = self.current;
    174                 },
    175                 '/' => {
    176                     if (self.peekNext() == '/') {
    177                         while (self.peek() != '\n' and !self.isAtEnd()) _ = self.advance();
    178                     } else return;
    179                 },
    180                 else => return,
    181             }
    182         }
    183     }
    184 
    185     fn number(self: *@This()) Token {
    186         while (isDigit(self.peek())) _ = self.advance();
    187 
    188         if (self.peek() == '.' and isDigit(self.peekNext())) {
    189             _ = self.advance();
    190             while (isDigit(self.peek())) _ = self.advance();
    191         }
    192 
    193         return self.makeToken(TokenType.NUMBER);
    194     }
    195 
    196     fn identifier(self: *@This()) Token {
    197         while (isAlpha(self.peek()) or isDigit(self.peek())) _ = self.advance();
    198         return self.makeToken(self.identifierType());
    199     }
    200 
    201     fn identifierType(self: *const @This()) TokenType {
    202         if (identifiers.get(self.lexeme())) |tok| {
    203             return tok;
    204         } else {
    205             return TokenType.IDENTIFIER;
    206         }
    207     }
    208 
    209     fn advance(self: *@This()) u8 {
    210         self.current += 1;
    211         return (self.current - 1)[0];
    212     }
    213 
    214     fn isDigit(c: u8) bool {
    215         return c >= '0' and c <= '9';
    216     }
    217 
    218     fn isAlpha(c: u8) bool {
    219         return (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or c == '_';
    220     }
    221 
    222     fn peek(self: *const @This()) u8 {
    223         return self.current[0];
    224     }
    225 
    226     fn peekNext(self: *const @This()) u8 {
    227         return if (self.isAtEnd()) return 0 else self.current[1];
    228     }
    229 
    230     fn match(self: *@This(), expected: u8) bool {
    231         if (self.isAtEnd()) return false;
    232         if (self.current[0] != expected) return false;
    233         self.current += 1;
    234         return true;
    235     }
    236 
    237     fn isAtEnd(self: *const @This()) bool {
    238         return self.current == self.end;
    239     }
    240 
    241     fn makeToken(self: *const @This(), tokentype: ScannerError!TokenType) Token {
    242         return Token{ .type = tokentype, .lexeme = self.lexeme(), .line = self.line, .column = self.column() };
    243     }
    244 
    245     fn lexeme_len(self: *const @This()) usize {
    246         return @intFromPtr(self.current) - @intFromPtr(self.start);
    247     }
    248 
    249     fn column(self: *const @This()) usize {
    250         return 1 + @intFromPtr(self.start) - @intFromPtr(self.line_ptr);
    251     }
    252 
    253     fn lexeme(self: *const @This()) []const u8 {
    254         return self.start[0..self.lexeme_len()];
    255     }
    256 
    257     start: [*]const u8,
    258     current: [*]const u8,
    259     end: [*]const u8,
    260     line_ptr: [*]const u8,
    261     line: i32,
    262 };