commit 80437b724f906283996193ab6c834446600e502f
parent 413070b52ff8ec387b559d48d4e3b1d2b2c4f0c4
Author: Szymon Mikulicz <szymon.mikulicz@posteo.net>
Date: Sun, 2 Oct 2022 23:46:41 +0200
Control flow p1
Diffstat:
16 files changed, 954 insertions(+), 196 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -14,3 +14,4 @@ lox-test-*
*.obj
*.lst
*.dlox_history
+dlox
diff --git a/dlox b/dlox
Binary files differ.
diff --git a/source/app.d b/source/app.d
@@ -2,11 +2,16 @@ import std.stdio;
import std.file;
import std.container;
import token;
-import scaner;
+import scanner;
import deimos.linenoise;
import std.string;
import std.conv;
import keywords;
+import tokentype;
+import expr;
+import parser;
+import error;
+import interpreter;
int main(string[] args) {
Lox lox = new Lox;
@@ -18,21 +23,25 @@ int main(string[] args) {
} else {
lox.runPrompt();
}
+ if (lox.hadRuntimeError) return 70;
if (lox.hadError) return 65;
return 0;
}
class Lox {
static bool hadError = false;
+ static bool hadRuntimeError = false;
extern(C) static void completion(const char *buf, linenoiseCompletions *lc) {
auto bufs = fromStringz(buf);
- auto lastsp = lastIndexOfAny(bufs, [' ', '\t']) + 1;
+ auto lastsp = 1 + lastIndexOfAny(bufs, [
+ ' ', '\t', '=', '+', '-', '<', '>',
+ '/', '*', '(', ')', '{', '}']);
auto klen = bufs.length - lastsp;
if (bufs.length <= 0) return;
foreach (key; keywords.keywords.keys) {
- if (klen <= key.length && bufs[lastsp..bufs.length] == key[0..klen]) {
+ if (klen <= key.length && bufs[lastsp..$] == key[0..klen]) {
linenoiseAddCompletion(lc, toStringz(bufs[0..lastsp] ~ key));
}
}
@@ -50,7 +59,7 @@ class Lox {
linenoiseHistoryLoad(history);
while((line = linenoise("lox> ")) !is null) {
- if (line[0] != '\0' && line[0] != '/') {
+ if (line[0] != '\0') {
linenoiseHistoryAdd(line);
linenoiseHistorySave(history);
}
@@ -61,17 +70,39 @@ class Lox {
void run(char[] source) {
Scanner scanner = new Scanner(source);
-
auto tokens = scanner.scanTokens();
- foreach (token; tokens) {
- writeln(token);
- }
+ if(hadError) return;
+
+ Parser parser = new Parser(tokens);
+ auto expression = parser.parse();
+ if(hadError) return;
+
+ Interpreter interpreter = new Interpreter();
+ auto result = interpreter.interpret(expression);
+ if(hadError) return;
+
+ if (!result.empty()) writeln(result);
+
}
static void error(int line, string msg) {
report(line, "", msg);
}
+ static void error(const TokenI token, string message) {
+ if (token.type == TokenType.EOF) {
+ report(token.line, " at end", message);
+ } else {
+ report(token.line, " at '" ~ token.lexeme ~ "'", message);
+ }
+ }
+
+ static void error(RuntimeError err) {
+ error(err.token, err.msg);
+ hadRuntimeError = true;
+ }
+
+
static void report(int line, string where, string msg) {
writefln("[line %s] Error%s: %s", line, where, msg);
hadError = true;
diff --git a/source/astgen.d b/source/astgen.d
@@ -0,0 +1,18 @@
+import std.array;
+import std.format;
+import std.algorithm.iteration;
+import std.uni;
+import std.algorithm.iteration;
+
+template GenVisitee(immutable string basename, immutable string[][] names) {
+ const char[] GenVisitee = format("interface %s { void accept(Visitor visitor); }", basename) ~
+ names.map!(name => format(
+ "class %s:%s{%s;this(%s){%s;}void accept(Visitor visitor){visitor.visit(this);}}",
+ name[0], basename, name[1..$].join(";"), name[1..$].join(","),
+ name[1..$].map!(s => "this." ~ [s.split(" ")[$-1]].replicate(2).join("=")).join(";"))).join();
+}
+
+template GenVisitor(immutable string[][] names) {
+ const char[] GenVisitor = format("interface Visitor{%s;}",
+ names.map!(s => format("void visit(%s %s)", s[0], "_" ~ toLower(s[0]))).join(";"));
+}
diff --git a/source/astprinter.d b/source/astprinter.d
@@ -0,0 +1,44 @@
+import std.conv;
+import std.algorithm.iteration;
+import std.array;
+import expr;
+
+class AstPrinter : Visitor {
+ string printed;
+
+ string print(Expr expr) {
+ expr.accept(this);
+ return printed;
+ }
+ void visit(Binary binary) {
+ printed = parenthesize(binary.operator.lexeme,
+ binary.left, binary.right);
+ }
+ void visit(Logical binary) {
+ printed = parenthesize(binary.operator.lexeme,
+ binary.left, binary.right);
+ }
+ void visit(Ternary ternary) {
+ printed = parenthesize(ternary.operator.lexeme,
+ ternary.left, ternary.middle, ternary.right);
+ }
+ void visit(Grouping grouping) {
+ printed = parenthesize("group", grouping.expression);
+ }
+ void visit(Assign assign) {
+ printed = parenthesize("= " ~ assign.name.lexeme, assign.value);
+ }
+ void visit(Variable variable) {
+ printed = variable.name.lexeme;
+ }
+ void visit(Literal literal) {
+ if (literal.value == null) printed = "nil";
+ printed = literal.value.toString();
+ }
+ void visit(Unary unary) {
+ printed = parenthesize(unary.operator.lexeme, unary.right);
+ }
+ private string parenthesize(string name, Expr[] exprs ...) {
+ return "(" ~ name ~ " " ~ exprs.map!(e => print(e)).join(" ") ~ ")";
+ }
+}
diff --git a/source/environment.d b/source/environment.d
@@ -0,0 +1,38 @@
+import std.variant;
+import token;
+import error;
+
+class Environment {
+ private Variant[string] values;
+
+ Environment enclosing;
+
+ this() {
+ enclosing = null;
+ }
+
+ this(Environment env) {
+ enclosing = env;
+ }
+
+ void define(string name, Variant value) {
+ values[name] = value;
+ }
+
+ private Variant* find(TokenI name) {
+ Variant* value = name.lexeme in values;
+ if (value is null) {
+ if (enclosing !is null) return enclosing.find(name);
+ throw new RuntimeError(name, "Undefined variable '" ~ name.lexeme ~ "'.");
+ }
+ return value;
+ }
+
+ Variant get(TokenI name) {
+ return *find(name);
+ }
+
+ void assign(TokenI name, Variant new_value) {
+ *find(name) = new_value;
+ }
+}
diff --git a/source/error.d b/source/error.d
@@ -0,0 +1,11 @@
+import std.exception;
+import token;
+
+class RuntimeError : Exception {
+ const TokenI token;
+
+ this(TokenI token, string msg, string file = __FILE__, size_t line = __LINE__) {
+ this.token = token;
+ super(msg, file, line);
+ }
+}
diff --git a/source/expr.d b/source/expr.d
@@ -0,0 +1,16 @@
+import astgen;
+import token;
+import std.variant;
+
+static immutable string[][] expressions = [
+ ["Ternary", "Expr left", "TokenI operator", "Expr middle", "Expr right"],
+ ["Binary", "Expr left", "TokenI operator", "Expr right"],
+ ["Grouping", "Expr expression"],
+ ["Literal", "Variant value"],
+ ["Unary", "TokenI operator", "Expr right"],
+ ["Variable", "TokenI name"],
+ ["Assign", "TokenI name", "Expr value"],
+ ["Logical", "Expr left", "TokenI operator", "Expr right"],
+];
+
+mixin(GenVisitor!(expressions) ~ GenVisitee!("Expr", expressions));
diff --git a/source/interpreter.d b/source/interpreter.d
@@ -0,0 +1,226 @@
+import expr;
+import stmt;
+import std.variant;
+import tokentype;
+import token;
+import std.format;
+import error;
+import app;
+import std.algorithm;
+import astprinter;
+import std.stdio;
+import std.container;
+import environment;
+
+class Interpreter : stmt.Visitor, expr.Visitor {
+ Variant value;
+ static Environment environment;
+
+ static this() {
+ environment = new Environment();
+ }
+
+ string interpret(Array!Stmt statements) {
+ try {
+ foreach(statement; statements) {
+ execute(statement);
+ }
+ } catch (RuntimeError error) {
+ Lox.error(error);
+ }
+ return value == null ? "" : stringify(value);
+ }
+
+ private string stringify(Variant var) {
+ if (var == null) return "nil";
+
+ string str = var.toString();
+
+ return str;
+ }
+
+ private void execute(Stmt stmt) {
+ stmt.accept(this);
+ }
+
+ void visit(Expression stmt) {
+ evaluate(stmt.expression);
+ }
+
+ void visit(Print stmt) {
+ writeln(stringify(evaluate(stmt.expression)));
+ value = null;
+ }
+
+ void visit(Var stmt) {
+ Variant variant;
+ if (stmt.initializer !is null) {
+ variant = evaluate(stmt.initializer);
+ }
+
+ environment.define(stmt.name.lexeme, variant);
+ value = null;
+ }
+
+ void visit(Block stmt) {
+ executeBlock(stmt.statements, new Environment(environment));
+ }
+
+ void visit(If stmt) {
+ if (isTruthy(evaluate(stmt.condition))) {
+ execute(stmt.thenBranch);
+ } else if (stmt.elseBranch !is null) {
+ execute(stmt.elseBranch);
+ }
+ }
+
+ void executeBlock(Stmt[] statements, Environment environment) {
+ Environment previous = this.environment;
+ try {
+ this.environment = environment;
+
+ foreach(statement; statements) {
+ execute(statement);
+ }
+ } finally {
+ this.environment = previous;
+ }
+ }
+
+ void visit(Assign expr) {
+ Variant variant = evaluate(expr.value);
+ environment.assign(expr.name, variant);
+ value = variant;
+ }
+
+ void visit(Variable expr) {
+ Variant var = environment.get(expr.name);
+ if (!var.hasValue())
+ throw new RuntimeError(expr.name, "Variable is not initialized.");
+ value = var;
+ }
+
+ void visit(Logical expr) {
+ Variant left = evaluate(expr.left);
+
+ if (expr.operator.type == TokenType.OR && isTruthy(left)) {
+ value = left;
+ } else if (expr.operator.type == TokenType.AND && !isTruthy(left)) {
+ value = left;
+ } else {
+ value = evaluate(expr.right);
+ }
+ }
+
+ void visit(Binary binary) {
+ Variant left = evaluate(binary.left);
+ Variant right = evaluate(binary.right);
+
+ with (TokenType) switch (binary.operator.type) {
+ case MINUS:
+ checkNumberOperands(binary.operator, left, right);
+ value = left.get!(double) - right.get!(double);
+ break;
+ case SLASH:
+ checkNumberOperands(binary.operator, left, right);
+ if (right.get!(double) == 0)
+ throw new RuntimeError(binary.operator, "Division by zero");
+ value = left.get!(double) / right.get!(double);
+ break;
+ case STAR:
+ checkNumberOperands(binary.operator, left, right);
+ value = left.get!(double) * right.get!(double);
+ break;
+ case PLUS:
+ if (left.convertsTo!(double) && right.convertsTo!(double)) {
+ value = left.get!(double) + right.get!(double);
+ } else if (left.convertsTo!(string) && right.convertsTo!(string)) {
+ value = left.get!(string) ~ right.get!(string);
+ } else throw new RuntimeError(binary.operator, "Operants can be doubles or strings");
+ break;
+ case GREATER:
+ checkNumberOperands(binary.operator, left, right);
+ value = left.get!(double) > right.get!(double);
+ break;
+ case GREATER_EQUAL:
+ checkNumberOperands(binary.operator, left, right);
+ value = left.get!(double) >= right.get!(double);
+ break;
+ case LESS:
+ checkNumberOperands(binary.operator, left, right);
+ value = left.get!(double) < right.get!(double);
+ break;
+ case LESS_EQUAL:
+ checkNumberOperands(binary.operator, left, right);
+ value = left.get!(double) <= right.get!(double);
+ break;
+ case EQUAL_EQUAL:
+ value = left == right;
+ break;
+ case BANG_EQUAL:
+ value = left != right;
+ break;
+ case COMMA:
+ value = right;
+ break;
+ default:
+ assert(0);
+ }
+ }
+
+ void visit(Ternary ternary) {
+ assert(ternary.operator.type == TokenType.QUERY);
+ if(isTruthy(evaluate(ternary.left))) {
+ value = evaluate(ternary.middle);
+ } else {
+ value = evaluate(ternary.right);
+ }
+ }
+
+ void visit(Grouping grouping) {
+ value = evaluate(grouping.expression);
+ }
+
+ void visit(Literal literal) {
+ value = literal.value;
+ }
+
+ void visit(Unary unary) {
+ Variant right;
+ if (unary.operator.type != TokenType.AST)
+ right = evaluate(unary.right);
+
+ with (TokenType) switch (unary.operator.type) {
+ case MINUS:
+ checkNumberOperands(unary.operator, right);
+ value = -right.get!(double);
+ break;
+ case BANG:
+ value = !isTruthy(right);
+ break;
+ case AST:
+ value = new AstPrinter().print(unary.right);
+ break;
+ default: assert(0);
+ }
+ }
+
+ private bool isTruthy(Variant var) {
+ if (var == null) return false;
+ if (var.type() == typeid(bool)) return var.get!(bool);
+ return true;
+ }
+
+ private void checkNumberOperands(TokenI operator, Variant[] vars...) {
+ foreach(var; vars) {
+ if (!var.convertsTo!(double))
+ throw new RuntimeError(operator,
+ format("Operand %s has wrong type: %s, double expected", stringify(var), var.type()));
+ }
+ }
+
+ private Variant evaluate(Expr expr) {
+ expr.accept(this);
+ return value;
+ }
+}
diff --git a/source/keywords.d b/source/keywords.d
@@ -21,6 +21,7 @@ shared static this() {
"true": TRUE,
"var": VAR,
"while": WHILE,
+ "ast": AST,
];
}
}
diff --git a/source/parser.d b/source/parser.d
@@ -0,0 +1,302 @@
+import token;
+import tokentype;
+import std.container;
+import std.variant;
+import expr;
+import stmt;
+import app;
+
+/*
+program → statement* EOF ;
+declaration → varDecl
+ | statement ;
+varDecl → "var" IDENTIFIER ( "=" expression )? ";" ;
+statement → exprStmt
+ | printStmt
+ | ifStmt
+ | block ;
+ifStmt → "if" "(" expression ")" statement
+ ( "else" statement )? ;
+block → "{" declaration* "}" ;
+exprStmt → expression ";" ;
+printStmt → "print" expression ";" ;
+expression → separator ;
+separator → assignment ( "," assignment )* ;
+assignment → IDENTIFIER "=" assignment
+ | ternary ;
+ternary → logic_or ( "?" expression ":" ternary )? ;
+logic_or → logic_and ( "or" logic_and )* ;
+logic_and → equality ( "and" equality )* ;
+equality → comparison ( ( "!=" | "==" ) comparison )* ;
+comparison → term ( ( ">" | ">=" | "<" | "<=" ) term )* ;
+term → factor ( ( "-" | "+" ) factor )* ;
+factor → unary ( ( "/" | "*" ) unary )* ;
+unary → ( "!" | "-" | "ast" ) unary
+ | primary ;
+primary → NUMBER | STRING | "true" | "false" | "nil"
+ | "(" expression ")" | IDENTIFIER ;
+*/
+
+class Parser {
+ private static class ParseError : Exception {
+ this(string msg, string file = __FILE__, size_t line = __LINE__) {
+ super(msg, file, line);
+ }
+ }
+
+ private Array!TokenI tokens;
+ private int current = 0;
+
+ this(Array!TokenI tokens) {
+ this.tokens = tokens;
+ }
+
+ Array!Stmt parse() {
+ Array!Stmt statements = Array!Stmt();
+ while (!isAtEnd()) {
+ statements.insert(declaration());
+ }
+ return statements;
+ }
+
+ private Stmt declaration() {
+ try {
+ if (match(TokenType.VAR))
+ return varDeclaration();
+ else
+ return matchStatement();
+ } catch (ParseError err) {
+ synchronize();
+ return null;
+ }
+ }
+
+ private Stmt varDeclaration() {
+ TokenI name = consume(TokenType.IDENTIFIER, "Expect variable name.");
+ Expr initializer = match(TokenType.EQUAL) ? expression() : null;
+
+ return statement!(Var)(name, initializer);
+ }
+
+ private Stmt matchStatement() {
+ if (match(TokenType.PRINT))
+ return statement!(Print)(expression());
+ if (match(TokenType.LEFT_BRACE))
+ return statement!(Block)(block());
+ if (match(TokenType.IF))
+ return ifStatement();
+ return statement!(Expression)(expression());
+ }
+
+ private Stmt ifStatement() {
+ consume(TokenType.LEFT_PAREN, "Expect '(' after 'if'.");
+ Expr condition = expression();
+ consume(TokenType.RIGHT_PAREN, "Expect ')' after if condition.");
+
+ Stmt thenBranch = matchStatement();
+ Stmt elseBranch = null;
+
+ if (match(TokenType.ELSE)) {
+ elseBranch = matchStatement();
+ }
+
+ return statement!(If)(condition, thenBranch, elseBranch);
+ }
+
+ private Stmt[] block() {
+ Stmt[] statements;
+
+ while(!check(TokenType.RIGHT_BRACE) && !isAtEnd()) {
+ statements ~= declaration();
+ }
+
+ consume(TokenType.RIGHT_BRACE, "Expect '}' after block.");
+ return statements;
+ }
+
+ private Stmt statement(T, A...)(A a) {
+ if (!isAtEnd()
+ && !match(TokenType.SEMICOLON)
+ && !check(TokenType.RIGHT_BRACE)
+ && !check(TokenType.ELSE)
+ && previous().type != TokenType.SEMICOLON
+ && previous().type != TokenType.RIGHT_BRACE)
+ error(previous(), "Expect ';' after value.");
+ return new T(a);
+ }
+
+ private Expr expression() {
+ return separator();
+ }
+
+ private Expr separator() {
+ with (TokenType) return rule!(Binary)(&assignment, COMMA);
+ }
+
+ private Expr assignment() {
+ Expr expr = ternary();
+ if (match(TokenType.EQUAL)) {
+ TokenI equals = previous();
+ Expr value = assignment();
+
+ if (auto variable = cast(Variable) expr) {
+ return new Assign(variable.name, value);
+ }
+
+ error(equals, "Invalid assignment target.");
+ }
+ return expr;
+ }
+
+ private Expr ternary() {
+ Expr expr = or();
+
+ if (match(TokenType.QUERY)) {
+ TokenI operator = previous();
+
+ Expr middle = expression();
+
+ consume(TokenType.COLON, "':' expected");
+
+ expr = new Ternary(expr, operator, middle, ternary);
+ }
+
+ return expr;
+ }
+
+ private Expr or() {
+ with (TokenType) return rule!(Logical)(&and, OR);
+ }
+
+ private Expr and() {
+ with (TokenType) return rule!(Logical)(&equality, AND);
+ }
+
+ private Expr equality() {
+ with (TokenType) return rule!(Binary)(&comparison, BANG_EQUAL, EQUAL_EQUAL);
+ }
+
+ private Expr comparison() {
+ with (TokenType) return rule!(Binary)(&term, GREATER, GREATER_EQUAL, LESS, LESS_EQUAL);
+ }
+
+ private Expr term() {
+ with (TokenType) return rule!(Binary)(&factor, MINUS, PLUS);
+ }
+
+ private Expr factor() {
+ with (TokenType) return rule!(Binary)(&unary, SLASH, STAR);
+ }
+
+ private Expr unary() {
+ with (TokenType) if (match(BANG, MINUS, AST)) {
+ TokenI operator = previous();
+ Expr right = primary();
+ return new Unary(operator, right);
+ }
+ return primary();
+ }
+
+ private Expr primary() {
+ with (TokenType) {
+ if (match(FALSE))
+ return new Literal(Variant(false));
+ if (match(TRUE))
+ return new Literal(Variant(true));
+ if (match(NIL))
+ return new Literal(Variant(null));
+ if (match(IDENTIFIER))
+ return new Variable(previous());
+
+ if (match(NUMBER, STRING)) {
+ return new Literal(previous().literal);
+ }
+
+ if (match(LEFT_PAREN)) {
+ Expr expr = expression();
+ consume(RIGHT_PAREN, "Expect ')' after expression.");
+ return new Grouping(expr);
+ }
+ }
+
+ throw error(peek(), "Expression expected");
+ }
+
+ private Expr rule(T)(Expr delegate() rule, TokenType[] types ...) {
+ Expr expr = rule();
+
+ while (match(types)) {
+ TokenI operator = previous();
+ Expr right = rule();
+ expr = new T(expr, operator, right);
+ }
+ return expr;
+ }
+
+ private TokenI consume(TokenType type, string message) {
+ if (check(type)) return advance();
+
+ throw error(peek(), message);
+ }
+
+ private ParseError error(TokenI token, string message) {
+ Lox.error(token, message);
+ return new ParseError(message);
+ }
+
+ private void synchronize() {
+ advance();
+
+ with (TokenType) while (!isAtEnd()) {
+ if (previous().type == SEMICOLON) return;
+
+ switch (peek().type) {
+ case CLASS:
+ case FUN:
+ case VAR:
+ case FOR:
+ case IF:
+ case WHILE:
+ case PRINT:
+ case RETURN:
+ return;
+ default: break;
+ }
+
+ advance();
+ }
+ }
+
+ private bool match(TokenType[] types ...) {
+ foreach (type; types) {
+ if (check(type)) {
+ advance();
+ return true;
+ }
+ }
+ return false;
+ }
+ private bool check(TokenType type) {
+ if (isAtEnd()) return false;
+ return peek().type == type;
+ }
+ private TokenI advance() {
+ if (!isAtEnd()) current++;
+ return previous();
+ }
+ private bool isAtEnd() {
+ return peek().type == TokenType.EOF;
+ }
+
+ private TokenI peek() {
+ return tokens[current];
+ }
+
+ private TokenI peekNext() {
+ return tokens[current + 1];
+ }
+
+ private TokenI previous() {
+ return tokens[current - 1];
+ }
+}
diff --git a/source/scaner.d b/source/scaner.d
@@ -1,171 +0,0 @@
-import token;
-import tokentype;
-import std.container;
-import std.format;
-import std.conv;
-import app;
-
-class Scanner {
- private string source;
- private int start = 0;
- private int current = 0;
- private int line = 0;
-
- private Array!Token tokens;
-
- this(string source) {
- this.source = source;
- this.tokens = Array!Token();
- }
-
- this(char[] source) {
- this.source = to!string(source);
- this.tokens = Array!Token();
- }
-
- Array!Token scanTokens() {
- while (!isAtEnd()) {
- start = current;
- scanToken();
- }
- tokens.insert(new Token(TokenType.EOF, "", null, line));
- return tokens;
- }
-
- void scanToken() {
- char c = advance();
- with (TokenType) {
- switch (c) {
- case '(': addToken(LEFT_PAREN); break;
- case ')': addToken(RIGHT_PAREN); break;
- case '{': addToken(LEFT_BRACE); break;
- case '}': addToken(RIGHT_BRACE); break;
- case ',': addToken(COMMA); break;
- case '.': addToken(DOT); break;
- case '-': addToken(MINUS); break;
- case '+': addToken(PLUS); break;
- case ';': addToken(SEMICOLON); break;
- case '*': addToken(STAR); break;
- case '!':
- addToken(match('=') ? BANG_EQUAL : BANG);
- break;
- case '=':
- addToken(match('=') ? EQUAL_EQUAL : EQUAL);
- break;
- case '<':
- addToken(match('=') ? LESS_EQUAL : LESS);
- break;
- case '>':
- addToken(match('=') ? GREATER_EQUAL : GREATER);
- break;
- case '/':
- if (match('/')) {
- // A comment goes until the end of the line.
- while (peek() != '\n' && !isAtEnd()) advance();
- } else if (match('*')) {
- int nest = 1;
- while (nest > 0) {
- while ((peek() != '*' || peekNext() != '/') && !isAtEnd()) {
- if (peek() == '\n') line++;
- if (peek() == '/' && peekNext() == '*') nest++;
- advance();
- }
- advance();
- advance();
- nest--;
- }
- } else {
- addToken(SLASH);
- }
- break;
- case '"': stringToken(); break;
- case ' ':
- case '\r':
- case '\t':
- break;
- case '\n':
- line++;
- break;
- default:
- if (isDigit(c)) {
- numberToken();
- } else {
- Lox.error(line, format("Unexpected character: %c", c));
- }
- break;
- }
- }
- }
-
- private char peekNext() {
- if (current + 1 >= source.length) return '\0';
- return source[current + 1];
- }
-
- private void numberToken() {
- while (isDigit(peek())) advance();
-
- // Look for a fractional part.
- if (peek() == '.' && isDigit(peekNext())) {
- // Consume the "."
- advance();
-
- while (isDigit(peek())) advance();
- }
-
- addToken(TokenType.NUMBER,
- to!double(source[start..current]));
- }
-
- private bool isDigit(char c) {
- return c >= '0' && c <= '9';
- }
-
- private void stringToken() {
- while (peek() != '"' && !isAtEnd()) {
- if (peek() == '\n') line++;
- advance();
- }
-
- if (isAtEnd()) {
- Lox.error(line, "Unterminated string.");
- return;
- }
-
- // The closing ".
- advance();
-
- // Trim the surrounding quotes.
- string value = source[start + 1..current - 1];
- addToken(TokenType.STRING, value);
- }
-
- private char peek() {
- if (isAtEnd()) return '\0';
- return source[current];
- }
-
- private bool match(char expected) {
- if (isAtEnd()) return false;
- if (source[current] != expected) return false;
-
- current++;
- return true;
- }
-
- private char advance() {
- return source[current++];
- }
-
- private void addToken(TokenType type) {
- addToken(type, null);
- }
-
- private void addToken(T)(TokenType type, T literal) {
- string text = source[start..current];
- tokens.insert(new Token(type, text, literal, line));
- }
- private bool isAtEnd() {
- return current >= source.length;
- }
-}
diff --git a/source/scanner.d b/source/scanner.d
@@ -0,0 +1,201 @@
+import token;
+import tokentype;
+import std.container;
+import std.format;
+import std.conv;
+import app;
+import keywords;
+
+class Scanner {
+ private string source;
+ private int start = 0;
+ private int current = 0;
+ private int line = 0;
+
+ private Array!TokenI tokens;
+
+ this(string source) {
+ this.source = source;
+ this.tokens = Array!TokenI();
+ }
+
+ this(char[] source) {
+ this.source = to!string(source);
+ this.tokens = Array!TokenI();
+ }
+
+ Array!TokenI scanTokens() {
+ while (!isAtEnd()) {
+ start = current;
+ scanToken();
+ }
+ start = current;
+ addToken(TokenType.EOF);
+ return tokens;
+ }
+
+ void scanToken() {
+ char c = advance();
+ with (TokenType) {
+ switch (c) {
+ case '(': addToken(LEFT_PAREN); break;
+ case ')': addToken(RIGHT_PAREN); break;
+ case '{': addToken(LEFT_BRACE); break;
+ case '}': addToken(RIGHT_BRACE); break;
+ case ',': addToken(COMMA); break;
+ case '.': addToken(DOT); break;
+ case '-': addToken(MINUS); break;
+ case '+': addToken(PLUS); break;
+ case ';': addToken(SEMICOLON); break;
+ case '*': addToken(STAR); break;
+ case '?': addToken(QUERY); break;
+ case ':': addToken(COLON); break;
+ case '!':
+ addToken(match('=') ? BANG_EQUAL : BANG);
+ break;
+ case '=':
+ addToken(match('=') ? EQUAL_EQUAL : EQUAL);
+ break;
+ case '<':
+ addToken(match('=') ? LESS_EQUAL : LESS);
+ break;
+ case '>':
+ addToken(match('=') ? GREATER_EQUAL : GREATER);
+ break;
+ case '/':
+ if (!skipComment()) {
+ addToken(SLASH);
+ }
+ break;
+ case '"': stringToken(); break;
+ case ' ':
+ case '\r':
+ case '\t':
+ break;
+ case '\n':
+ line++;
+ break;
+ default:
+ if (isDigit(c)) {
+ numberToken();
+ } else if (isAlpha(c)) {
+ identifierToken();
+ } else {
+ Lox.error(line, format("Unexpected character: %c", c));
+ }
+ break;
+ }
+ }
+ }
+
+ private bool skipComment() {
+ if (match('/')) {
+ while (peek() != '\n' && !isAtEnd()) advance();
+ return true;
+ } else if (match('*')) {
+ int nest = 1;
+ while (nest > 0) {
+ while ((peek() != '*' || peekNext() != '/') && !isAtEnd()) {
+ if (peek() == '\n') line++;
+ if (peek() == '/' && peekNext() == '*') nest++;
+ advance();
+ }
+ advance();
+ advance();
+ nest--;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ private char peekNext() {
+ if (current + 1 >= source.length) return '\0';
+ return source[current + 1];
+ }
+
+ private void numberToken() {
+ while (isDigit(peek())) advance();
+
+ // Look for a fractional part.
+ if (peek() == '.' && isDigit(peekNext())) {
+ // Consume the "."
+ advance();
+
+ while (isDigit(peek())) advance();
+ }
+
+ addToken(TokenType.NUMBER,
+ to!double(source[start..current]));
+ }
+
+ private bool isDigit(char c) {
+ return c >= '0' && c <= '9';
+ }
+
+ private void stringToken() {
+ while (peek() != '"' && !isAtEnd()) {
+ if (peek() == '\n') line++;
+ advance();
+ }
+
+ if (isAtEnd()) {
+ Lox.error(line, "Unterminated string.");
+ return;
+ }
+
+ // The closing ".
+ advance();
+
+ // Trim the surrounding quotes.
+ string value = source[start + 1..current - 1];
+ addToken(TokenType.STRING, value);
+ }
+
+ private char peek() {
+ if (isAtEnd()) return '\0';
+ return source[current];
+ }
+
+ private void identifierToken() {
+ while (isAlphaNumeric(peek())) advance();
+
+ string text = source[start..current];
+ TokenType type = keywords.keywords.get(text, TokenType.IDENTIFIER);
+ addToken(type);
+ }
+
+ private bool isAlpha(char c) {
+ return (c >= 'a' && c <= 'z') ||
+ (c >= 'A' && c <= 'Z') ||
+ c == '_';
+ }
+
+ private bool isAlphaNumeric(char c) {
+ return isAlpha(c) || isDigit(c);
+ }
+
+ private bool match(char expected) {
+ if (isAtEnd()) return false;
+ if (source[current] != expected) return false;
+
+ current++;
+ return true;
+ }
+
+ private char advance() {
+ return source[current++];
+ }
+
+ private void addToken(TokenType type) {
+ addToken(type, null);
+ }
+
+ private void addToken(T)(TokenType type, T literal) {
+ string text = source[start..current];
+ tokens.insert(TokenI(type, text, literal, line));
+ }
+ private bool isAtEnd() {
+ return current >= source.length;
+ }
+}
diff --git a/source/stmt.d b/source/stmt.d
@@ -0,0 +1,14 @@
+import astgen;
+import token;
+import std.variant;
+import expr;
+
+static immutable string[][] statements = [
+ ["Print", "Expr expression"],
+ ["Expression", "Expr expression"],
+ ["Var", "TokenI name", "Expr initializer"],
+ ["Block", "Stmt[] statements"],
+ ["If", "Expr condition", "Stmt thenBranch", "Stmt elseBranch"],
+];
+
+mixin(GenVisitor!(statements) ~ GenVisitee!("Stmt", statements));
diff --git a/source/token.d b/source/token.d
@@ -1,25 +1,50 @@
import tokentype;
import std.conv;
import std.variant;
+import std.format;
-class Token {
- immutable TokenType type;
- immutable string lexeme;
- const Variant literal;
- immutable int line;
-
- this(T)(TokenType type, string lexeme, T literal, int line) {
- this.type = type;
- this.lexeme = lexeme;
- this.literal = literal;
- this.line = line;
+interface TokenI {
+ void toString(scope void delegate(const(char)[]) sink) const;
+ static Token!T opCall(T)(TokenType type, string lexeme, T literal, int line) {
+ return new Token!T(type, lexeme, literal, line);
+ }
+ @property Variant literal();
+ @property string lexeme() const;
+ @property TokenType type() const;
+ @property int line() const;
+}
+
+class Token(T) : TokenI {
+ const TokenType _type;
+ const string _lexeme;
+ T _literal;
+ const int _line;
+
+ this(TokenType type, string lexeme, T literal, int line) {
+ _type = type;
+ _lexeme = lexeme;
+ _literal = literal;
+ _line = line;
}
void toString(scope void delegate(const(char)[]) sink) const {
- sink(text(type));
- sink(" ");
- sink(lexeme);
- sink(" ");
- sink(text(line));
+ sink(format("Token!(%s)(%s, %s, %s, %s)",
+ T.stringof, _type, _lexeme, _literal, _line));
+ }
+
+ @property Variant literal() {
+ return Variant(_literal);
}
+
+ @property string lexeme() const {
+ return _lexeme;
+ }
+
+ @property TokenType type() const {
+ return _type;
+ }
+ @property int line() const {
+ return _line;
+ }
+
}
diff --git a/source/tokentype.d b/source/tokentype.d
@@ -2,6 +2,7 @@ enum TokenType {
// Single-character tokens.
LEFT_PAREN, RIGHT_PAREN, LEFT_BRACE, RIGHT_BRACE,
COMMA, DOT, MINUS, PLUS, SEMICOLON, SLASH, STAR,
+ COLON, QUERY,
// One or two character tokens.
BANG, BANG_EQUAL,
@@ -14,7 +15,7 @@ enum TokenType {
// Keywords.
AND, CLASS, ELSE, FALSE, FUN, FOR, IF, NIL, OR,
- PRINT, RETURN, SUPER, THIS, TRUE, VAR, WHILE,
+ PRINT, RETURN, SUPER, THIS, TRUE, VAR, WHILE, AST,
EOF
}
