modules

compiler/ast.ts

@@ -1,6 +1,12 @@
+import type { Syms } from "./resolver_syms.ts";
 import { Pos } from "./token.ts";
 import { GenericArgsMap, VType } from "./vtype.ts";
 
+export type Mod = {
+    filePath: string;
+    ast: Stmt[];
+};
+
 export type Stmt = {
     kind: StmtKind;
     pos: Pos;
@@ -9,7 +15,9 @@ export type Stmt = {
 
 export type StmtKind =
     | { type: "error" }
-    | { type: "import"; path: Expr }
+    | { type: "mod_block"; ident: string; stmts: Stmt[] }
+    | { type: "mod_file"; ident: string; filePath: string }
+    | { type: "mod"; ident: string; mod: Mod }
     | { type: "break"; expr?: Expr }
     | { type: "return"; expr?: Expr }
     | {
@@ -106,7 +114,8 @@ export type SymKind =
     | { type: "fn"; stmt: Stmt }
     | { type: "fn_param"; param: Param }
     | { type: "closure"; inner: Sym }
-    | { type: "generic"; stmt: Stmt; genericParam: GenericParam };
+    | { type: "generic"; stmt: Stmt; genericParam: GenericParam }
+    | { type: "mod"; syms: Syms };
 
 export type EType = {
     kind: ETypeKind;

compiler/ast_visitor.ts

@@ -6,7 +6,9 @@ export interface AstVisitor<Args extends unknown[] = []> {
     visitStmts?(stmts: Stmt[], ...args: Args): VisitRes;
     visitStmt?(stmt: Stmt, ...args: Args): VisitRes;
     visitErrorStmt?(stmt: Stmt, ...args: Args): VisitRes;
-    visitImportStmt?(stmt: Stmt, ...args: Args): VisitRes;
+    visitModFileStmt?(stmt: Stmt, ...args: Args): VisitRes;
+    visitModBlockStmt?(stmt: Stmt, ...args: Args): VisitRes;
+    visitModStmt?(stmt: Stmt, ...args: Args): VisitRes;
     visitBreakStmt?(stmt: Stmt, ...args: Args): VisitRes;
     visitReturnStmt?(stmt: Stmt, ...args: Args): VisitRes;
     visitFnStmt?(stmt: Stmt, ...args: Args): VisitRes;
@@ -68,9 +70,16 @@ export function visitStmt<Args extends unknown[] = []>(
         case "error":
             if (v.visitErrorStmt?.(stmt, ...args) == "stop") return;
             break;
-        case "import":
-            if (v.visitImportStmt?.(stmt, ...args) == "stop") return;
-            visitExpr(stmt.kind.path, v, ...args);
+        case "mod_file":
+            if (v.visitModFileStmt?.(stmt, ...args) == "stop") return;
+            break;
+        case "mod_block":
+            if (v.visitModBlockStmt?.(stmt, ...args) == "stop") return;
+            visitStmts(stmt.kind.stmts, v, ...args);
+            break;
+        case "mod":
+            if (v.visitModStmt?.(stmt, ...args) == "stop") return;
+            visitStmts(stmt.kind.mod.ast, v, ...args);
             break;
         case "break":
             if (v.visitBreakStmt?.(stmt, ...args) == "stop") return;

compiler/checker.ts

@@ -1,4 +1,4 @@
-import { EType, Expr, Stmt } from "./ast.ts";
+import { EType, Expr, Stmt, Sym } from "./ast.ts";
 import { printStackTrace, Reporter } from "./info.ts";
 import { Pos } from "./token.ts";
 import {
@@ -69,6 +69,11 @@ export class Checker {
         switch (stmt.kind.type) {
             case "error":
                 return { type: "error" };
+            case "mod_block":
+            case "mod_file":
+                throw new Error("mod declaration in ast, should be resolved");
+            case "mod":
+                return this.checkModStmt(stmt);
             case "break":
                 return this.checkBreakStmt(stmt);
             case "return":
@@ -84,6 +89,17 @@ export class Checker {
         }
     }
 
+    public checkModStmt(stmt: Stmt) {
+        if (stmt.kind.type !== "mod") {
+            throw new Error();
+        }
+        const { ast } = stmt.kind.mod;
+        this.checkFnHeaders(ast);
+        for (const stmt of ast) {
+            this.checkStmt(stmt);
+        }
+    }
+
     public checkBreakStmt(stmt: Stmt) {
         if (stmt.kind.type !== "break") {
             throw new Error();
@@ -346,11 +362,15 @@ export class Checker {
         if (expr.kind.type !== "sym") {
             throw new Error();
         }
-        switch (expr.kind.sym.type) {
+        return this.checkSym(expr.kind.sym);
+    }
+
+    private checkSym(sym: Sym): VType {
+        switch (sym.type) {
             case "let":
-                return expr.kind.sym.param.vtype!;
+                return sym.param.vtype!;
             case "fn": {
-                const fnStmt = expr.kind.sym.stmt!;
+                const fnStmt = sym.stmt!;
                 if (fnStmt.kind.type !== "fn") {
                     throw new Error();
                 }
@@ -361,13 +381,15 @@ export class Checker {
                 return vtype;
             }
             case "fn_param":
-                return expr.kind.sym.param.vtype!;
+                return sym.param.vtype!;
             case "let_static":
             case "closure":
             case "generic":
                 throw new Error(
-                    `not implemented, sym type '${expr.kind.sym.type}'`,
+                    `not implemented, sym type '${sym.type}'`,
                 );
+            case "mod":
+                throw new Error("should already be resolved");
         }
     }
 
@@ -709,7 +731,7 @@ export class Checker {
         if (expr.kind.type !== "path") {
             throw new Error();
         }
-        throw new Error("not implemented");
+        throw new Error("should already be resolved");
     }
 
     public checkETypeArgsExpr(expr: Expr): VType {

compiler/compiler.ts

@@ -1,4 +1,4 @@
-import { AstCreator } from "./ast.ts";
+import { AstCreator, Mod, Stmt } from "./ast.ts";
 import { Checker } from "./checker.ts";
 import { CompoundAssignDesugarer } from "./desugar/compound_assign.ts";
 import { SpecialLoopDesugarer } from "./desugar/special_loop.ts";
@@ -8,13 +8,10 @@ import { Monomorphizer } from "./mono.ts";
 import { FnNamesMap, Lowerer } from "./lowerer.ts";
 import { Parser } from "./parser.ts";
 import { Resolver } from "./resolver.ts";
+import { AstVisitor, VisitRes, visitStmts } from "./ast_visitor.ts";
 
 import * as path from "jsr:@std/path";
-
-export type CompiledFile = {
-    filepath: string;
-    program: number[];
-};
+import { Pos } from "./token.ts";
 
 export type CompileResult = {
     program: number[];
@@ -23,46 +20,118 @@ export type CompileResult = {
 
 export class Compiler {
     private astCreator = new AstCreator();
-    private reporter = new Reporter();
+    private reporter;
 
-    public constructor(private startFilePath: string) {}
+    public constructor(private startFilePath: string) {
+        this.reporter = new Reporter(this.startFilePath);
+    }
 
     public async compile(): Promise<CompileResult> {
-        const text = await Deno.readTextFile(this.startFilePath);
+        const mod = new ModTree(
+            this.startFilePath,
+            this.astCreator,
+            this.reporter,
+        ).resolve();
 
-        const stdlib = await Deno.readTextFile(
-            path.join(
-                path.dirname(path.fromFileUrl(Deno.mainModule)),
-                "../stdlib.slg",
-            ),
-        );
+        new SpecialLoopDesugarer(this.astCreator).desugar(mod.ast);
 
-        const totalText = text + stdlib;
+        new Resolver(this.reporter).resolve(mod.ast);
 
-        const lexer = new Lexer(totalText, this.reporter);
+        new CompoundAssignDesugarer(this.astCreator).desugar(mod.ast);
 
-        const parser = new Parser(lexer, this.astCreator, this.reporter);
-        const ast = parser.parse();
-
-        new SpecialLoopDesugarer(this.astCreator).desugar(ast);
-
-        new Resolver(this.reporter).resolve(ast);
-
-        new CompoundAssignDesugarer(this.astCreator).desugar(ast);
-
-        new Checker(this.reporter).check(ast);
+        new Checker(this.reporter).check(mod.ast);
 
         if (this.reporter.errorOccured()) {
             console.error("Errors occurred, stopping compilation.");
             Deno.exit(1);
         }
 
-        const { monoFns, callMap } = new Monomorphizer(ast).monomorphize();
+        const { monoFns, callMap } = new Monomorphizer(mod.ast).monomorphize();
 
-        const lowerer = new Lowerer(monoFns, callMap, lexer.currentPos());
+        const lastPos = await lastPosInTextFile(this.startFilePath);
+
+        const lowerer = new Lowerer(monoFns, callMap, lastPos);
         const { program, fnNames } = lowerer.lower();
         //lowerer.printProgram();
 
         return { program, fnNames };
     }
 }
+
+export class ModTree implements AstVisitor<[string]> {
+    constructor(
+        private entryFilePath: string,
+        private astCreator: AstCreator,
+        private reporter: Reporter,
+    ) {}
+
+    public resolve(): Mod {
+        const entryAst = this.parseFile(this.entryFilePath);
+
+        visitStmts(entryAst, this, this.entryFilePath);
+
+        return { filePath: this.entryFilePath, ast: entryAst };
+    }
+
+    private parseFile(filePath: string): Stmt[] {
+        const text = Deno.readTextFileSync(filePath);
+
+        const lexer = new Lexer(text, this.reporter);
+
+        const parser = new Parser(lexer, this.astCreator, this.reporter);
+        const ast = parser.parse();
+
+        return ast;
+    }
+
+    visitModBlockStmt(stmt: Stmt, filePath: string): VisitRes {
+        if (stmt.kind.type !== "mod_block") {
+            throw new Error();
+        }
+        const { ident, stmts: ast } = stmt.kind;
+        stmt.kind = {
+            type: "mod",
+            ident,
+            mod: { filePath, ast },
+        };
+        visitStmts(ast, this, filePath);
+        return "stop";
+    }
+
+    visitModFileStmt(stmt: Stmt, filePath: string): VisitRes {
+        if (stmt.kind.type !== "mod_file") {
+            throw new Error();
+        }
+        const { ident, filePath: modFilePath } = stmt.kind;
+        const ast = this.parseFile(
+            path.join(path.dirname(filePath), modFilePath),
+        );
+        stmt.kind = {
+            type: "mod",
+            ident,
+            mod: { filePath, ast },
+        };
+        visitStmts(ast, this, filePath);
+        return "stop";
+    }
+}
+
+async function lastPosInTextFile(filePath: string): Promise<Pos> {
+    const text = await Deno.readTextFile(filePath);
+
+    let index = 0;
+    let line = 1;
+    let col = 1;
+
+    while (index < text.length) {
+        if (text[index] == "\n") {
+            line += 1;
+            col = 1;
+        } else {
+            col += 1;
+        }
+        index += 1;
+    }
+
+    return { index, line, col };
+}

compiler/info.ts

@@ -11,6 +11,12 @@ export class Reporter {
     private reports: Report[] = [];
     private errorSet = false;
 
+    public constructor(private filePath: string) {}
+
+    public setFilePath(filePath: string) {
+        this.filePath = filePath;
+    }
+
     public reportError(report: Omit<Report, "type">) {
         this.reports.push({ ...report, type: "error" });
         this.printReport({ ...report, type: "error" });
@@ -20,7 +26,7 @@ export class Reporter {
     private printReport({ reporter, type, pos, msg }: Report) {
         console.error(
             `${reporter} ${type}: ${msg}${
-                pos ? ` at ${pos.line}:${pos.col}` : ""
+                pos ? `\n  at ${this.filePath}:${pos.line}:${pos.col}` : ""
             }`,
         );
     }

compiler/lexer.ts

@@ -27,6 +27,12 @@ export class Lexer {
                 this.step();
             }
             const keywords = [
+                "false",
+                "true",
+                "null",
+                "int",
+                "bool",
+                "string",
                 "break",
                 "return",
                 "let",
@@ -35,19 +41,13 @@ export class Lexer {
                 "if",
                 "else",
                 "struct",
-                "import",
                 "or",
                 "and",
                 "not",
                 "while",
                 "for",
                 "in",
-                "false",
-                "true",
-                "null",
-                "int",
-                "bool",
-                "string",
+                "mod",
             ];
             if (keywords.includes(value)) {
                 return this.token(value, pos);

compiler/parser.ts

@@ -37,24 +37,68 @@ export class Parser {
     private parseStmts(): Stmt[] {
         const stmts: Stmt[] = [];
         while (!this.done()) {
-            if (this.test("fn")) {
-                stmts.push(this.parseFn());
+            stmts.push(this.parseModStmt());
+        }
+        return stmts;
+    }
+
+    private parseModStmt(): Stmt {
+        if (this.test("mod")) {
+            return (this.parseMod());
+        } else if (this.test("fn")) {
+            return (this.parseFn());
         } else if (
             this.test("let") || this.test("return") || this.test("break")
         ) {
-                stmts.push(this.parseSingleLineBlockStmt());
+            const expr = this.parseSingleLineBlockStmt();
             this.eatSemicolon();
+            return expr;
         } else if (
             ["{", "if", "loop", "while", "for"].some((tt) => this.test(tt))
         ) {
             const expr = this.parseMultiLineBlockExpr();
-                stmts.push(this.stmt({ type: "expr", expr }, expr.pos));
+            return (this.stmt({ type: "expr", expr }, expr.pos));
         } else {
-                stmts.push(this.parseAssign());
+            const expr = this.parseAssign();
             this.eatSemicolon();
+            return expr;
         }
     }
-        return stmts;
+
+    private parseMod(): Stmt {
+        const pos = this.pos();
+        this.step();
+        if (!this.test("ident")) {
+            this.report("expected 'ident'");
+            return this.stmt({ type: "error" }, pos);
+        }
+        const ident = this.current().identValue!;
+        this.step();
+        if (this.test("string")) {
+            const filePath = this.current().stringValue!;
+            this.step();
+            this.eatSemicolon();
+            return this.stmt({ type: "mod_file", ident, filePath }, pos);
+        }
+
+        if (!this.test("{")) {
+            this.report("expected '{' or 'string'");
+            return this.stmt({ type: "error" }, pos);
+        }
+        this.step();
+
+        const stmts: Stmt[] = [];
+        while (!this.done() && !this.test("}")) {
+            stmts.push(this.parseModStmt());
+        }
+
+        if (!this.test("}")) {
+            this.report("expected '}'");
+            return this.stmt({ type: "error" }, pos);
+        }
+        this.step();
+
+        return this.stmt({ type: "mod_block", ident, stmts }, pos);
     }
 
     private parseMultiLineBlockExpr(): Expr {
@@ -110,7 +154,7 @@ export class Parser {
     private parseBlock(): Expr {
         const pos = this.pos();
         this.step();
-        let stmts: Stmt[] = [];
+        const stmts: Stmt[] = [];
         while (!this.done()) {
             if (this.test("}")) {
                 this.step();

compiler/resolver.ts

@@ -10,24 +10,45 @@ import {
 } from "./ast_visitor.ts";
 import { printStackTrace, Reporter } from "./info.ts";
 import {
+    EntryModSyms,
     FnSyms,
-    GlobalSyms,
     LeafSyms,
-    StaticSyms,
+    ModSyms,
     Syms,
 } from "./resolver_syms.ts";
 import { Pos } from "./token.ts";
 
 export class Resolver implements AstVisitor<[Syms]> {
-    private root = new GlobalSyms();
-
     public constructor(private reporter: Reporter) {
     }
 
     public resolve(stmts: Stmt[]): VisitRes {
-        const scopeSyms = new StaticSyms(this.root);
-        this.scoutFnStmts(stmts, scopeSyms);
-        visitStmts(stmts, this, scopeSyms);
+        const syms = new EntryModSyms();
+        this.scoutFnStmts(stmts, syms);
+        visitStmts(stmts, this, syms);
+        return "stop";
+    }
+
+    visitModStmt(stmt: Stmt, syms: Syms): VisitRes {
+        if (stmt.kind.type !== "mod") {
+            throw new Error("expected let statement");
+        }
+        const modSyms = new ModSyms(syms);
+        const { mod, ident } = stmt.kind;
+        this.scoutFnStmts(mod.ast, modSyms);
+        visitStmts(mod.ast, this, modSyms);
+
+        if (syms.definedLocally(ident)) {
+            this.reportAlreadyDefined(ident, stmt.pos, syms);
+            return;
+        }
+        syms.define(ident, {
+            type: "mod",
+            ident,
+            pos: stmt.pos,
+            syms: modSyms,
+        });
+
         return "stop";
     }
 
@@ -123,6 +144,43 @@ export class Resolver implements AstVisitor<[Syms]> {
         return "stop";
     }
 
+    visitPathExpr(expr: Expr, syms: Syms): VisitRes {
+        if (expr.kind.type !== "path") {
+            throw new Error("expected ident");
+        }
+        visitExpr(expr.kind.subject, this, syms);
+        if (expr.kind.subject.kind.type !== "sym") {
+            throw new Error("this error is not handled properly");
+        }
+        const subjectSym = expr.kind.subject.kind.sym;
+        if (subjectSym.type !== "mod") {
+            this.reporter.reportError({
+                reporter: "Resolver",
+                msg: `path expression are not implemented for '${subjectSym.type}' symbols`,
+                pos: expr.pos,
+            });
+            printStackTrace();
+            return "stop";
+        }
+        const getRes = subjectSym.syms.get(expr.kind.ident);
+        if (!getRes.ok) {
+            this.reportUseOfUndefined(
+                expr.kind.ident,
+                expr.pos,
+                subjectSym.syms,
+            );
+            return "stop";
+        }
+
+        expr.kind = {
+            type: "sym",
+            ident: expr.kind.ident,
+            sym: getRes.sym,
+        };
+
+        return "stop";
+    }
+
     visitBlockExpr(expr: Expr, syms: Syms): VisitRes {
         if (expr.kind.type !== "block") {
             throw new Error();

compiler/resolver_syms.ts

@@ -1,14 +1,16 @@
-import { Sym } from "./ast.ts";
+import type { Sym } from "./ast.ts";
 
 export type SymMap = { [ident: string]: Sym };
 
+type GetRes = { ok: true; sym: Sym } | { ok: false };
+
 export interface Syms {
     define(ident: string, sym: Sym): void;
     definedLocally(ident: string): boolean;
-    get(ident: string): { ok: true; sym: Sym } | { ok: false };
+    get(ident: string): GetRes;
 }
 
-export class GlobalSyms implements Syms {
+export class EntryModSyms implements Syms {
     private syms: SymMap = {};
 
     public constructor() {}
@@ -28,7 +30,7 @@ export class GlobalSyms implements Syms {
         return ident in this.syms;
     }
 
-    public get(ident: string): { ok: true; sym: Sym } | { ok: false } {
+    public get(ident: string): GetRes {
         if (ident in this.syms) {
             return { ok: true, sym: this.syms[ident] };
         }
@@ -36,10 +38,16 @@ export class GlobalSyms implements Syms {
     }
 }
 
-export class StaticSyms implements Syms {
+export class ModSyms implements Syms {
     private syms: SymMap = {};
 
-    public constructor(private parent: GlobalSyms) {}
+    public constructor(private parent: Syms) {
+        this.syms["super"] = {
+            type: "mod",
+            ident: "super",
+            syms: this.parent,
+        };
+    }
 
     public define(ident: string, sym: Sym) {
         if (sym.type === "let") {
@@ -56,11 +64,11 @@ export class StaticSyms implements Syms {
         return ident in this.syms;
     }
 
-    public get(ident: string): { ok: true; sym: Sym } | { ok: false } {
+    public get(ident: string): GetRes {
         if (ident in this.syms) {
             return { ok: true, sym: this.syms[ident] };
         }
-        return this.parent.get(ident);
+        return { ok: false };
     }
 }
 
@@ -85,7 +93,7 @@ export class FnSyms implements Syms {
         return ident in this.syms;
     }
 
-    public get(ident: string): { ok: true; sym: Sym } | { ok: false } {
+    public get(ident: string): GetRes {
         if (ident in this.syms) {
             return { ok: true, sym: this.syms[ident] };
         }
@@ -106,7 +114,7 @@ export class LeafSyms implements Syms {
         return ident in this.syms;
     }
 
-    public get(ident: string): { ok: true; sym: Sym } | { ok: false } {
+    public get(ident: string): GetRes {
         if (ident in this.syms) {
             return { ok: true, sym: this.syms[ident] };
         }

tests/import_modules_entry.slg

@@ -0,0 +1,19 @@
+
+
+fn exit(status_code: int) #[builtin(Exit)] {}
+
+fn print(msg: string) #[builtin(Print)] {}
+fn println(msg: string) { print(msg + "\n") }
+
+mod inner "import_modules_inner.slg";
+
+fn main() {
+    println("test function from module");
+    let res = inner::inner_fn(32);
+    if res != 64 {
+        println("failed");
+        exit(1);
+    }
+    println("all tests ran successfully");
+    exit(0);
+}

tests/import_modules_inner.slg

@@ -0,0 +1,5 @@
+
+fn inner_fn(a: int) -> int {
+    a + 32
+}
+

tests/modules.slg

@@ -0,0 +1,25 @@
+
+fn exit(status_code: int) #[builtin(Exit)] {}
+
+fn print(msg: string) #[builtin(Print)] {}
+fn println(msg: string) { print(msg + "\n") }
+
+mod my_module {
+    
+    fn inner_fn(a: int) -> int {
+        a + 32
+    }
+
+}
+
+fn main() {
+    println("test function from module");
+    let res = my_module::inner_fn(32);
+    if res != 64 {
+        println("failed");
+        exit(1);
+    }
+    println("all tests ran successfully");
+    exit(0);
+}
+