scriptlang: strict value parser done

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug Browser",
+            "program": "${workspaceFolder}/builddir/web-browser",
+            "args": [],
+            "cwd": "${workspaceFolder}"
+        }
+    ]
+}

browser/main.cpp

@@ -3,6 +3,7 @@
 #include "SDL_rect.h"
 #include "SDL_render.h"
 #include "SDL_video.h"
+#include "scriptlang/parser.hpp"
 #include "utils/all.hpp"
 #include <SDL.h>
 #include <fmt/core.h>
@@ -68,6 +69,17 @@ private:
 
 auto main() -> int
 {
+
+    const auto* text = "{name: \"test\", value: [true, false, 123, \"bruh\"], "
+                       "int: 123, float: 3.14}";
+    auto ast = scriptlang::Parser(text).parse_expression(true);
+    if (!ast)
+        fmt::print("parser error at {}:{}: {}\n\t\n",
+            ast.unwrap_error().span.from.line,
+            ast.unwrap_error().span.from.column, ast.unwrap_error().message);
+    else
+        fmt::print("ast = {}\n", ast.unwrap()->to_string());
+
     // test
     fmt::print("browser: hello world!\n");
     auto gui = GUI::create().unwrap();

scriptlang/error.hpp

@@ -1,17 +0,0 @@
-#pragma once
-
-namespace scriptlang {
-
-enum class Errors {
-    NotImplemented,
-    LexerNoTokenYet,
-    LexerStringNotTerminated,
-    LexerUnexpectedCharacer,
-    LexerMultilineCommentNotTerminated,
-    NoLexerOutput,
-    ParserExhausted,
-    ParserMalformed,
-    ParserUnexpected,
-};
-
-}

scriptlang/lexer.cpp

@@ -1,14 +1,16 @@
 #include "lexer.hpp"
-#include "error.hpp"
 #include <cctype>
+#include <string>
 #include <string_view>
 
 namespace scriptlang {
 
-auto Lexer::make_token() noexcept -> Result<Token, Errors>
+auto Lexer::make_token() noexcept -> Result<Token, Error>
 {
     if (done())
         return token(Tokens::Eof, index, current_location());
+    if (std::isspace(current()) != 0)
+        return skip_whitespace();
     if (std::isdigit(current()) != 0)
         return make_number();
     if (std::isalpha(current()) != 0 or current() == '_')
@@ -18,7 +20,14 @@ auto Lexer::make_token() noexcept -> Result<Token, Errors>
     return make_static();
 }
 
-auto Lexer::make_number() noexcept -> Result<Token, Errors>
+auto Lexer::skip_whitespace() noexcept -> Result<Token, Error>
+{
+    while (!done() and std::isspace(current()) != 0)
+        step();
+    return make_token();
+}
+
+auto Lexer::make_number() noexcept -> Result<Token, Error>
 {
     auto begin = index;
     auto span_from = current_location();
@@ -33,7 +42,7 @@ auto Lexer::make_number() noexcept -> Result<Token, Errors>
     return token(Tokens::Int, begin, span_from);
 }
 
-auto Lexer::make_id() noexcept -> Result<Token, Errors>
+auto Lexer::make_id() noexcept -> Result<Token, Error>
 {
     auto begin = index;
     auto span_from = current_location();
@@ -78,7 +87,7 @@ auto Lexer::id_or_keyword_type(std::string_view substring) noexcept -> Tokens
     return Tokens::Id;
 }
 
-auto Lexer::make_string() noexcept -> Result<Token, Errors>
+auto Lexer::make_string() noexcept -> Result<Token, Error>
 {
     auto begin = index;
     auto span_from = current_location();
@@ -89,12 +98,15 @@ auto Lexer::make_string() noexcept -> Result<Token, Errors>
         step();
     }
     if (current() != '"')
-        return Errors::LexerStringNotTerminated;
+        return Error {
+            { span_from, { line, column } },
+            "unterminated string",
+        };
     step();
     return token(Tokens::String, begin, span_from);
 }
 
-auto Lexer::make_static() noexcept -> Result<Token, Errors>
+auto Lexer::make_static() noexcept -> Result<Token, Error>
 {
     auto begin = index;
     auto span_from = current_location();
@@ -105,7 +117,7 @@ auto Lexer::make_static() noexcept -> Result<Token, Errors>
 }
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-auto Lexer::static_token_type() noexcept -> Result<Tokens, Errors>
+auto Lexer::static_token_type() noexcept -> Result<Tokens, Error>
 {
     using TT = Tokens;
     auto stepped = [&](Tokens v) {
@@ -113,13 +125,13 @@ auto Lexer::static_token_type() noexcept -> Result<Tokens, Errors>
         return v;
     };
 
-    if (current() == ')')
-        return stepped(TT::LParen);
     if (current() == '(')
+        return stepped(TT::LParen);
+    if (current() == ')')
         return stepped(TT::RParen);
-    if (current() == '}')
-        return stepped(TT::LBrace);
     if (current() == '{')
+        return stepped(TT::LBrace);
+    if (current() == '}')
         return stepped(TT::RBrace);
     if (current() == '[')
         return stepped(TT::LBracket);
@@ -207,10 +219,13 @@ auto Lexer::static_token_type() noexcept -> Result<Tokens, Errors>
             return stepped(TT::GreaterEqual);
         return TT::Greater;
     }
-    return Errors::LexerUnexpectedCharacer;
+    return Error {
+        { { line, column - 1 }, { line, column } },
+        "unexpected character",
+    };
 }
 
-auto Lexer::skip_multiline_comment() noexcept -> Result<Tokens, Errors>
+auto Lexer::skip_multiline_comment() noexcept -> Result<Tokens, Error>
 {
     step();
     auto last = current();
@@ -218,12 +233,15 @@ auto Lexer::skip_multiline_comment() noexcept -> Result<Tokens, Errors>
     while (!done() and last != '*' and current() != '/')
         step();
     if (last != '*' or current() != '/')
-        return Errors::LexerMultilineCommentNotTerminated;
+        return Error {
+            { { line, column - 1 }, { line, column } },
+            "unterminated multiline comment",
+        };
     step();
     return Tokens::MultilineComment;
 }
 
-auto Lexer::skip_singleline_comment() noexcept -> Result<Tokens, Errors>
+auto Lexer::skip_singleline_comment() noexcept -> Result<Tokens, Error>
 {
     step();
     while (!done() and current() != '\n')

scriptlang/lexer.hpp

@@ -1,6 +1,6 @@
-#include "error.hpp"
 #include "utils/all.hpp"
 #include <optional>
+#include <string>
 #include <string_view>
 
 namespace scriptlang {
@@ -89,6 +89,17 @@ struct Token {
     Tokens type;
     size_t index, length;
     Span span;
+
+    [[nodiscard]] static auto token_span(
+        const Token& from, const Token& to) noexcept -> Span
+    {
+        return { from.span.from, to.span.to };
+    }
+};
+
+struct Error {
+    Span span;
+    std::string message;
 };
 
 class Lexer {
@@ -96,24 +107,25 @@ public:
     Lexer(std::string_view text)
         : text { text }
     { }
-    auto next() noexcept -> Result<Token, Errors> { return make_token(); }
+    auto next() noexcept -> Result<Token, Error> { return make_token(); }
-    auto peek() noexcept -> Result<Token, Errors>
+    auto peek() noexcept -> Result<Token, Error>
     {
-        if (last_token)
+        if (!last_token)
-            return Result<Token, Errors>::create_ok(*last_token);
+            return Error { { { 0, 0 }, { 0, 0 } }, "no token yet" };
-        return Errors::LexerNoTokenYet;
+        return Result<Token, Error>::create_ok(*last_token);
     }
 
 private:
-    auto make_token() noexcept -> Result<Token, Errors>;
+    auto make_token() noexcept -> Result<Token, Error>;
-    auto make_number() noexcept -> Result<Token, Errors>;
+    auto skip_whitespace() noexcept -> Result<Token, Error>;
-    auto make_id() noexcept -> Result<Token, Errors>;
+    auto make_number() noexcept -> Result<Token, Error>;
+    auto make_id() noexcept -> Result<Token, Error>;
     auto id_or_keyword_type(std::string_view substring) noexcept -> Tokens;
-    auto make_string() noexcept -> Result<Token, Errors>;
+    auto make_string() noexcept -> Result<Token, Error>;
-    auto make_static() noexcept -> Result<Token, Errors>;
+    auto make_static() noexcept -> Result<Token, Error>;
-    auto static_token_type() noexcept -> Result<Tokens, Errors>;
+    auto static_token_type() noexcept -> Result<Tokens, Error>;
-    auto skip_multiline_comment() noexcept -> Result<Tokens, Errors>;
+    auto skip_multiline_comment() noexcept -> Result<Tokens, Error>;
-    auto skip_singleline_comment() noexcept -> Result<Tokens, Errors>;
+    auto skip_singleline_comment() noexcept -> Result<Tokens, Error>;
 
     [[nodiscard]] auto constexpr inline current_location() const noexcept
         -> Location

scriptlang/parser.cpp

@@ -1,85 +1,177 @@
 #include "parser.hpp"
-#include "error.hpp"
+#include "utils/result.hpp"
 #include <cstdlib>
 #include <memory>
+#include <string>
+#include <vector>
 
 namespace scriptlang {
 
 auto Parser::parse_expression(bool strictly_values) noexcept
-    -> Result<std::unique_ptr<Expression>, Errors>
+    -> Result<std::unique_ptr<Expression>, Error>
 {
     if (strictly_values)
-        return parse_struct(true);
+        return parse_array(true);
-    return Errors::NotImplemented;
+    return Error { { { 0, 0 }, { 0, 0 } }, "not implemented" };
 }
 
+auto Parser::parse_array(bool strictly_values) noexcept
+    -> Result<std::unique_ptr<Expression>, Error>
+{
+    auto values = std::vector<std::unique_ptr<Expression>> {};
+    auto first_bracket = *lexer.peek();
+    if (first_bracket.type == Tokens::LBracket) {
+        (void)lexer.next();
+        auto value = parse_expression(strictly_values);
+        if (!value)
+            return value;
+        values.emplace_back(std::move(*value));
+        while (lexer.peek()->type == Tokens::Comma) {
+            (void)lexer.next();
+            if (lexer.peek()->type == Tokens::LBracket)
+                break;
+            auto value2 = parse_expression(strictly_values);
+            values.emplace_back(std::move(*value2));
+        }
+        auto last_bracket = *lexer.peek();
+        if (last_bracket.type != Tokens::RBracket)
+            return Error {
+                last_bracket.span,
+                "unterminated array",
+            };
+        (void)lexer.next().unwrap();
+        return {
+            std::make_unique<Array>(
+                Token::token_span(first_bracket, last_bracket),
+                std::move(values)),
+        };
+    }
+    return parse_struct(strictly_values);
+}
+
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
 auto Parser::parse_struct(bool strictly_values) noexcept
-    -> Result<std::unique_ptr<Expression>, Errors>
+    -> Result<std::unique_ptr<Expression>, Error>
 {
     auto values = std::map<std::string, std::unique_ptr<Expression>> {};
     auto first_brace = *lexer.peek();
     if (first_brace.type == Tokens::LBrace) {
         auto name = *lexer.next();
-        if (name.type != Tokens::Eof && name.type != Tokens::LBrace) {
+        if (name.type != Tokens::LBrace) {
             if (name.type != Tokens::Id)
-                return Errors::ParserUnexpected;
+                return Error {
+                    name.span,
+                    "unexpected token, expected Id or String",
+                };
             if (lexer.next()->type != Tokens::Colon)
-                return Errors::ParserUnexpected;
+                return Error {
+                    lexer.peek()->span,
+                    "unexpected token, expected ':'",
+                };
+            if (auto result = lexer.next(); !result)
+                return { std::move(result.unwrap_error()) };
+            auto value = parse_expression(strictly_values);
+            if (!value)
+                return value.transform<std::unique_ptr<Expression>>();
+            if (values.find(token_text(name)) != values.end())
+                return Error {
+                    name.span,
+                    "multiple definitions of struct field",
+                };
+            values.insert_or_assign(token_text(name), std::move(*value));
+            while (lexer.peek()->type == Tokens::Comma) {
+                auto name2 = *lexer.next();
+                if (name2.type == Tokens::RBrace)
+                    break;
+                if (name2.type != Tokens::Id)
+                    return Error {
+                        name2.span,
+                        "unexpected token, expected Id",
+                    };
+                if (lexer.next()->type != Tokens::Colon)
+                    return Error {
+                        lexer.peek()->span,
+                        "unexpected token, expected ':'",
+                    };
+                (void)lexer.next();
+                auto value2 = parse_expression(strictly_values);
+                if (!value2)
+                    return value2.transform<std::unique_ptr<Expression>>();
+                if (values.find(token_text(name2)) != values.end())
+                    return Error {
+                        name2.span,
+                        "multiple definitions of struct field",
+                    };
+                values.insert_or_assign(token_text(name2), std::move(*value2));
+            }
         }
         auto last_brace = *lexer.peek();
         if (last_brace.type != Tokens::RBrace)
-            return Errors::ParserMalformed;
+            return Error {
+                last_brace.span,
+                fmt::format("unterminated struct, expected '}}', got {}",
+                    last_brace.type),
+            };
+        (void)lexer.next().unwrap();
         return {
             std::make_unique<Struct>(
-                token_span(first_brace, last_brace), std::move(values)),
+                Token::token_span(first_brace, last_brace), std::move(values)),
         };
     }
     return parse_atom();
 }
 
-auto Parser::parse_atom() noexcept
+auto Parser::parse_atom() noexcept -> Result<std::unique_ptr<Expression>, Error>
-    -> Result<std::unique_ptr<Expression>, Errors>
 {
     auto token = *lexer.peek();
     switch (token.type) {
-        case Tokens::Id:
+        case Tokens::Id: {
-            return {
+            auto node = std::make_unique<Id>(Token::token_span(token, token),
-                std::make_unique<Id>(token_span(token, token),
+                token_text(token.index, token.length));
-                    token_text(token.index, token.length)),
+            (void)lexer.next().unwrap();
-            };
+            return { std::move(node) };
-        case Tokens::Int:
+        }
-            return {
+        case Tokens::Int: {
-                std::make_unique<Int>(token_span(token, token),
+            auto node = std::make_unique<Int>(Token::token_span(token, token),
-                    std::atol(token_text(token.index, token.length).c_str())),
+                std::atol(token_text(token.index, token.length).c_str()));
-            };
+            (void)lexer.next().unwrap();
-        case Tokens::Float:
+            return { std::move(node) };
-            return {
+        }
-                std::make_unique<Float>(token_span(token, token),
+        case Tokens::Float: {
-                    std::atof(token_text(token.index, token.length).c_str())),
+            auto node = std::make_unique<Float>(Token::token_span(token, token),
-            };
+                std::atof(token_text(token.index, token.length).c_str()));
-        case Tokens::False:
+            (void)lexer.next().unwrap();
-            return {
+            return { std::move(node) };
-                std::make_unique<Bool>(token_span(token, token), false),
+        }
-            };
+        case Tokens::False: {
-        case Tokens::True:
+            auto node = std::make_unique<Bool>(
-            return {
+                Token::token_span(token, token), false);
-                std::make_unique<Bool>(token_span(token, token), true),
+            (void)lexer.next().unwrap();
-            };
+            return { std::move(node) };
-        case Tokens::String:
+        }
-            return {
+        case Tokens::True: {
-                std::make_unique<String>(token_span(token, token),
+            auto node
-                    *parse_string_value(token_text(token.index, token.length))),
+                = std::make_unique<Bool>(Token::token_span(token, token), true);
-            };
+            (void)lexer.next().unwrap();
+            return { std::move(node) };
+        }
+        case Tokens::String: {
+            auto node
+                = std::make_unique<String>(Token::token_span(token, token),
+                    *parse_string_value(token_text(token.index, token.length)));
+            (void)lexer.next().unwrap();
+            return { std::move(node) };
+        }
         default:
-            return Errors::ParserExhausted;
+            return Error { token.span, "unexpected token, expected value" };
     }
 }
 
 [[nodiscard]] auto Parser::parse_string_value(std::string_view literal) noexcept
-    -> Result<std::string, Errors>
+    -> Result<std::string, utils::result::Error<std::string>>
 {
     if (literal.size() < 2)
-        return Errors::ParserMalformed;
+        return utils::result::Error<std::string> { "malformed string" };
     auto value = std::string {};
     auto escaped = false;
     for (const auto c : literal.substr(1, literal.size() - 2)) {

scriptlang/parser.hpp

@@ -1,9 +1,9 @@
 #pragma once
 
-#include "error.hpp"
 #include "lexer.hpp"
 #include "utils/all.hpp"
 #include "utils/result.hpp"
+#include <fmt/core.h>
 #include <map>
 #include <memory>
 #include <string>
@@ -22,6 +22,7 @@ enum class Expressions {
     Call,
     Operator,
 
+    Array,
     Struct,
     Id,
     Int,
@@ -41,10 +42,41 @@ public:
     [[nodiscard]] virtual auto expression_type() const noexcept -> Expressions
         = 0;
     [[nodiscard]] virtual auto span() const noexcept -> Span = 0;
+    [[nodiscard]] virtual auto to_string() const noexcept -> std::string = 0;
 
 private:
 };
 
+class Array final : public Expression {
+public:
+    Array(Span span, std::vector<std::unique_ptr<Expression>> values)
+        : m_span { span }
+        , m_values { std::move(values) }
+    { }
+    [[nodiscard]] auto expression_type() const noexcept -> Expressions override
+    {
+        return Expressions::Array;
+    }
+    [[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
+    [[nodiscard]] auto values() const noexcept -> auto& { return m_values; }
+    [[nodiscard]] auto to_string() const noexcept -> std::string override
+    {
+        auto values_strings = std::string {};
+        auto first = true;
+        for (const auto& value : m_values) {
+            if (!first)
+                values_strings.append(", ");
+            first = false;
+            values_strings.append(value->to_string());
+        }
+        return fmt::format("Array {{ [ {} ] }}", values_strings);
+    };
+
+private:
+    Span m_span;
+    std::vector<std::unique_ptr<Expression>> m_values;
+};
+
 class Struct final : public Expression {
 public:
     Struct(Span span, std::map<std::string, std::unique_ptr<Expression>> values)
@@ -53,10 +85,22 @@ public:
     { }
     [[nodiscard]] auto expression_type() const noexcept -> Expressions override
     {
-        return Expressions::Id;
+        return Expressions::Struct;
     }
     [[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
     [[nodiscard]] auto values() const noexcept -> auto& { return m_values; }
+    [[nodiscard]] auto to_string() const noexcept -> std::string override
+    {
+        auto values_strings = std::string {};
+        auto first = true;
+        for (const auto& [name, value] : m_values) {
+            if (!first)
+                values_strings.append(", ");
+            first = false;
+            values_strings.append(value->to_string());
+        }
+        return fmt::format("Struct {{ [ {} ] }}", values_strings);
+    };
 
 private:
     Span m_span;
@@ -75,6 +119,10 @@ public:
     }
     [[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
     [[nodiscard]] auto value() const noexcept { return m_value; }
+    [[nodiscard]] auto to_string() const noexcept -> std::string override
+    {
+        return fmt::format("Id {{ {} }}", m_value);
+    }
 
 private:
     Span m_span;
@@ -93,6 +141,10 @@ public:
     }
     [[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
     [[nodiscard]] auto value() const noexcept { return m_value; }
+    [[nodiscard]] auto to_string() const noexcept -> std::string override
+    {
+        return fmt::format("Int {{ {} }}", m_value);
+    }
 
 private:
     Span m_span;
@@ -111,6 +163,10 @@ public:
     }
     [[nodiscard]] auto value() const noexcept { return m_value; }
     [[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
+    [[nodiscard]] auto to_string() const noexcept -> std::string override
+    {
+        return fmt::format("Float {{ {} }}", m_value);
+    }
 
 private:
     Span m_span;
@@ -129,6 +185,10 @@ public:
     }
     [[nodiscard]] auto value() const noexcept { return m_value; }
     [[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
+    [[nodiscard]] auto to_string() const noexcept -> std::string override
+    {
+        return fmt::format("Bool {{ {} }}", m_value);
+    }
 
 private:
     Span m_span;
@@ -150,6 +210,10 @@ public:
         return m_value;
     }
     [[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
+    [[nodiscard]] auto to_string() const noexcept -> std::string override
+    {
+        return fmt::format("String {{ \"{}\" }}", m_value);
+    }
 
 private:
     Span m_span;
@@ -161,24 +225,30 @@ public:
     Parser(std::string_view text)
         : text { text }
         , lexer(text)
-    { }
+    {
+        [[maybe_unused]] auto _ = lexer.next();
+    }
     auto parse_expression(bool strictly_values) noexcept
-        -> Result<std::unique_ptr<Expression>, Errors>;
+        -> Result<std::unique_ptr<Expression>, Error>;
+    auto parse_array(bool strictly_values) noexcept
+        -> Result<std::unique_ptr<Expression>, Error>;
     auto parse_struct(bool strictly_values) noexcept
-        -> Result<std::unique_ptr<Expression>, Errors>;
+        -> Result<std::unique_ptr<Expression>, Error>;
-    auto parse_atom() noexcept -> Result<std::unique_ptr<Expression>, Errors>;
+    auto parse_atom() noexcept -> Result<std::unique_ptr<Expression>, Error>;
 
 private:
     [[nodiscard]] static auto parse_string_value(
-        std::string_view literal) noexcept -> Result<std::string, Errors>;
+        std::string_view literal) noexcept
+        -> Result<std::string, utils::result::Error<std::string>>;
     [[nodiscard]] auto token_text(size_t index, size_t length) const noexcept
         -> std::string
     {
         return std::string { text.substr(index, length) };
     }
-    [[nodiscard]] static auto token_span(Token from, Token to) noexcept -> Span
+    [[nodiscard]] auto token_text(const Token& token) const noexcept
+        -> std::string
     {
-        return { from.span.from, to.span.to };
+        return std::string { text.substr(token.index, token.length) };
     }
 
     std::string_view text;

utils/result.hpp

@@ -23,6 +23,10 @@ struct StatesValueTypes {
     struct Error { };
 };
 
+template <typename ErrorV> struct Error {
+    ErrorV value;
+};
+
 template <typename InnerValue> struct Extracter {
     using Value = void;
     using Error = void;