sfja.skp/web-stack-project
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

scriptlang: strict value parser done

Browse Source
This commit is contained in:
SimonFJ20 2023-01-12 13:33:19 +01:00
parent f632ad6805
commit 1417c1cee5
8 changed files with 306 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,16 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "lldb",
"request": "launch",
"name": "Debug Browser",
"program": "${workspaceFolder}/builddir/web-browser",
"args": [],
"cwd": "${workspaceFolder}"
}
]
}

12
browser/main.cpp
View File

@ -3,6 +3,7 @@
#include "SDL_rect.h"
#include "SDL_render.h"
#include "SDL_video.h"
#include "scriptlang/parser.hpp"
#include "utils/all.hpp"
#include <SDL.h>
#include <fmt/core.h>
@ -68,6 +69,17 @@ private:
auto main() -> int
{
const auto* text = "{name: \"test\", value: [true, false, 123, \"bruh\"], "
"int: 123, float: 3.14}";
auto ast = scriptlang::Parser(text).parse_expression(true);
if (!ast)
fmt::print("parser error at {}:{}: {}\n\t\n",
ast.unwrap_error().span.from.line,
ast.unwrap_error().span.from.column, ast.unwrap_error().message);
else
fmt::print("ast = {}\n", ast.unwrap()->to_string());
// test
fmt::print("browser: hello world!\n");
auto gui = GUI::create().unwrap();

17
scriptlang/error.hpp
View File

@ -1,17 +0,0 @@
#pragma once
namespace scriptlang {
enum class Errors {
NotImplemented,
LexerNoTokenYet,
LexerStringNotTerminated,
LexerUnexpectedCharacer,
LexerMultilineCommentNotTerminated,
NoLexerOutput,
ParserExhausted,
ParserMalformed,
ParserUnexpected,
};
}

50
scriptlang/lexer.cpp
View File

@ -1,14 +1,16 @@
#include "lexer.hpp"
#include "error.hpp"
#include <cctype>
#include <string>
#include <string_view>
namespace scriptlang {
auto Lexer::make_token() noexcept -> Result<Token, Errors>
auto Lexer::make_token() noexcept -> Result<Token, Error>
{
if (done())
return token(Tokens::Eof, index, current_location());
if (std::isspace(current()) != 0)
return skip_whitespace();
if (std::isdigit(current()) != 0)
return make_number();
if (std::isalpha(current()) != 0 or current() == '_')
@ -18,7 +20,14 @@ auto Lexer::make_token() noexcept -> Result<Token, Errors>
return make_static();
}
auto Lexer::make_number() noexcept -> Result<Token, Errors>
auto Lexer::skip_whitespace() noexcept -> Result<Token, Error>
{
while (!done() and std::isspace(current()) != 0)
step();
return make_token();
}
auto Lexer::make_number() noexcept -> Result<Token, Error>
{
auto begin = index;
auto span_from = current_location();
@ -33,7 +42,7 @@ auto Lexer::make_number() noexcept -> Result<Token, Errors>
return token(Tokens::Int, begin, span_from);
}
auto Lexer::make_id() noexcept -> Result<Token, Errors>
auto Lexer::make_id() noexcept -> Result<Token, Error>
{
auto begin = index;
auto span_from = current_location();
@ -78,7 +87,7 @@ auto Lexer::id_or_keyword_type(std::string_view substring) noexcept -> Tokens
return Tokens::Id;
}
auto Lexer::make_string() noexcept -> Result<Token, Errors>
auto Lexer::make_string() noexcept -> Result<Token, Error>
{
auto begin = index;
auto span_from = current_location();
@ -89,12 +98,15 @@ auto Lexer::make_string() noexcept -> Result<Token, Errors>
step();
}
if (current() != '"')
return Errors::LexerStringNotTerminated;
return Error {
{ span_from, { line, column } },
"unterminated string",
};
step();
return token(Tokens::String, begin, span_from);
}
auto Lexer::make_static() noexcept -> Result<Token, Errors>
auto Lexer::make_static() noexcept -> Result<Token, Error>
{
auto begin = index;
auto span_from = current_location();
@ -105,7 +117,7 @@ auto Lexer::make_static() noexcept -> Result<Token, Errors>
}
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
auto Lexer::static_token_type() noexcept -> Result<Tokens, Errors>
auto Lexer::static_token_type() noexcept -> Result<Tokens, Error>
{
using TT = Tokens;
auto stepped = [&](Tokens v) {
@ -113,13 +125,13 @@ auto Lexer::static_token_type() noexcept -> Result<Tokens, Errors>
return v;
};
if (current() == ')')
return stepped(TT::LParen);
if (current() == '(')
return stepped(TT::LParen);
if (current() == ')')
return stepped(TT::RParen);
if (current() == '}')
return stepped(TT::LBrace);
if (current() == '{')
return stepped(TT::LBrace);
if (current() == '}')
return stepped(TT::RBrace);
if (current() == '[')
return stepped(TT::LBracket);
@ -207,10 +219,13 @@ auto Lexer::static_token_type() noexcept -> Result<Tokens, Errors>
return stepped(TT::GreaterEqual);
return TT::Greater;
}
return Errors::LexerUnexpectedCharacer;
return Error {
{ { line, column - 1 }, { line, column } },
"unexpected character",
};
}
auto Lexer::skip_multiline_comment() noexcept -> Result<Tokens, Errors>
auto Lexer::skip_multiline_comment() noexcept -> Result<Tokens, Error>
{
step();
auto last = current();
@ -218,12 +233,15 @@ auto Lexer::skip_multiline_comment() noexcept -> Result<Tokens, Errors>
while (!done() and last != '*' and current() != '/')
step();
if (last != '*' or current() != '/')
return Errors::LexerMultilineCommentNotTerminated;
return Error {
{ { line, column - 1 }, { line, column } },
"unterminated multiline comment",
};
step();
return Tokens::MultilineComment;
}
auto Lexer::skip_singleline_comment() noexcept -> Result<Tokens, Errors>
auto Lexer::skip_singleline_comment() noexcept -> Result<Tokens, Error>
{
step();
while (!done() and current() != '\n')

40
scriptlang/lexer.hpp
View File

@ -1,6 +1,6 @@
#include "error.hpp"
#include "utils/all.hpp"
#include <optional>
#include <string>
#include <string_view>
namespace scriptlang {
@ -89,6 +89,17 @@ struct Token {
Tokens type;
size_t index, length;
Span span;
[[nodiscard]] static auto token_span(
const Token& from, const Token& to) noexcept -> Span
{
return { from.span.from, to.span.to };
}
};
struct Error {
Span span;
std::string message;
};
class Lexer {
@ -96,24 +107,25 @@ public:
Lexer(std::string_view text)
: text { text }
{ }
auto next() noexcept -> Result<Token, Errors> { return make_token(); }
auto peek() noexcept -> Result<Token, Errors>
auto next() noexcept -> Result<Token, Error> { return make_token(); }
auto peek() noexcept -> Result<Token, Error>
{
if (last_token)
return Result<Token, Errors>::create_ok(*last_token);
return Errors::LexerNoTokenYet;
if (!last_token)
return Error { { { 0, 0 }, { 0, 0 } }, "no token yet" };
return Result<Token, Error>::create_ok(*last_token);
}
private:
auto make_token() noexcept -> Result<Token, Errors>;
auto make_number() noexcept -> Result<Token, Errors>;
auto make_id() noexcept -> Result<Token, Errors>;
auto make_token() noexcept -> Result<Token, Error>;
auto skip_whitespace() noexcept -> Result<Token, Error>;
auto make_number() noexcept -> Result<Token, Error>;
auto make_id() noexcept -> Result<Token, Error>;
auto id_or_keyword_type(std::string_view substring) noexcept -> Tokens;
auto make_string() noexcept -> Result<Token, Errors>;
auto make_static() noexcept -> Result<Token, Errors>;
auto static_token_type() noexcept -> Result<Tokens, Errors>;
auto skip_multiline_comment() noexcept -> Result<Tokens, Errors>;
auto skip_singleline_comment() noexcept -> Result<Tokens, Errors>;
auto make_string() noexcept -> Result<Token, Error>;
auto make_static() noexcept -> Result<Token, Error>;
auto static_token_type() noexcept -> Result<Tokens, Error>;
auto skip_multiline_comment() noexcept -> Result<Tokens, Error>;
auto skip_singleline_comment() noexcept -> Result<Tokens, Error>;
[[nodiscard]] auto constexpr inline current_location() const noexcept
-> Location

178
scriptlang/parser.cpp
View File

@ -1,85 +1,177 @@
#include "parser.hpp"
#include "error.hpp"
#include "utils/result.hpp"
#include <cstdlib>
#include <memory>
#include <string>
#include <vector>
namespace scriptlang {
auto Parser::parse_expression(bool strictly_values) noexcept
-> Result<std::unique_ptr<Expression>, Errors>
-> Result<std::unique_ptr<Expression>, Error>
{
if (strictly_values)
return parse_struct(true);
return Errors::NotImplemented;
return parse_array(true);
return Error { { { 0, 0 }, { 0, 0 } }, "not implemented" };
}
auto Parser::parse_array(bool strictly_values) noexcept
-> Result<std::unique_ptr<Expression>, Error>
{
auto values = std::vector<std::unique_ptr<Expression>> {};
auto first_bracket = *lexer.peek();
if (first_bracket.type == Tokens::LBracket) {
(void)lexer.next();
auto value = parse_expression(strictly_values);
if (!value)
return value;
values.emplace_back(std::move(*value));
while (lexer.peek()->type == Tokens::Comma) {
(void)lexer.next();
if (lexer.peek()->type == Tokens::LBracket)
break;
auto value2 = parse_expression(strictly_values);
values.emplace_back(std::move(*value2));
}
auto last_bracket = *lexer.peek();
if (last_bracket.type != Tokens::RBracket)
return Error {
last_bracket.span,
"unterminated array",
};
(void)lexer.next().unwrap();
return {
std::make_unique<Array>(
Token::token_span(first_bracket, last_bracket),
std::move(values)),
};
}
return parse_struct(strictly_values);
}
// NOLINTNEXTLINE(readability-function-cognitive-complexity)
auto Parser::parse_struct(bool strictly_values) noexcept
-> Result<std::unique_ptr<Expression>, Errors>
-> Result<std::unique_ptr<Expression>, Error>
{
auto values = std::map<std::string, std::unique_ptr<Expression>> {};
auto first_brace = *lexer.peek();
if (first_brace.type == Tokens::LBrace) {
auto name = *lexer.next();
if (name.type != Tokens::Eof && name.type != Tokens::LBrace) {
if (name.type != Tokens::LBrace) {
if (name.type != Tokens::Id)
return Errors::ParserUnexpected;
return Error {
name.span,
"unexpected token, expected Id or String",
};
if (lexer.next()->type != Tokens::Colon)
return Errors::ParserUnexpected;
return Error {
lexer.peek()->span,
"unexpected token, expected ':'",
};
if (auto result = lexer.next(); !result)
return { std::move(result.unwrap_error()) };
auto value = parse_expression(strictly_values);
if (!value)
return value.transform<std::unique_ptr<Expression>>();
if (values.find(token_text(name)) != values.end())
return Error {
name.span,
"multiple definitions of struct field",
};
values.insert_or_assign(token_text(name), std::move(*value));
while (lexer.peek()->type == Tokens::Comma) {
auto name2 = *lexer.next();
if (name2.type == Tokens::RBrace)
break;
if (name2.type != Tokens::Id)
return Error {
name2.span,
"unexpected token, expected Id",
};
if (lexer.next()->type != Tokens::Colon)
return Error {
lexer.peek()->span,
"unexpected token, expected ':'",
};
(void)lexer.next();
auto value2 = parse_expression(strictly_values);
if (!value2)
return value2.transform<std::unique_ptr<Expression>>();
if (values.find(token_text(name2)) != values.end())
return Error {
name2.span,
"multiple definitions of struct field",
};
values.insert_or_assign(token_text(name2), std::move(*value2));
}
}
auto last_brace = *lexer.peek();
if (last_brace.type != Tokens::RBrace)
return Errors::ParserMalformed;
return Error {
last_brace.span,
fmt::format("unterminated struct, expected '}}', got {}",
last_brace.type),
};
(void)lexer.next().unwrap();
return {
std::make_unique<Struct>(
token_span(first_brace, last_brace), std::move(values)),
Token::token_span(first_brace, last_brace), std::move(values)),
};
}
return parse_atom();
}
auto Parser::parse_atom() noexcept
-> Result<std::unique_ptr<Expression>, Errors>
auto Parser::parse_atom() noexcept -> Result<std::unique_ptr<Expression>, Error>
{
auto token = *lexer.peek();
switch (token.type) {
case Tokens::Id:
return {
std::make_unique<Id>(token_span(token, token),
token_text(token.index, token.length)),
};
case Tokens::Int:
return {
std::make_unique<Int>(token_span(token, token),
std::atol(token_text(token.index, token.length).c_str())),
};
case Tokens::Float:
return {
std::make_unique<Float>(token_span(token, token),
std::atof(token_text(token.index, token.length).c_str())),
};
case Tokens::False:
return {
std::make_unique<Bool>(token_span(token, token), false),
};
case Tokens::True:
return {
std::make_unique<Bool>(token_span(token, token), true),
};
case Tokens::String:
return {
std::make_unique<String>(token_span(token, token),
*parse_string_value(token_text(token.index, token.length))),
};
case Tokens::Id: {
auto node = std::make_unique<Id>(Token::token_span(token, token),
token_text(token.index, token.length));
(void)lexer.next().unwrap();
return { std::move(node) };
}
case Tokens::Int: {
auto node = std::make_unique<Int>(Token::token_span(token, token),
std::atol(token_text(token.index, token.length).c_str()));
(void)lexer.next().unwrap();
return { std::move(node) };
}
case Tokens::Float: {
auto node = std::make_unique<Float>(Token::token_span(token, token),
std::atof(token_text(token.index, token.length).c_str()));
(void)lexer.next().unwrap();
return { std::move(node) };
}
case Tokens::False: {
auto node = std::make_unique<Bool>(
Token::token_span(token, token), false);
(void)lexer.next().unwrap();
return { std::move(node) };
}
case Tokens::True: {
auto node
= std::make_unique<Bool>(Token::token_span(token, token), true);
(void)lexer.next().unwrap();
return { std::move(node) };
}
case Tokens::String: {
auto node
= std::make_unique<String>(Token::token_span(token, token),
*parse_string_value(token_text(token.index, token.length)));
(void)lexer.next().unwrap();
return { std::move(node) };
}
default:
return Errors::ParserExhausted;
return Error { token.span, "unexpected token, expected value" };
}
}
[[nodiscard]] auto Parser::parse_string_value(std::string_view literal) noexcept
-> Result<std::string, Errors>
-> Result<std::string, utils::result::Error<std::string>>
{
if (literal.size() < 2)
return Errors::ParserMalformed;
return utils::result::Error<std::string> { "malformed string" };
auto value = std::string {};
auto escaped = false;
for (const auto c : literal.substr(1, literal.size() - 2)) {

88
scriptlang/parser.hpp
View File

@ -1,9 +1,9 @@
#pragma once
#include "error.hpp"
#include "lexer.hpp"
#include "utils/all.hpp"
#include "utils/result.hpp"
#include <fmt/core.h>
#include <map>
#include <memory>
#include <string>
@ -22,6 +22,7 @@ enum class Expressions {
Call,
Operator,
Array,
Struct,
Id,
Int,
@ -41,10 +42,41 @@ public:
[[nodiscard]] virtual auto expression_type() const noexcept -> Expressions
= 0;
[[nodiscard]] virtual auto span() const noexcept -> Span = 0;
[[nodiscard]] virtual auto to_string() const noexcept -> std::string = 0;
private:
};
class Array final : public Expression {
public:
Array(Span span, std::vector<std::unique_ptr<Expression>> values)
: m_span { span }
, m_values { std::move(values) }
{ }
[[nodiscard]] auto expression_type() const noexcept -> Expressions override
{
return Expressions::Array;
}
[[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
[[nodiscard]] auto values() const noexcept -> auto& { return m_values; }
[[nodiscard]] auto to_string() const noexcept -> std::string override
{
auto values_strings = std::string {};
auto first = true;
for (const auto& value : m_values) {
if (!first)
values_strings.append(", ");
first = false;
values_strings.append(value->to_string());
}
return fmt::format("Array {{ [ {} ] }}", values_strings);
};
private:
Span m_span;
std::vector<std::unique_ptr<Expression>> m_values;
};
class Struct final : public Expression {
public:
Struct(Span span, std::map<std::string, std::unique_ptr<Expression>> values)
@ -53,10 +85,22 @@ public:
{ }
[[nodiscard]] auto expression_type() const noexcept -> Expressions override
{
return Expressions::Id;
return Expressions::Struct;
}
[[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
[[nodiscard]] auto values() const noexcept -> auto& { return m_values; }
[[nodiscard]] auto to_string() const noexcept -> std::string override
{
auto values_strings = std::string {};
auto first = true;
for (const auto& [name, value] : m_values) {
if (!first)
values_strings.append(", ");
first = false;
values_strings.append(value->to_string());
}
return fmt::format("Struct {{ [ {} ] }}", values_strings);
};
private:
Span m_span;
@ -75,6 +119,10 @@ public:
}
[[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
[[nodiscard]] auto value() const noexcept { return m_value; }
[[nodiscard]] auto to_string() const noexcept -> std::string override
{
return fmt::format("Id {{ {} }}", m_value);
}
private:
Span m_span;
@ -93,6 +141,10 @@ public:
}
[[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
[[nodiscard]] auto value() const noexcept { return m_value; }
[[nodiscard]] auto to_string() const noexcept -> std::string override
{
return fmt::format("Int {{ {} }}", m_value);
}
private:
Span m_span;
@ -111,6 +163,10 @@ public:
}
[[nodiscard]] auto value() const noexcept { return m_value; }
[[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
[[nodiscard]] auto to_string() const noexcept -> std::string override
{
return fmt::format("Float {{ {} }}", m_value);
}
private:
Span m_span;
@ -129,6 +185,10 @@ public:
}
[[nodiscard]] auto value() const noexcept { return m_value; }
[[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
[[nodiscard]] auto to_string() const noexcept -> std::string override
{
return fmt::format("Bool {{ {} }}", m_value);
}
private:
Span m_span;
@ -150,6 +210,10 @@ public:
return m_value;
}
[[nodiscard]] auto span() const noexcept -> Span override { return m_span; }
[[nodiscard]] auto to_string() const noexcept -> std::string override
{
return fmt::format("String {{ \"{}\" }}", m_value);
}
private:
Span m_span;
@ -161,24 +225,30 @@ public:
Parser(std::string_view text)
: text { text }
, lexer(text)
{ }
{
[[maybe_unused]] auto _ = lexer.next();
}
auto parse_expression(bool strictly_values) noexcept
-> Result<std::unique_ptr<Expression>, Errors>;
-> Result<std::unique_ptr<Expression>, Error>;
auto parse_array(bool strictly_values) noexcept
-> Result<std::unique_ptr<Expression>, Error>;
auto parse_struct(bool strictly_values) noexcept
-> Result<std::unique_ptr<Expression>, Errors>;
auto parse_atom() noexcept -> Result<std::unique_ptr<Expression>, Errors>;
-> Result<std::unique_ptr<Expression>, Error>;
auto parse_atom() noexcept -> Result<std::unique_ptr<Expression>, Error>;
private:
[[nodiscard]] static auto parse_string_value(
std::string_view literal) noexcept -> Result<std::string, Errors>;
std::string_view literal) noexcept
-> Result<std::string, utils::result::Error<std::string>>;
[[nodiscard]] auto token_text(size_t index, size_t length) const noexcept
-> std::string
{
return std::string { text.substr(index, length) };
}
[[nodiscard]] static auto token_span(Token from, Token to) noexcept -> Span
[[nodiscard]] auto token_text(const Token& token) const noexcept
-> std::string
{
return { from.span.from, to.span.to };
return std::string { text.substr(token.index, token.length) };
}
std::string_view text;

4
utils/result.hpp
View File

@ -23,6 +23,10 @@ struct StatesValueTypes {
struct Error { };
};
template <typename ErrorV> struct Error {
ErrorV value;
};
template <typename InnerValue> struct Extracter {
using Value = void;
using Error = void;