From bffa9847d2707c24a90723501b12b979202cdd56 Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Tue, 24 Jul 2018 17:29:25 -0700 Subject: [PATCH] Implement basic parser and lexer. --- shard.yml | 4 ++ src/chalk/builder.cr | 28 +++++++++++ src/chalk/lexer.cr | 10 +++- src/chalk/parser.cr | 117 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 src/chalk/builder.cr create mode 100644 src/chalk/parser.cr diff --git a/shard.yml b/shard.yml index 060d2b8..f6d1c83 100644 --- a/shard.yml +++ b/shard.yml @@ -8,6 +8,10 @@ targets: chalk: main: src/chalk.cr +dependencies: + lex: + git: https://dev.danilafe.com/Chip-8-Wizardry/lex.git + crystal: 0.25.1 license: MIT diff --git a/src/chalk/builder.cr b/src/chalk/builder.cr new file mode 100644 index 0000000..4f645e5 --- /dev/null +++ b/src/chalk/builder.cr @@ -0,0 +1,28 @@ +require "./lexer.cr" +require "./parser.cr" + +module Chalk + def self.type(type): Parser(Token) + return TypeParser.new(type).as(Parser(Token)) + end + + def self.transform(parser : Parser(T), &transform : T -> R) forall T, R + return TransformParser.new(parser, &transform).as(Parser(R)) + end + + def self.optional(parser : Parser(T)): Parser(T?) forall T + return OptionalParser.new(parser).as(Parser(T?)) + end + + def self.either(*args : Parser(T)): Parser(T) forall T + return EitherParser.new(args.to_a).as(Parser(T)) + end + + def self.many(parser : Parser(T)): Parser(Array(T)) forall T + return ManyParser.new(parser).as(Parser(Array(T))) + end + + def self.then(first : Parser(T), second : Parser(R)) forall T, R + return NextParser.new(first, second).as(Parser(Array(T | R))) + end +end diff --git a/src/chalk/lexer.cr b/src/chalk/lexer.cr index 4e77075..57a0042 100644 --- a/src/chalk/lexer.cr +++ b/src/chalk/lexer.cr @@ -28,6 +28,14 @@ module Chalk KwReturn end + class Token + def initialize(@string : String, @type : TokenType) + end + + getter string : String + getter type : TokenType + end + class Lexer def initialize @lexer = Lex::Lexer.new @@ -67,7 +75,7 @@ module Chalk .select { |t| t[0] != " " } .map do |tuple| string, id = tuple - { string, TokenType.new(id) } + Token.new(string, TokenType.new(id)) end end end diff --git a/src/chalk/parser.cr b/src/chalk/parser.cr new file mode 100644 index 0000000..3e9797f --- /dev/null +++ b/src/chalk/parser.cr @@ -0,0 +1,117 @@ +require "./builder.cr" + +module Chalk + + abstract class Parser(T) + abstract def parse?(tokens : Array(Token), + index : Int64): Tuple(T, Int64)? + def parse(tokens : Array(Token), + index : Int64): Tuple(T, Int64) + return parse?(tokens, index).not_nil! + end + + def transform(&transform : T -> R) forall R + return TransformParser.new(self, &transform).as(Parser(R)) + end + + def then(other : Parser(R)): Parser(Array(T | R)) forall R + return NextParser + .new(self, other) + .as(Parser(Array(T | R))) + end + end + + class TypeParser < Parser(Token) + def initialize(@type : TokenType) + end + + def parse?(tokens, index) + return nil unless index < tokens.size + return nil unless tokens[index].type == @type + return { tokens[index], index + 1} + end + end + + class TransformParser(T, R) < Parser(R) + def initialize(@parser : Parser(T), &@block : T -> R) + end + + def parse?(tokens, index) + if parsed = @parser.parse?(tokens, index) + return { @block.call(parsed[0]), parsed[1] } + end + return nil + end + end + + class OptionalParser(T) < Parser(T?) + def initialize(@parser : Parser(T)) + end + + def parse?(tokens, index) + if parsed = @parser.parse?(tokens, index) + return { parsed[0], parsed[1] } + end + return { nil, index } + end + end + + class EitherParser(T) < Parser(T) + def initialize(@parsers : Array(Parser(T))) + end + + def parse?(tokens, index) + @parsers.each do |parser| + if parsed = parser.parse?(tokens, index) + return parsed + end + end + return nil + end + end + + class ManyParser(T) < Parser(Array(T)) + def initialize(@parser : Parser(T)) + end + + def parse?(tokens, index) + many = [] of T + while parsed = @parser.parse?(tokens, index) + item, index = parsed + many << item + end + return { many, index } + end + end + + class NextParser(T, R) < Parser(Array(T | R)) + def initialize(@first : Parser(T), @second : Parser(R)) + end + + def parse?(tokens, index) + first = @first.parse?(tokens, index) + return nil unless first + first_value, index = first + + second = @second.parse?(tokens, index) + return nil unless second + second_value, index = second + + array = Array(T | R).new + array << first_value << second_value + return { array, index } + end + end + + class PlaceholderParser(T) < Parser(T) + property parser : Parser(T)? + + def initialize + @parser = nil + end + + def parse?(tokens, index) + @parser.try &.parse(tokens, index) + end + end +end