Implement basic parser and lexer.

shard.yml

@@ -8,6 +8,10 @@ targets:
   chalk:
     main: src/chalk.cr
 
+dependencies:
+  lex:
+    git: https://dev.danilafe.com/Chip-8-Wizardry/lex.git
+
 crystal: 0.25.1
 
 license: MIT

src/chalk/builder.cr

@@ -0,0 +1,28 @@
+require "./lexer.cr"
+require "./parser.cr"
+
+module Chalk
+    def self.type(type): Parser(Token)
+        return TypeParser.new(type).as(Parser(Token))
+    end
+
+    def self.transform(parser : Parser(T), &transform : T -> R) forall T, R
+        return TransformParser.new(parser, &transform).as(Parser(R))
+    end
+
+    def self.optional(parser : Parser(T)): Parser(T?) forall T
+        return OptionalParser.new(parser).as(Parser(T?))
+    end
+
+    def self.either(*args : Parser(T)): Parser(T) forall T
+        return EitherParser.new(args.to_a).as(Parser(T))
+    end
+
+    def self.many(parser : Parser(T)): Parser(Array(T)) forall T
+        return ManyParser.new(parser).as(Parser(Array(T)))
+    end
+
+    def self.then(first : Parser(T), second : Parser(R)) forall T, R
+        return NextParser.new(first, second).as(Parser(Array(T | R)))
+    end
+end

src/chalk/lexer.cr

@@ -28,6 +28,14 @@ module Chalk
         KwReturn
     end
 
+    class Token
+        def initialize(@string : String, @type : TokenType)
+        end
+
+        getter string : String
+        getter type : TokenType
+    end
+
     class Lexer
         def initialize
             @lexer = Lex::Lexer.new
@@ -67,7 +75,7 @@ module Chalk
                 .select { |t| t[0] != " " }
                 .map do |tuple|
                     string, id = tuple
-                    { string, TokenType.new(id) }
+                    Token.new(string, TokenType.new(id))
                 end
         end
     end

src/chalk/parser.cr

@@ -0,0 +1,117 @@
+require "./builder.cr"
+
+module Chalk
+
+    abstract class Parser(T)
+        abstract def parse?(tokens : Array(Token),
+                           index : Int64): Tuple(T, Int64)?
+        def parse(tokens : Array(Token),
+                  index : Int64): Tuple(T, Int64)
+            return parse?(tokens, index).not_nil!
+        end
+
+        def transform(&transform : T -> R) forall R
+            return TransformParser.new(self, &transform).as(Parser(R))
+        end
+
+        def then(other : Parser(R)): Parser(Array(T | R)) forall R
+            return NextParser
+                .new(self, other)
+                .as(Parser(Array(T | R)))
+        end
+    end
+
+    class TypeParser < Parser(Token)
+        def initialize(@type : TokenType)
+        end
+
+        def parse?(tokens, index)
+            return nil unless index < tokens.size
+            return nil unless tokens[index].type == @type
+            return { tokens[index], index + 1}
+        end
+    end
+
+    class TransformParser(T, R) < Parser(R)
+        def initialize(@parser : Parser(T), &@block : T -> R)
+        end
+
+        def parse?(tokens, index)
+            if parsed = @parser.parse?(tokens, index)
+                return { @block.call(parsed[0]), parsed[1] }
+            end
+            return nil
+        end
+    end
+
+    class OptionalParser(T) < Parser(T?)
+        def initialize(@parser : Parser(T))
+        end
+
+        def parse?(tokens, index)
+            if parsed = @parser.parse?(tokens, index)
+                return { parsed[0], parsed[1] }
+            end
+            return { nil, index }
+        end
+    end
+
+    class EitherParser(T) < Parser(T)
+        def initialize(@parsers : Array(Parser(T)))
+        end
+
+        def parse?(tokens, index)
+            @parsers.each do |parser|
+                if parsed = parser.parse?(tokens, index)
+                    return parsed
+                end
+            end
+            return nil
+        end
+    end
+
+    class ManyParser(T) < Parser(Array(T))
+        def initialize(@parser : Parser(T))
+        end
+
+        def parse?(tokens, index)
+            many = [] of T
+            while parsed = @parser.parse?(tokens, index)
+                item, index = parsed
+                many << item
+            end
+            return { many, index }
+        end
+    end
+
+    class NextParser(T, R) < Parser(Array(T | R))
+        def initialize(@first : Parser(T), @second : Parser(R))
+        end
+
+        def parse?(tokens, index)
+            first = @first.parse?(tokens, index)
+            return nil unless first
+            first_value, index = first
+            
+            second = @second.parse?(tokens, index)
+            return nil unless second
+            second_value, index = second
+
+            array = Array(T | R).new
+            array << first_value << second_value
+            return { array, index }
+        end
+    end
+
+    class PlaceholderParser(T) < Parser(T)
+        property parser : Parser(T)?
+
+        def initialize
+            @parser = nil
+        end
+
+        def parse?(tokens, index)
+            @parser.try &.parse(tokens, index)
+        end
+    end
+end