Implement basic parser and lexer.

This commit is contained in:
Danila Fedorin 2018-07-24 17:29:25 -07:00
parent 380b12d02c
commit bffa9847d2
4 changed files with 158 additions and 1 deletions

View File

@ -8,6 +8,10 @@ targets:
chalk:
main: src/chalk.cr
dependencies:
lex:
git: https://dev.danilafe.com/Chip-8-Wizardry/lex.git
crystal: 0.25.1
license: MIT

28
src/chalk/builder.cr Normal file
View File

@ -0,0 +1,28 @@
require "./lexer.cr"
require "./parser.cr"
module Chalk
def self.type(type): Parser(Token)
return TypeParser.new(type).as(Parser(Token))
end
def self.transform(parser : Parser(T), &transform : T -> R) forall T, R
return TransformParser.new(parser, &transform).as(Parser(R))
end
def self.optional(parser : Parser(T)): Parser(T?) forall T
return OptionalParser.new(parser).as(Parser(T?))
end
def self.either(*args : Parser(T)): Parser(T) forall T
return EitherParser.new(args.to_a).as(Parser(T))
end
def self.many(parser : Parser(T)): Parser(Array(T)) forall T
return ManyParser.new(parser).as(Parser(Array(T)))
end
def self.then(first : Parser(T), second : Parser(R)) forall T, R
return NextParser.new(first, second).as(Parser(Array(T | R)))
end
end

View File

@ -28,6 +28,14 @@ module Chalk
KwReturn
end
class Token
def initialize(@string : String, @type : TokenType)
end
getter string : String
getter type : TokenType
end
class Lexer
def initialize
@lexer = Lex::Lexer.new
@ -67,7 +75,7 @@ module Chalk
.select { |t| t[0] != " " }
.map do |tuple|
string, id = tuple
{ string, TokenType.new(id) }
Token.new(string, TokenType.new(id))
end
end
end

117
src/chalk/parser.cr Normal file
View File

@ -0,0 +1,117 @@
require "./builder.cr"
module Chalk
abstract class Parser(T)
abstract def parse?(tokens : Array(Token),
index : Int64): Tuple(T, Int64)?
def parse(tokens : Array(Token),
index : Int64): Tuple(T, Int64)
return parse?(tokens, index).not_nil!
end
def transform(&transform : T -> R) forall R
return TransformParser.new(self, &transform).as(Parser(R))
end
def then(other : Parser(R)): Parser(Array(T | R)) forall R
return NextParser
.new(self, other)
.as(Parser(Array(T | R)))
end
end
class TypeParser < Parser(Token)
def initialize(@type : TokenType)
end
def parse?(tokens, index)
return nil unless index < tokens.size
return nil unless tokens[index].type == @type
return { tokens[index], index + 1}
end
end
class TransformParser(T, R) < Parser(R)
def initialize(@parser : Parser(T), &@block : T -> R)
end
def parse?(tokens, index)
if parsed = @parser.parse?(tokens, index)
return { @block.call(parsed[0]), parsed[1] }
end
return nil
end
end
class OptionalParser(T) < Parser(T?)
def initialize(@parser : Parser(T))
end
def parse?(tokens, index)
if parsed = @parser.parse?(tokens, index)
return { parsed[0], parsed[1] }
end
return { nil, index }
end
end
class EitherParser(T) < Parser(T)
def initialize(@parsers : Array(Parser(T)))
end
def parse?(tokens, index)
@parsers.each do |parser|
if parsed = parser.parse?(tokens, index)
return parsed
end
end
return nil
end
end
class ManyParser(T) < Parser(Array(T))
def initialize(@parser : Parser(T))
end
def parse?(tokens, index)
many = [] of T
while parsed = @parser.parse?(tokens, index)
item, index = parsed
many << item
end
return { many, index }
end
end
class NextParser(T, R) < Parser(Array(T | R))
def initialize(@first : Parser(T), @second : Parser(R))
end
def parse?(tokens, index)
first = @first.parse?(tokens, index)
return nil unless first
first_value, index = first
second = @second.parse?(tokens, index)
return nil unless second
second_value, index = second
array = Array(T | R).new
array << first_value << second_value
return { array, index }
end
end
class PlaceholderParser(T) < Parser(T)
property parser : Parser(T)?
def initialize
@parser = nil
end
def parse?(tokens, index)
@parser.try &.parse(tokens, index)
end
end
end