Chip-8-Wizardry/chalk
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

Clean up parsing code.

Browse Source
This commit is contained in:
Danila Fedorin 2018-07-25 13:51:31 -07:00
parent c6319e7920
commit fa00bda8ab
5 changed files with 324 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
src/chalk.cr
View File

@ -2,170 +2,11 @@ require "./chalk/*"
module Chalk
lexer = Lexer.new
parser = Parser.new
tokens = lexer.lex(File.read("test.txt"))
def self.create_op(atom, op)
pl = PlaceholderParser(Tree).new
recurse = atom.then(op).then(pl).transform do |arr|
arr = arr.flatten
TreeOp.new(
arr[1].as(Token).type,
arr[0].as(Tree),
arr[2].as(Tree)).as(Tree)
end
pl.parser = either(recurse, atom)
return pl
trees = parser.parse?(tokens)
trees.try do |trees|
trees.each &.accept(PrintVisitor.new)
end
def self.create_ops(source, ops)
ops.reduce(source) do |previous, current|
create_op(previous, current)
end
end
def self.create_type
either(type(TokenType::KwU0), type(TokenType::KwU8), type(TokenType::KwU12))
end
def self.create_lit
dec_parser = type(TokenType::LitDec).transform &.string.to_i64
hex_parser = type(TokenType::LitHex).transform &.string.lchop("0x").to_i64(16)
bin_parser = type(TokenType::LitBin).transform &.string.lchop("0b").to_i64(2)
lit_parser = either(dec_parser, hex_parser, bin_parser).transform { |it| TreeLit.new(it).as(Tree) }
return lit_parser
end
def self.create_call(expr)
call = type(TokenType::Id).then(char '(').then(delimited(expr, char ',')).then(char ')').transform do |arr|
arr = arr.flatten
name = arr[0].as(Token).string
params = arr[2..arr.size - 2].map &.as(Tree)
TreeCall.new(name, params).as(Tree)
end
return call
end
def self.create_block(statement)
block = char('{').then(many(statement)).then(char '}').transform do |arr|
arr = arr.flatten
params = arr[1..arr.size - 2].map &.as(Tree)
TreeBlock.new(params).as(Tree)
end
return block
end
def self.create_if(expr, block)
iff = type(TokenType::KwIf).then(char '(').then(expr).then(char ')').then(block)
.then(optional(type(TokenType::KwElse).then(block)))
.transform do |arr|
arr = arr.flatten
cond = arr[2].as(Tree)
code = arr[4].as(Tree)
otherwise = arr.size == 7 ? arr[6].as(Tree) : nil
TreeIf.new(cond, code, otherwise).as(Tree)
end
return iff
end
def self.create_while(expr, block)
whilee = type(TokenType::KwWhile).then(char '(').then(expr).then(char ')').then(block).transform do |arr|
arr = arr.flatten
cond = arr[2].as(Tree)
code = arr[4].as(Tree)
TreeWhile.new(cond, code).as(Tree)
end
return whilee
end
def self.create_return(expr)
returnn = type(TokenType::KwReturn).then(expr).then(char ';').transform do |arr|
arr = arr.flatten
value = arr[1].as(Tree)
TreeReturn.new(value).as(Tree)
end
return returnn
end
def self.create_func(block, type)
func = type(TokenType::KwFun).then(type(TokenType::Id))
.then(char '(').then(delimited(type(TokenType::Id), char ',')).then(char ')')
.then(char ':').then(type)
.then(block).transform do |arr|
arr = arr.flatten
name = arr[1].as(Token).string
params = arr[3..arr.size - 5].map &.as(Token).string
code = arr[arr.size - 1].as(Tree)
type = arr[arr.size - 2].as(Token).type
TreeFunction.new(name, params, code).as(Tree)
end
return func
end
def self.create_var(expr)
var = type(TokenType::KwVar).then(type(TokenType::Id)).then(char '=').then(expr).then(char ';').transform do |arr|
arr = arr.flatten
name = arr[1].as(Token).string
exp = arr[arr.size - 2].as(Tree)
TreeVar.new(name, exp).as(Tree)
end
return var
end
def self.create_assign(expr)
assign = type(TokenType::Id).then(char '=').then(expr).then(char ';').transform do |arr|
arr = arr.flatten
name = arr[0].as(Token).string
exp = arr[arr.size - 2].as(Tree)
TreeAssign.new(name, exp).as(Tree)
end
return assign
end
def self.create_basic(expr)
basic = expr.then(char ';').transform do |arr|
arr.flatten[0].as(Tree)
end
return basic
end
def self.create_expression
expr_place = PlaceholderParser(Tree).new
literal = create_lit
id = type(TokenType::Id).transform { |it| TreeId.new(it.string).as(Tree) }
call = create_call(expr_place)
atom = either(literal, call, id)
ops = [ either(type(TokenType::OpMul), type(TokenType::OpDiv)),
either(type(TokenType::OpAdd), type(TokenType::OpSub)),
type(TokenType::OpXor),
type(TokenType::OpAnd),
type(TokenType::OpOr) ]
expr = create_ops(atom, ops)
expr_place.parser = expr
return expr
end
def self.create_block
expr = create_expression
statement_place = PlaceholderParser(Tree).new
block = create_block(statement_place)
iff = create_if(expr, block)
whilee = create_while(expr, block)
returnn = create_return(expr)
var = create_var(expr)
assign = create_assign(expr)
basic = create_basic(expr)
statement = either(basic, var, assign, block, iff, whilee, returnn)
statement_place.parser = statement
return block
end
func = create_func(create_block, create_type)
body = many(func)
final = body
final.parse(tokens, 0)[0].each &.accept(PrintVisitor.new)
end

50
src/chalk/builder.cr
View File

@ -1,36 +1,38 @@
require "./lexer.cr"
require "./parser.cr"
require "./parsers.cr"
module Chalk
def self.type(type): Parser(Token)
return TypeParser.new(type).as(Parser(Token))
end
module Builder
def type(type): BasicParser(Token)
return TypeParser.new(type).as(BasicParser(Token))
end
def self.char(type): Parser(Token)
return CharParser.new(type).as(Parser(Token))
end
def char(type): BasicParser(Token)
return CharParser.new(type).as(BasicParser(Token))
end
def self.transform(parser : Parser(T), &transform : T -> R) forall T, R
return TransformParser.new(parser, &transform).as(Parser(R))
end
def transform(parser : BasicParser(T), &transform : T -> R) forall T, R
return TransformParser.new(parser, &transform).as(BasicParser(R))
end
def self.optional(parser : Parser(T)): Parser(T?) forall T
return OptionalParser.new(parser).as(Parser(T?))
end
def optional(parser : BasicParser(T)): BasicParser(T?) forall T
return OptionalParser.new(parser).as(BasicParser(T?))
end
def self.either(*args : Parser(T)): Parser(T) forall T
return EitherParser.new(args.to_a).as(Parser(T))
end
def either(*args : BasicParser(T)): BasicParser(T) forall T
return EitherParser.new(args.to_a).as(BasicParser(T))
end
def self.many(parser : Parser(T)): Parser(Array(T)) forall T
return ManyParser.new(parser).as(Parser(Array(T)))
end
def many(parser : BasicParser(T)): BasicParser(Array(T)) forall T
return ManyParser.new(parser).as(BasicParser(Array(T)))
end
def self.delimited(parser : Parser(T), delimiter : Parser(R)): Parser(Array(T)) forall T, R
return DelimitedParser.new(parser, delimiter).as(Parser(Array(T)))
end
def delimited(parser : BasicParser(T), delimiter : BasicParser(R)): BasicParser(Array(T)) forall T, R
return DelimitedParser.new(parser, delimiter).as(BasicParser(Array(T)))
end
def self.then(first : Parser(T), second : Parser(R)) forall T, R
return NextParser.new(first, second).as(Parser(Array(T | R)))
def then(first : BasicParser(T), second : BasicParser(R)) forall T, R
return NextParser.new(first, second).as(BasicParser(Array(T | R)))
end
end
end

2
src/chalk/lexer.cr
View File

@ -72,7 +72,7 @@ module Chalk
def lex(string)
return @lexer.lex(string)
.select { |t| t[0] != " " }
.select { |t| !t[0][0].whitespace? }
.map do |tuple|
string, id = tuple
Token.new(string, TokenType.new(id))

260
src/chalk/parser.cr
View File

@ -1,151 +1,173 @@
require "./builder.cr"
module Chalk
class Parser
include Builder
abstract class Parser(T)
abstract def parse?(tokens : Array(Token),
index : Int64): Tuple(T, Int64)?
def parse(tokens : Array(Token),
index : Int64): Tuple(T, Int64)
return parse?(tokens, index).not_nil!
private def create_type
either(type(TokenType::KwU0), type(TokenType::KwU8), type(TokenType::KwU12))
end
def transform(&transform : T -> R) forall R
return TransformParser.new(self, &transform).as(Parser(R))
private def create_lit
dec_parser = type(TokenType::LitDec).transform &.string.to_i64
hex_parser = type(TokenType::LitHex).transform &.string.lchop("0x").to_i64(16)
bin_parser = type(TokenType::LitBin).transform &.string.lchop("0b").to_i64(2)
lit_parser = either(dec_parser, hex_parser, bin_parser).transform { |it| TreeLit.new(it).as(Tree) }
return lit_parser
end
def then(other : Parser(R)): Parser(Array(T | R)) forall R
return NextParser
.new(self, other)
.as(Parser(Array(T | R)))
end
end
class TypeParser < Parser(Token)
def initialize(@type : TokenType)
end
def parse?(tokens, index)
return nil unless index < tokens.size
return nil unless tokens[index].type == @type
return { tokens[index], index + 1}
end
end
class CharParser < Parser(Token)
def initialize(@char : Char)
end
def parse?(tokens, index)
return nil unless index < tokens.size
return nil unless (tokens[index].type == TokenType::Any) &&
tokens[index].string[0] == @char
return { tokens[index], index + 1}
end
end
class TransformParser(T, R) < Parser(R)
def initialize(@parser : Parser(T), &@block : T -> R)
end
def parse?(tokens, index)
if parsed = @parser.parse?(tokens, index)
return { @block.call(parsed[0]), parsed[1] }
private def create_op_expr(atom, op)
pl = PlaceholderParser(Tree).new
recurse = atom.then(op).then(pl).transform do |arr|
arr = arr.flatten
TreeOp.new(
arr[1].as(Token).type,
arr[0].as(Tree),
arr[2].as(Tree)).as(Tree)
end
return nil
end
end
class OptionalParser(T) < Parser(T?)
def initialize(@parser : Parser(T))
pl.parser = either(recurse, atom)
return pl
end
def parse?(tokens, index)
if parsed = @parser.parse?(tokens, index)
return { parsed[0], parsed[1] }
private def create_op_exprs(atom, ops)
ops.reduce(atom) do |previous, current|
create_op_expr(previous, current)
end
return { nil, index }
end
end
class EitherParser(T) < Parser(T)
def initialize(@parsers : Array(Parser(T)))
end
def parse?(tokens, index)
@parsers.each do |parser|
if parsed = parser.parse?(tokens, index)
return parsed
end
private def create_call(expr)
call = type(TokenType::Id).then(char '(').then(delimited(expr, char ',')).then(char ')').transform do |arr|
arr = arr.flatten
name = arr[0].as(Token).string
params = arr[2..arr.size - 2].map &.as(Tree)
TreeCall.new(name, params).as(Tree)
end
return nil
end
end
class ManyParser(T) < Parser(Array(T))
def initialize(@parser : Parser(T))
return call
end
def parse?(tokens, index)
many = [] of T
while parsed = @parser.parse?(tokens, index)
item, index = parsed
many << item
private def create_expr
expr_place = PlaceholderParser(Tree).new
literal = create_lit
id = type(TokenType::Id).transform { |it| TreeId.new(it.string).as(Tree) }
call = create_call(expr_place)
atom = either(literal, call, id)
ops = [ either(type(TokenType::OpMul), type(TokenType::OpDiv)),
either(type(TokenType::OpAdd), type(TokenType::OpSub)),
type(TokenType::OpXor),
type(TokenType::OpAnd),
type(TokenType::OpOr) ]
expr = create_op_exprs(atom, ops)
expr_place.parser = expr
return expr
end
private def create_var(expr)
var = type(TokenType::KwVar).then(type(TokenType::Id)).then(char '=').then(expr).then(char ';').transform do |arr|
arr = arr.flatten
name = arr[1].as(Token).string
exp = arr[arr.size - 2].as(Tree)
TreeVar.new(name, exp).as(Tree)
end
return { many, index }
end
end
class DelimitedParser(T, R) < Parser(Array(T))
def initialize(@parser : Parser(T), @delimiter : Parser(R))
return var
end
def parse?(tokens, index)
array = [] of T
first = @parser.parse?(tokens, index)
return { array, index } unless first
first_value, index = first
array << first_value
while delimiter = @delimiter.parse?(tokens, index)
_, new_index = delimiter
new = @parser.parse?(tokens, new_index)
break unless new
new_value, index = new
array << new_value
private def create_assign(expr)
assign = type(TokenType::Id).then(char '=').then(expr).then(char ';').transform do |arr|
arr = arr.flatten
name = arr[0].as(Token).string
exp = arr[arr.size - 2].as(Tree)
TreeAssign.new(name, exp).as(Tree)
end
return { array, index }
end
end
class NextParser(T, R) < Parser(Array(T | R))
def initialize(@first : Parser(T), @second : Parser(R))
return assign
end
def parse?(tokens, index)
first = @first.parse?(tokens, index)
return nil unless first
first_value, index = first
second = @second.parse?(tokens, index)
return nil unless second
second_value, index = second
array = Array(T | R).new
array << first_value << second_value
return { array, index }
private def create_basic(expr)
basic = expr.then(char ';').transform do |arr|
arr.flatten[0].as(Tree)
end
return basic
end
end
class PlaceholderParser(T) < Parser(T)
property parser : Parser(T)?
private def create_if(expr, block)
iff = type(TokenType::KwIf).then(char '(').then(expr).then(char ')').then(block)
.then(optional(type(TokenType::KwElse).then(block)))
.transform do |arr|
arr = arr.flatten
cond = arr[2].as(Tree)
code = arr[4].as(Tree)
otherwise = arr.size == 7 ? arr[6].as(Tree) : nil
TreeIf.new(cond, code, otherwise).as(Tree)
end
return iff
end
private def create_while(expr, block)
whilee = type(TokenType::KwWhile).then(char '(').then(expr).then(char ')').then(block).transform do |arr|
arr = arr.flatten
cond = arr[2].as(Tree)
code = arr[4].as(Tree)
TreeWhile.new(cond, code).as(Tree)
end
return whilee
end
private def create_return(expr)
returnn = type(TokenType::KwReturn).then(expr).then(char ';').transform do |arr|
arr = arr.flatten
value = arr[1].as(Tree)
TreeReturn.new(value).as(Tree)
end
return returnn
end
private def create_block(statement)
block = char('{').then(many(statement)).then(char '}').transform do |arr|
arr = arr.flatten
params = arr[1..arr.size - 2].map &.as(Tree)
TreeBlock.new(params).as(Tree)
end
return block
end
private def create_statement_block
statement_place = PlaceholderParser(Tree).new
expr = create_expr
block = create_block(statement_place)
iff = create_if(expr, block)
whilee = create_while(expr, block)
returnn = create_return(expr)
var = create_var(expr)
assign = create_assign(expr)
basic = create_basic(expr)
statement = either(basic, var, assign, block, iff, whilee, returnn)
statement_place.parser = statement
return {statement, block}
end
private def create_func(block, type)
func = type(TokenType::KwFun).then(type(TokenType::Id))
.then(char '(').then(delimited(type(TokenType::Id), char ',')).then(char ')')
.then(char ':').then(type)
.then(block).transform do |arr|
arr = arr.flatten
name = arr[1].as(Token).string
params = arr[3..arr.size - 5].map &.as(Token).string
code = arr[arr.size - 1].as(Tree)
type = arr[arr.size - 2].as(Token).type
TreeFunction.new(name, params, code).as(Tree)
end
return func
end
def initialize
@parser = nil
_, block = create_statement_block
@parser = many(create_func(block, create_type)).as(BasicParser(Array(Tree)))
end
def parse?(tokens, index)
@parser.try &.parse?(tokens, index)
def parse?(tokens)
return @parser.parse?(tokens, 0).try &.[0]
end
end
end

151
src/chalk/parsers.cr Normal file
View File

@ -0,0 +1,151 @@
require "./builder.cr"
module Chalk
abstract class BasicParser(T)
abstract def parse?(tokens : Array(Token),
index : Int64): Tuple(T, Int64)?
def parse(tokens : Array(Token),
index : Int64): Tuple(T, Int64)
return parse?(tokens, index).not_nil!
end
def transform(&transform : T -> R) forall R
return TransformParser.new(self, &transform).as(BasicParser(R))
end
def then(other : BasicParser(R)): BasicParser(Array(T | R)) forall R
return NextParser
.new(self, other)
.as(BasicParser(Array(T | R)))
end
end
class TypeParser < BasicParser(Token)
def initialize(@type : TokenType)
end
def parse?(tokens, index)
return nil unless index < tokens.size
return nil unless tokens[index].type == @type
return { tokens[index], index + 1}
end
end
class CharParser < BasicParser(Token)
def initialize(@char : Char)
end
def parse?(tokens, index)
return nil unless index < tokens.size
return nil unless (tokens[index].type == TokenType::Any) &&
tokens[index].string[0] == @char
return { tokens[index], index + 1}
end
end
class TransformParser(T, R) < BasicParser(R)
def initialize(@parser : BasicParser(T), &@block : T -> R)
end
def parse?(tokens, index)
if parsed = @parser.parse?(tokens, index)
return { @block.call(parsed[0]), parsed[1] }
end
return nil
end
end
class OptionalParser(T) < BasicParser(T?)
def initialize(@parser : BasicParser(T))
end
def parse?(tokens, index)
if parsed = @parser.parse?(tokens, index)
return { parsed[0], parsed[1] }
end
return { nil, index }
end
end
class EitherParser(T) < BasicParser(T)
def initialize(@parsers : Array(BasicParser(T)))
end
def parse?(tokens, index)
@parsers.each do |parser|
if parsed = parser.parse?(tokens, index)
return parsed
end
end
return nil
end
end
class ManyParser(T) < BasicParser(Array(T))
def initialize(@parser : BasicParser(T))
end
def parse?(tokens, index)
many = [] of T
while parsed = @parser.parse?(tokens, index)
item, index = parsed
many << item
end
return { many, index }
end
end
class DelimitedParser(T, R) < BasicParser(Array(T))
def initialize(@parser : BasicParser(T), @delimiter : BasicParser(R))
end
def parse?(tokens, index)
array = [] of T
first = @parser.parse?(tokens, index)
return { array, index } unless first
first_value, index = first
array << first_value
while delimiter = @delimiter.parse?(tokens, index)
_, new_index = delimiter
new = @parser.parse?(tokens, new_index)
break unless new
new_value, index = new
array << new_value
end
return { array, index }
end
end
class NextParser(T, R) < BasicParser(Array(T | R))
def initialize(@first : BasicParser(T), @second : BasicParser(R))
end
def parse?(tokens, index)
first = @first.parse?(tokens, index)
return nil unless first
first_value, index = first
second = @second.parse?(tokens, index)
return nil unless second
second_value, index = second
array = Array(T | R).new
array << first_value << second_value
return { array, index }
end
end
class PlaceholderParser(T) < BasicParser(T)
property parser : BasicParser(T)?
def initialize
@parser = nil
end
def parse?(tokens, index)
@parser.try &.parse?(tokens, index)
end
end
end