DanilaFe
/
Abacus
mirror of https://github.com/DanilaFe/abacus
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'new-parser-prep' into unit-tests

This commit is contained in:
Danila Fedorin 2017-07-29 23:42:23 -07:00
parent 542f4b26ab 274826cc09
commit 3131d96d07
8 changed files with 205 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/org/nwapw/abacus/Abacus.java
View File

@ -1,15 +1,15 @@
package org.nwapw.abacus;
import org.nwapw.abacus.config.ConfigurationObject;
import org.nwapw.abacus.function.Operator;
import org.nwapw.abacus.number.NaiveNumber;
import org.nwapw.abacus.number.NumberInterface;
import org.nwapw.abacus.parsing.LexerTokenizer;
import org.nwapw.abacus.parsing.ShuntingYardParser;
import org.nwapw.abacus.parsing.TreeBuilder;
import org.nwapw.abacus.plugin.ClassFinder;
import org.nwapw.abacus.plugin.PluginListener;
import org.nwapw.abacus.plugin.PluginManager;
import org.nwapw.abacus.plugin.StandardPlugin;
import org.nwapw.abacus.tree.NumberReducer;
import org.nwapw.abacus.tree.TreeBuilder;
import org.nwapw.abacus.tree.TreeNode;
import org.nwapw.abacus.window.Window;
@ -23,7 +23,7 @@ import java.lang.reflect.InvocationTargetException;
* for piecing together all of the components, allowing
* their interaction with each other.
*/
public class Abacus implements PluginListener {
public class Abacus {
/**
* The default implementation to use for the number representation.
@ -44,11 +44,6 @@ public class Abacus implements PluginListener {
* and getting functions from them.
*/
private PluginManager pluginManager;
/**
* Tree builder built from plugin manager,
* used to construct parse trees.
*/
private TreeBuilder treeBuilder;
/**
* The reducer used to evaluate the tree.
*/
@ -57,6 +52,11 @@ public class Abacus implements PluginListener {
* The configuration loaded from a file.
*/
private ConfigurationObject configuration;
/**
* The tree builder used to construct a tree
* from a string.
*/
private TreeBuilder treeBuilder;
/**
* Creates a new instance of the Abacus calculator.
@ -67,8 +67,12 @@ public class Abacus implements PluginListener {
numberReducer = new NumberReducer(this);
configuration = new ConfigurationObject(CONFIG_FILE);
configuration.save(CONFIG_FILE);
LexerTokenizer lexerTokenizer = new LexerTokenizer();
ShuntingYardParser shuntingYardParser = new ShuntingYardParser(this);
treeBuilder = new TreeBuilder<>(lexerTokenizer, shuntingYardParser);
pluginManager.addListener(this);
pluginManager.addListener(lexerTokenizer);
pluginManager.addListener(shuntingYardParser);
pluginManager.addInstantiated(new StandardPlugin(pluginManager));
try {
ClassFinder.loadJars("plugins")
@ -129,7 +133,6 @@ public class Abacus implements PluginListener {
* @return the resulting tree, null if the tree builder or the produced tree are null.
*/
public TreeNode parseString(String input){
if(treeBuilder == null) return null;
return treeBuilder.fromString(input);
}
@ -156,26 +159,6 @@ public class Abacus implements PluginListener {
return null;
}
@Override
public void onLoad(PluginManager manager) {
treeBuilder = new TreeBuilder(this);
for(String function : manager.getAllFunctions()){
treeBuilder.registerFunction(function);
}
for(String operator : manager.getAllOperators()){
Operator operatorObject = manager.operatorFor(operator);
treeBuilder.registerOperator(operator,
operatorObject.getAssociativity(),
operatorObject.getType(),
operatorObject.getPrecedence());
}
}
@Override
public void onUnload(PluginManager manager) {
treeBuilder = null;
}
public static void main(String[] args){
try {
UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());

9
src/org/nwapw/abacus/lexing/Lexer.java
View File

@ -102,7 +102,7 @@ public class Lexer<T> {
if(index < from.length() && node.matches(from.charAt(index))) {
node.addOutputsInto(futureSet);
} else if(node instanceof EndNode){
matches.add(new Match<>(startAt, index, ((EndNode<T>) node).getPatternId()));
matches.add(new Match<>(from.substring(startAt, index), ((EndNode<T>) node).getPatternId()));
}
}
@ -115,7 +115,7 @@ public class Lexer<T> {
}
matches.sort((a, b) -> compare.compare(a.getType(), b.getType()));
if(compare != null) {
matches.sort(Comparator.comparingInt(a -> a.getTo() - a.getFrom()));
matches.sort(Comparator.comparingInt(a -> a.getContent().length()));
}
return matches.isEmpty() ? null : matches.get(matches.size() - 1);
}
@ -132,9 +132,10 @@ public class Lexer<T> {
ArrayList<Match<T>> matches = new ArrayList<>();
Match<T> lastMatch = null;
while(index < from.length() && (lastMatch = lexOne(from, index, compare)) != null){
if(lastMatch.getTo() == lastMatch.getFrom()) return null;
int length = lastMatch.getContent().length();
if(length == 0) return null;
matches.add(lastMatch);
index += lastMatch.getTo() - lastMatch.getFrom();
index += length;
}
if(lastMatch == null) return null;
return matches;

32
src/org/nwapw/abacus/lexing/pattern/Match.java
View File

@ -7,13 +7,9 @@ package org.nwapw.abacus.lexing.pattern;
public class Match<T> {
/**
* The bottom range of the string, inclusive.
* The content of this match.
*/
private int from;
/**
* The top range of the string, exclusive.
*/
private int to;
private String content;
/**
* The pattern type this match matched.
*/
@ -21,30 +17,20 @@ public class Match<T> {
/**
* Creates a new match with the given parameters.
* @param from the bottom range of the string.
* @param to the top range of the string.
* @param content the content of this match.
* @param type the type of the match.
*/
public Match(int from, int to, T type){
this.from = from;
this.to = to;
public Match(String content, T type){
this.content = content;
this.type = type;
}
/**
* Gets the bottom range bound of the string.
* @return the bottom range bound of the string.
* Gets the content of this match.
* @return the content.
*/
public int getFrom() {
return from;
}
/**
* Gets the top range bound of the string.
* @return the top range bound of the string.
*/
public int getTo() {
return to;
public String getContent() {
return content;
}
/**

67
src/org/nwapw/abacus/parsing/LexerTokenizer.java Normal file
View File

@ -0,0 +1,67 @@
package org.nwapw.abacus.parsing;
import org.nwapw.abacus.lexing.Lexer;
import org.nwapw.abacus.lexing.pattern.Match;
import org.nwapw.abacus.lexing.pattern.Pattern;
import org.nwapw.abacus.plugin.PluginListener;
import org.nwapw.abacus.plugin.PluginManager;
import org.nwapw.abacus.tree.TokenType;
import java.util.Comparator;
import java.util.List;
/**
* A tokenzier that uses the lexer class and registered function and operator
* names to turn input into tokens in O(n) time.
*/
public class LexerTokenizer implements Tokenizer<Match<TokenType>>, PluginListener {
/**
* Comparator used to sort the tokens produced by the lexer.
*/
protected static final Comparator<TokenType> TOKEN_SORTER = Comparator.comparingInt(e -> e.priority);
/**
* The lexer instance used to turn strings into matches.
*/
private Lexer<TokenType> lexer;
/**
* Creates a new lexer tokenizer.
*/
public LexerTokenizer(){
lexer = new Lexer<TokenType>() {{
register(" ", TokenType.WHITESPACE);
register(",", TokenType.COMMA);
register("[0-9]*(\\.[0-9]+)?", TokenType.NUM);
register("\\(", TokenType.OPEN_PARENTH);
register("\\)", TokenType.CLOSE_PARENTH);
}};
}
@Override
public List<Match<TokenType>> tokenizeString(String string) {
return lexer.lexAll(string, 0, TOKEN_SORTER);
}
@Override
public void onLoad(PluginManager manager) {
for(String operator : manager.getAllOperators()){
lexer.register(Pattern.sanitize(operator), TokenType.OP);
}
for(String function : manager.getAllFunctions()){
lexer.register(Pattern.sanitize(function), TokenType.FUNCTION);
}
}
@Override
public void onUnload(PluginManager manager) {
for(String operator : manager.getAllOperators()){
lexer.unregister(Pattern.sanitize(operator), TokenType.OP);
}
for(String function : manager.getAllFunctions()){
lexer.unregister(Pattern.sanitize(function), TokenType.FUNCTION);
}
}
}

20
src/org/nwapw/abacus/parsing/Parser.java Normal file
View File

@ -0,0 +1,20 @@
package org.nwapw.abacus.parsing;
import org.nwapw.abacus.tree.TreeNode;
import java.util.List;
/**
* An itnerface that provides the ability to convert a list of tokens
* into a parse tree.
* @param <T> the type of tokens accepted by this parser.
*/
public interface Parser<T> {
/**
* Constructs a tree out of the given tokens.
* @param tokens the tokens to construct a tree from.
* @return the constructed tree, or null on error.
*/
public TreeNode constructTree(List<T> tokens);
}

137
src/org/nwapw/abacus/tree/TreeBuilder.java → src/org/nwapw/abacus/parsing/ShuntingYardParser.java
View File

@ -1,101 +1,56 @@
package org.nwapw.abacus.tree;
package org.nwapw.abacus.parsing;
import org.nwapw.abacus.Abacus;
import org.nwapw.abacus.function.Operator;
import org.nwapw.abacus.function.OperatorAssociativity;
import org.nwapw.abacus.function.OperatorType;
import org.nwapw.abacus.lexing.Lexer;
import org.nwapw.abacus.lexing.pattern.Match;
import org.nwapw.abacus.lexing.pattern.Pattern;
import org.nwapw.abacus.plugin.PluginListener;
import org.nwapw.abacus.plugin.PluginManager;
import org.nwapw.abacus.tree.*;
import java.util.*;
/**
* The builder responsible for turning strings into trees.
* A parser that uses shunting yard to rearranged matches into postfix
* and then convert them into a parse tree.
*/
public class TreeBuilder {
public class ShuntingYardParser implements Parser<Match<TokenType>>, PluginListener {
/**
* The lexer used to get the input tokens.
* The Abacus instance used to create number instances.
*/
private Lexer<TokenType> lexer;
private Abacus abacus;
/**
* The map of operator precedences.
* Map of operator precedences, loaded from the plugin operators.
*/
private Map<String, Integer> precedenceMap;
/**
* The map of operator associativity.
* Map of operator associativity, loaded from the plugin operators.
*/
private Map<String, OperatorAssociativity> associativityMap;
/**
* The map of operator types.
* Map of operator types, loaded from plugin operators.
*/
private Map<String, OperatorType> typeMap;
/**
* The abacus instance required to interact with
* other components of the calculator.
*/
private Abacus abacus;
/**
* Comparator used to sort token types.
* Creates a new Shunting Yard parser with the given Abacus instance.
* @param abacus the abacus instance.
*/
protected static Comparator<TokenType> tokenSorter = Comparator.comparingInt(e -> e.priority);
/**
* Creates a new TreeBuilder.
*/
public TreeBuilder(Abacus abacus){
public ShuntingYardParser(Abacus abacus){
this.abacus = abacus;
lexer = new Lexer<TokenType>(){{
register(" ", TokenType.WHITESPACE);
register(",", TokenType.COMMA);
register("[0-9]*(\\.[0-9]+)?", TokenType.NUM);
register("\\(", TokenType.OPEN_PARENTH);
register("\\)", TokenType.CLOSE_PARENTH);
}};
precedenceMap = new HashMap<>();
associativityMap = new HashMap<>();
typeMap = new HashMap<>();
}
/**
* Registers a function with the TreeBuilder.
* @param function the function to register.
*/
public void registerFunction(String function){
lexer.register(Pattern.sanitize(function), TokenType.FUNCTION);
}
/**
* Registers an operator with the TreeBuilder.
* @param operator the operator to register.
* @param precedence the precedence of the operator.
* @param associativity the associativity of the operator.
*/
public void registerOperator(String operator, OperatorAssociativity associativity,
OperatorType operatorType, int precedence){
lexer.register(Pattern.sanitize(operator), TokenType.OP);
precedenceMap.put(operator, precedence);
associativityMap.put(operator, associativity);
typeMap.put(operator, operatorType);
}
/**
* Tokenizes a string, converting it into matches
* @param string the string to tokenize.
* @return the list of tokens produced.
*/
public List<Match<TokenType>> tokenize(String string){
return lexer.lexAll(string, 0, tokenSorter);
}
/**
* Rearranges tokens into a postfix list, using Shunting Yard.
* @param source the source string.
* @param from the tokens to be rearranged.
* @return the resulting list of rearranged tokens.
*/
public List<Match<TokenType>> intoPostfix(String source, List<Match<TokenType>> from){
public List<Match<TokenType>> intoPostfix(List<Match<TokenType>> from){
ArrayList<Match<TokenType>> output = new ArrayList<>();
Stack<Match<TokenType>> tokenStack = new Stack<>();
while(!from.isEmpty()){
@ -104,10 +59,10 @@ public class TreeBuilder {
if(matchType == TokenType.NUM) {
output.add(match);
} else if(matchType == TokenType.FUNCTION) {
output.add(new Match<>(0, 0, TokenType.INTERNAL_FUNCTION_END));
output.add(new Match<>("" , TokenType.INTERNAL_FUNCTION_END));
tokenStack.push(match);
} else if(matchType == TokenType.OP){
String tokenString = source.substring(match.getFrom(), match.getTo());
String tokenString = match.getContent();
OperatorType type = typeMap.get(tokenString);
int precedence = precedenceMap.get(tokenString);
OperatorAssociativity associativity = associativityMap.get(tokenString);
@ -123,7 +78,7 @@ public class TreeBuilder {
if(!(otherMatchType == TokenType.OP || otherMatchType == TokenType.FUNCTION)) break;
if(otherMatchType == TokenType.OP){
int otherPrecedence = precedenceMap.get(source.substring(otherMatch.getFrom(), otherMatch.getTo()));
int otherPrecedence = precedenceMap.get(match.getContent());
if(otherPrecedence < precedence ||
(associativity == OperatorAssociativity.RIGHT && otherPrecedence == precedence)) {
break;
@ -155,34 +110,33 @@ public class TreeBuilder {
/**
* Constructs a tree recursively from a list of tokens.
* @param source the source string.
* @param matches the list of tokens from the source string.
* @return the construct tree expression.
*/
public TreeNode fromStringRecursive(String source, List<Match<TokenType>> matches){
public TreeNode constructRecursive(List<Match<TokenType>> matches){
if(matches.size() == 0) return null;
Match<TokenType> match = matches.remove(0);
TokenType matchType = match.getType();
if(matchType == TokenType.OP){
String operator = source.substring(match.getFrom(), match.getTo());
String operator = match.getContent();
OperatorType type = typeMap.get(operator);
if(type == OperatorType.BINARY_INFIX){
TreeNode right = fromStringRecursive(source, matches);
TreeNode left = fromStringRecursive(source, matches);
TreeNode right = constructRecursive(matches);
TreeNode left = constructRecursive(matches);
if(left == null || right == null) return null;
else return new BinaryInfixNode(operator, left, right);
} else {
TreeNode applyTo = fromStringRecursive(source, matches);
TreeNode applyTo = constructRecursive(matches);
if(applyTo == null) return null;
else return new UnaryPrefixNode(operator, applyTo);
}
} else if(matchType == TokenType.NUM){
return new NumberNode(abacus.numberFromString(source.substring(match.getFrom(), match.getTo())));
return new NumberNode(abacus.numberFromString(match.getContent()));
} else if(matchType == TokenType.FUNCTION){
String functionName = source.substring(match.getFrom(), match.getTo());
String functionName = match.getContent();
FunctionNode node = new FunctionNode(functionName);
while(!matches.isEmpty() && matches.get(0).getType() != TokenType.INTERNAL_FUNCTION_END){
TreeNode argument = fromStringRecursive(source, matches);
TreeNode argument = constructRecursive(matches);
if(argument == null) return null;
node.prependChild(argument);
}
@ -193,20 +147,27 @@ public class TreeBuilder {
return null;
}
/**
* Creates a tree node from a string.
* @param string the string to create a node from.
* @return the resulting tree.
*/
public TreeNode fromString(String string){
List<Match<TokenType>> matches = tokenize(string);
if(matches == null) return null;
matches.removeIf(m -> m.getType() == TokenType.WHITESPACE);
matches = intoPostfix(string, matches);
if(matches == null) return null;
Collections.reverse(matches);
return fromStringRecursive(string, matches);
@Override
public TreeNode constructTree(List<Match<TokenType>> tokens) {
tokens = intoPostfix(new ArrayList<>(tokens));
Collections.reverse(tokens);
return constructRecursive(tokens);
}
@Override
public void onLoad(PluginManager manager) {
for(String operator : manager.getAllOperators()){
Operator operatorInstance = manager.operatorFor(operator);
precedenceMap.put(operator, operatorInstance.getPrecedence());
associativityMap.put(operator, operatorInstance.getAssociativity());
typeMap.put(operator, operatorInstance.getType());
}
}
@Override
public void onUnload(PluginManager manager) {
precedenceMap.clear();
associativityMap.clear();
typeMap.clear();
}
}

18
src/org/nwapw/abacus/parsing/Tokenizer.java Normal file
View File

@ -0,0 +1,18 @@
package org.nwapw.abacus.parsing;
import java.util.List;
/**
* Interface that provides the ability to convert a string into a list of tokens.
* @param <T> the type of the tokens produced.
*/
public interface Tokenizer<T> {
/**
* Converts a string into tokens.
* @param string the string to convert.
* @return the list of tokens, or null on error.
*/
public List<T> tokenizeString(String string);
}

23
src/org/nwapw/abacus/parsing/TreeBuilder.java Normal file
View File

@ -0,0 +1,23 @@
package org.nwapw.abacus.parsing;
import org.nwapw.abacus.tree.TreeNode;
import java.util.List;
public class TreeBuilder<T> {
private Tokenizer<T> tokenizer;
private Parser<T> parser;
public TreeBuilder(Tokenizer<T> tokenizer, Parser<T> parser){
this.tokenizer = tokenizer;
this.parser = parser;
}
public TreeNode fromString(String input){
List<T> tokens = tokenizer.tokenizeString(input);
if(tokens == null) return null;
return parser.constructTree(tokens);
}
}