Merge branch 'new-parser-prep' into unit-tests

src/org/nwapw/abacus/Abacus.java

@@ -1,15 +1,15 @@
 package org.nwapw.abacus;
 
 import org.nwapw.abacus.config.ConfigurationObject;
-import org.nwapw.abacus.function.Operator;
 import org.nwapw.abacus.number.NaiveNumber;
 import org.nwapw.abacus.number.NumberInterface;
+import org.nwapw.abacus.parsing.LexerTokenizer;
+import org.nwapw.abacus.parsing.ShuntingYardParser;
+import org.nwapw.abacus.parsing.TreeBuilder;
 import org.nwapw.abacus.plugin.ClassFinder;
-import org.nwapw.abacus.plugin.PluginListener;
 import org.nwapw.abacus.plugin.PluginManager;
 import org.nwapw.abacus.plugin.StandardPlugin;
 import org.nwapw.abacus.tree.NumberReducer;
-import org.nwapw.abacus.tree.TreeBuilder;
 import org.nwapw.abacus.tree.TreeNode;
 import org.nwapw.abacus.window.Window;
 
@@ -23,7 +23,7 @@ import java.lang.reflect.InvocationTargetException;
  * for piecing together all of the components, allowing
  * their interaction with each other.
  */
-public class Abacus implements PluginListener {
+public class Abacus {
 
     /**
      * The default implementation to use for the number representation.
@@ -44,11 +44,6 @@ public class Abacus implements PluginListener {
      * and getting functions from them.
      */
     private PluginManager pluginManager;
-    /**
-     * Tree builder built from plugin manager,
-     * used to construct parse trees.
-     */
-    private TreeBuilder treeBuilder;
     /**
      * The reducer used to evaluate the tree.
      */
@@ -57,6 +52,11 @@ public class Abacus implements PluginListener {
      * The configuration loaded from a file.
      */
     private ConfigurationObject configuration;
+    /**
+     * The tree builder used to construct a tree
+     * from a string.
+     */
+    private TreeBuilder treeBuilder;
 
     /**
      * Creates a new instance of the Abacus calculator.
@@ -67,8 +67,12 @@ public class Abacus implements PluginListener {
         numberReducer = new NumberReducer(this);
         configuration = new ConfigurationObject(CONFIG_FILE);
         configuration.save(CONFIG_FILE);
+        LexerTokenizer lexerTokenizer = new LexerTokenizer();
+        ShuntingYardParser shuntingYardParser = new ShuntingYardParser(this);
+        treeBuilder = new TreeBuilder<>(lexerTokenizer, shuntingYardParser);
 
-        pluginManager.addListener(this);
+        pluginManager.addListener(lexerTokenizer);
+        pluginManager.addListener(shuntingYardParser);
         pluginManager.addInstantiated(new StandardPlugin(pluginManager));
         try {
             ClassFinder.loadJars("plugins")
@@ -129,7 +133,6 @@ public class Abacus implements PluginListener {
      * @return the resulting tree, null if the tree builder or the produced tree are null.
      */
     public TreeNode parseString(String input){
-        if(treeBuilder == null) return null;
         return treeBuilder.fromString(input);
     }
 
@@ -156,26 +159,6 @@ public class Abacus implements PluginListener {
         return null;
     }
 
-    @Override
-    public void onLoad(PluginManager manager) {
-        treeBuilder = new TreeBuilder(this);
-        for(String function : manager.getAllFunctions()){
-            treeBuilder.registerFunction(function);
-        }
-        for(String operator : manager.getAllOperators()){
-            Operator operatorObject = manager.operatorFor(operator);
-            treeBuilder.registerOperator(operator,
-                    operatorObject.getAssociativity(),
-                    operatorObject.getType(),
-                    operatorObject.getPrecedence());
-        }
-    }
-
-    @Override
-    public void onUnload(PluginManager manager) {
-        treeBuilder = null;
-    }
-
     public static void main(String[] args){
         try {
             UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());

src/org/nwapw/abacus/lexing/Lexer.java

@@ -102,7 +102,7 @@ public class Lexer<T> {
                 if(index < from.length() && node.matches(from.charAt(index))) {
                     node.addOutputsInto(futureSet);
                 } else if(node instanceof EndNode){
-                    matches.add(new Match<>(startAt, index, ((EndNode<T>) node).getPatternId()));
+                    matches.add(new Match<>(from.substring(startAt, index), ((EndNode<T>) node).getPatternId()));
                 }
             }
 
@@ -115,7 +115,7 @@ public class Lexer<T> {
         }
         matches.sort((a, b) -> compare.compare(a.getType(), b.getType()));
         if(compare != null) {
-            matches.sort(Comparator.comparingInt(a -> a.getTo() - a.getFrom()));
+            matches.sort(Comparator.comparingInt(a -> a.getContent().length()));
         }
         return matches.isEmpty() ? null : matches.get(matches.size() - 1);
     }
@@ -132,9 +132,10 @@ public class Lexer<T> {
         ArrayList<Match<T>> matches = new ArrayList<>();
         Match<T> lastMatch = null;
         while(index < from.length() && (lastMatch = lexOne(from, index, compare)) != null){
-            if(lastMatch.getTo() == lastMatch.getFrom()) return null;
+            int length = lastMatch.getContent().length();
+            if(length == 0) return null;
             matches.add(lastMatch);
-            index += lastMatch.getTo() - lastMatch.getFrom();
+            index += length;
         }
         if(lastMatch == null) return null;
         return matches;

src/org/nwapw/abacus/lexing/pattern/Match.java

@@ -7,13 +7,9 @@ package org.nwapw.abacus.lexing.pattern;
 public class Match<T> {
 
     /**
-     * The bottom range of the string, inclusive.
+     * The content of this match.
      */
-    private int from;
+    private String content;
-    /**
-     * The top range of the string, exclusive.
-     */
-    private int to;
     /**
      * The pattern type this match matched.
      */
@@ -21,30 +17,20 @@ public class Match<T> {
 
     /**
      * Creates a new match with the given parameters.
-     * @param from the bottom range of the string.
+     * @param content the content of this match.
-     * @param to the top range of the string.
      * @param type the type of the match.
      */
-    public Match(int from, int to, T type){
+    public Match(String content, T type){
-        this.from = from;
+        this.content = content;
-        this.to = to;
         this.type = type;
     }
 
     /**
-     * Gets the bottom range bound of the string.
+     * Gets the content of this match.
-     * @return the bottom range bound of the string.
+     * @return the content.
      */
-    public int getFrom() {
+    public String getContent() {
-        return from;
+        return content;
-    }
-
-    /**
-     * Gets the top range bound of the string.
-     * @return the top range bound of the string.
-     */
-    public int getTo() {
-        return to;
     }
 
     /**

src/org/nwapw/abacus/parsing/LexerTokenizer.java

@@ -0,0 +1,67 @@
+package org.nwapw.abacus.parsing;
+
+import org.nwapw.abacus.lexing.Lexer;
+import org.nwapw.abacus.lexing.pattern.Match;
+import org.nwapw.abacus.lexing.pattern.Pattern;
+import org.nwapw.abacus.plugin.PluginListener;
+import org.nwapw.abacus.plugin.PluginManager;
+import org.nwapw.abacus.tree.TokenType;
+
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * A tokenzier that uses the lexer class and registered function and operator
+ * names to turn input into tokens in O(n) time.
+ */
+public class LexerTokenizer implements Tokenizer<Match<TokenType>>, PluginListener {
+
+    /**
+     * Comparator used to sort the tokens produced by the lexer.
+     */
+    protected static final Comparator<TokenType> TOKEN_SORTER = Comparator.comparingInt(e -> e.priority);
+
+    /**
+     * The lexer instance used to turn strings into matches.
+     */
+    private Lexer<TokenType> lexer;
+
+    /**
+     * Creates a new lexer tokenizer.
+     */
+    public LexerTokenizer(){
+        lexer = new Lexer<TokenType>() {{
+            register(" ", TokenType.WHITESPACE);
+            register(",", TokenType.COMMA);
+            register("[0-9]*(\\.[0-9]+)?", TokenType.NUM);
+            register("\\(", TokenType.OPEN_PARENTH);
+            register("\\)", TokenType.CLOSE_PARENTH);
+        }};
+    }
+
+    @Override
+    public List<Match<TokenType>> tokenizeString(String string) {
+        return lexer.lexAll(string, 0, TOKEN_SORTER);
+    }
+
+    @Override
+    public void onLoad(PluginManager manager) {
+        for(String operator : manager.getAllOperators()){
+           lexer.register(Pattern.sanitize(operator), TokenType.OP);
+        }
+        for(String function : manager.getAllFunctions()){
+            lexer.register(Pattern.sanitize(function), TokenType.FUNCTION);
+        }
+    }
+
+    @Override
+    public void onUnload(PluginManager manager) {
+        for(String operator : manager.getAllOperators()){
+            lexer.unregister(Pattern.sanitize(operator), TokenType.OP);
+        }
+        for(String function : manager.getAllFunctions()){
+            lexer.unregister(Pattern.sanitize(function), TokenType.FUNCTION);
+        }
+    }
+
+}

src/org/nwapw/abacus/parsing/Parser.java

@@ -0,0 +1,20 @@
+package org.nwapw.abacus.parsing;
+
+import org.nwapw.abacus.tree.TreeNode;
+
+import java.util.List;
+
+/**
+ * An itnerface that provides the ability to convert a list of tokens
+ * into a parse tree.
+ * @param <T> the type of tokens accepted by this parser.
+ */
+public interface Parser<T> {
+
+    /**
+     * Constructs a tree out of the given tokens.
+     * @param tokens the tokens to construct a tree from.
+     * @return the constructed tree, or null on error.
+     */
+    public TreeNode constructTree(List<T> tokens);
+}

src/org/nwapw/abacus/parsing/ShuntingYardParser.java

@@ -1,101 +1,56 @@
-package org.nwapw.abacus.tree;
+package org.nwapw.abacus.parsing;
 
 import org.nwapw.abacus.Abacus;
+import org.nwapw.abacus.function.Operator;
 import org.nwapw.abacus.function.OperatorAssociativity;
 import org.nwapw.abacus.function.OperatorType;
-import org.nwapw.abacus.lexing.Lexer;
 import org.nwapw.abacus.lexing.pattern.Match;
-import org.nwapw.abacus.lexing.pattern.Pattern;
+import org.nwapw.abacus.plugin.PluginListener;
+import org.nwapw.abacus.plugin.PluginManager;
+import org.nwapw.abacus.tree.*;
 
 import java.util.*;
 
 /**
- * The builder responsible for turning strings into trees.
+ * A parser that uses shunting yard to rearranged matches into postfix
+ * and then convert them into a parse tree.
  */
-public class TreeBuilder {
+public class ShuntingYardParser implements Parser<Match<TokenType>>, PluginListener {
 
     /**
-     * The lexer used to get the input tokens.
+     * The Abacus instance used to create number instances.
      */
-    private Lexer<TokenType> lexer;
+    private Abacus abacus;
     /**
-     * The map of operator precedences.
+     * Map of operator precedences, loaded from the plugin operators.
      */
     private Map<String, Integer> precedenceMap;
     /**
-     * The map of operator associativity.
+     * Map of operator associativity, loaded from the plugin operators.
      */
     private Map<String, OperatorAssociativity> associativityMap;
     /**
-     * The map of operator types.
+     * Map of operator types, loaded from plugin operators.
      */
     private Map<String, OperatorType> typeMap;
-    /**
-     * The abacus instance required to interact with
-     * other components of the calculator.
-     */
-    private Abacus abacus;
 
     /**
-     * Comparator used to sort token types.
+     * Creates a new Shunting Yard parser with the given Abacus instance.
+     * @param abacus the abacus instance.
      */
-    protected static Comparator<TokenType> tokenSorter = Comparator.comparingInt(e -> e.priority);
+    public ShuntingYardParser(Abacus abacus){
-
-    /**
-     * Creates a new TreeBuilder.
-     */
-    public TreeBuilder(Abacus abacus){
         this.abacus = abacus;
-        lexer = new Lexer<TokenType>(){{
-            register(" ", TokenType.WHITESPACE);
-            register(",", TokenType.COMMA);
-            register("[0-9]*(\\.[0-9]+)?", TokenType.NUM);
-            register("\\(", TokenType.OPEN_PARENTH);
-            register("\\)", TokenType.CLOSE_PARENTH);
-        }};
         precedenceMap = new HashMap<>();
         associativityMap = new HashMap<>();
         typeMap = new HashMap<>();
     }
 
-    /**
-     * Registers a function with the TreeBuilder.
-     * @param function the function to register.
-     */
-    public void registerFunction(String function){
-        lexer.register(Pattern.sanitize(function), TokenType.FUNCTION);
-    }
-
-    /**
-     * Registers an operator with the TreeBuilder.
-     * @param operator the operator to register.
-     * @param precedence the precedence of the operator.
-     * @param associativity the associativity of the operator.
-     */
-    public void registerOperator(String operator, OperatorAssociativity associativity,
-                                 OperatorType operatorType, int precedence){
-        lexer.register(Pattern.sanitize(operator), TokenType.OP);
-        precedenceMap.put(operator, precedence);
-        associativityMap.put(operator, associativity);
-        typeMap.put(operator, operatorType);
-    }
-
-    /**
-     * Tokenizes a string, converting it into matches
-     * @param string the string to tokenize.
-     * @return the list of tokens produced.
-     */
-    public List<Match<TokenType>> tokenize(String string){
-        return lexer.lexAll(string, 0, tokenSorter);
-    }
-
     /**
      * Rearranges tokens into a postfix list, using Shunting Yard.
-     * @param source the source string.
      * @param from the tokens to be rearranged.
      * @return the resulting list of rearranged tokens.
      */
-    public List<Match<TokenType>> intoPostfix(String source, List<Match<TokenType>> from){
+    public List<Match<TokenType>> intoPostfix(List<Match<TokenType>> from){
         ArrayList<Match<TokenType>> output = new ArrayList<>();
         Stack<Match<TokenType>> tokenStack = new Stack<>();
         while(!from.isEmpty()){
@@ -104,10 +59,10 @@ public class TreeBuilder {
             if(matchType == TokenType.NUM) {
                 output.add(match);
             } else if(matchType == TokenType.FUNCTION) {
-                output.add(new Match<>(0, 0, TokenType.INTERNAL_FUNCTION_END));
+                output.add(new Match<>("" , TokenType.INTERNAL_FUNCTION_END));
                 tokenStack.push(match);
             } else if(matchType == TokenType.OP){
-                String tokenString = source.substring(match.getFrom(), match.getTo());
+                String tokenString = match.getContent();
                 OperatorType type = typeMap.get(tokenString);
                 int precedence = precedenceMap.get(tokenString);
                 OperatorAssociativity associativity = associativityMap.get(tokenString);
@@ -123,7 +78,7 @@ public class TreeBuilder {
                     if(!(otherMatchType == TokenType.OP || otherMatchType == TokenType.FUNCTION)) break;
 
                     if(otherMatchType == TokenType.OP){
-                        int otherPrecedence = precedenceMap.get(source.substring(otherMatch.getFrom(), otherMatch.getTo()));
+                        int otherPrecedence = precedenceMap.get(match.getContent());
                         if(otherPrecedence < precedence ||
                                 (associativity == OperatorAssociativity.RIGHT && otherPrecedence == precedence)) {
                             break;
@@ -155,34 +110,33 @@ public class TreeBuilder {
 
     /**
      * Constructs a tree recursively from a list of tokens.
-     * @param source the source string.
      * @param matches the list of tokens from the source string.
      * @return the construct tree expression.
      */
-    public TreeNode fromStringRecursive(String source, List<Match<TokenType>> matches){
+    public TreeNode constructRecursive(List<Match<TokenType>> matches){
         if(matches.size() == 0) return null;
         Match<TokenType> match = matches.remove(0);
         TokenType matchType = match.getType();
         if(matchType == TokenType.OP){
-            String operator = source.substring(match.getFrom(), match.getTo());
+            String operator = match.getContent();
             OperatorType type = typeMap.get(operator);
             if(type == OperatorType.BINARY_INFIX){
-                TreeNode right = fromStringRecursive(source, matches);
+                TreeNode right = constructRecursive(matches);
-                TreeNode left = fromStringRecursive(source, matches);
+                TreeNode left = constructRecursive(matches);
                 if(left == null || right == null) return null;
                 else return new BinaryInfixNode(operator, left, right);
             } else {
-                TreeNode applyTo = fromStringRecursive(source, matches);
+                TreeNode applyTo = constructRecursive(matches);
                 if(applyTo == null) return null;
                 else return new UnaryPrefixNode(operator, applyTo);
             }
         } else if(matchType == TokenType.NUM){
-            return new NumberNode(abacus.numberFromString(source.substring(match.getFrom(), match.getTo())));
+            return new NumberNode(abacus.numberFromString(match.getContent()));
         } else if(matchType == TokenType.FUNCTION){
-            String functionName = source.substring(match.getFrom(), match.getTo());
+            String functionName = match.getContent();
             FunctionNode node = new FunctionNode(functionName);
             while(!matches.isEmpty() && matches.get(0).getType() != TokenType.INTERNAL_FUNCTION_END){
-                TreeNode argument = fromStringRecursive(source, matches);
+                TreeNode argument = constructRecursive(matches);
                 if(argument == null) return null;
                 node.prependChild(argument);
             }
@@ -193,20 +147,27 @@ public class TreeBuilder {
         return null;
     }
 
-    /**
+    @Override
-     * Creates a tree node from a string.
+    public TreeNode constructTree(List<Match<TokenType>> tokens) {
-     * @param string the string to create a node from.
+        tokens = intoPostfix(new ArrayList<>(tokens));
-     * @return the resulting tree.
+        Collections.reverse(tokens);
-     */
+        return constructRecursive(tokens);
-    public TreeNode fromString(String string){
-        List<Match<TokenType>> matches = tokenize(string);
-        if(matches == null) return null;
-        matches.removeIf(m -> m.getType() == TokenType.WHITESPACE);
-        matches = intoPostfix(string, matches);
-        if(matches == null) return null;
-
-        Collections.reverse(matches);
-        return fromStringRecursive(string, matches);
     }
 
+    @Override
+    public void onLoad(PluginManager manager) {
+        for(String operator : manager.getAllOperators()){
+            Operator operatorInstance = manager.operatorFor(operator);
+            precedenceMap.put(operator, operatorInstance.getPrecedence());
+            associativityMap.put(operator, operatorInstance.getAssociativity());
+            typeMap.put(operator, operatorInstance.getType());
+        }
+    }
+
+    @Override
+    public void onUnload(PluginManager manager) {
+        precedenceMap.clear();
+        associativityMap.clear();
+        typeMap.clear();
+    }
 }

src/org/nwapw/abacus/parsing/Tokenizer.java

@@ -0,0 +1,18 @@
+package org.nwapw.abacus.parsing;
+
+import java.util.List;
+
+/**
+ * Interface that provides the ability to convert a string into a list of tokens.
+ * @param <T> the type of the tokens produced.
+ */
+public interface Tokenizer<T> {
+
+    /**
+     * Converts a string into tokens.
+     * @param string the string to convert.
+     * @return the list of tokens, or null on error.
+     */
+    public List<T> tokenizeString(String string);
+
+}

src/org/nwapw/abacus/parsing/TreeBuilder.java

@@ -0,0 +1,23 @@
+package org.nwapw.abacus.parsing;
+
+import org.nwapw.abacus.tree.TreeNode;
+
+import java.util.List;
+
+public class TreeBuilder<T> {
+
+    private Tokenizer<T> tokenizer;
+    private Parser<T> parser;
+
+    public TreeBuilder(Tokenizer<T> tokenizer, Parser<T> parser){
+        this.tokenizer = tokenizer;
+        this.parser = parser;
+    }
+
+    public TreeNode fromString(String input){
+        List<T> tokens = tokenizer.tokenizeString(input);
+        if(tokens == null) return null;
+        return parser.constructTree(tokens);
+    }
+
+}