From f931b9f3225dde98803177941b206d535249115d Mon Sep 17 00:00:00 2001 From: Danila Fedorin Date: Thu, 27 Jul 2017 12:53:58 -0700 Subject: [PATCH] Move parsing code into TreeBuilder, change lexing and parsing algorithms --- .../nwapw/abacus/plugin/PluginManager.java | 1 + src/org/nwapw/abacus/tree/OpNode.java | 9 +- src/org/nwapw/abacus/tree/TokenType.java | 3 +- src/org/nwapw/abacus/tree/TreeBuilder.java | 142 ++++++++++++++++ src/org/nwapw/abacus/tree/TreeNode.java | 152 ------------------ src/org/nwapw/abacus/window/Window.java | 8 +- 6 files changed, 153 insertions(+), 162 deletions(-) create mode 100644 src/org/nwapw/abacus/tree/TreeBuilder.java diff --git a/src/org/nwapw/abacus/plugin/PluginManager.java b/src/org/nwapw/abacus/plugin/PluginManager.java index 7442b17..97b421c 100644 --- a/src/org/nwapw/abacus/plugin/PluginManager.java +++ b/src/org/nwapw/abacus/plugin/PluginManager.java @@ -33,6 +33,7 @@ public class PluginManager { public PluginManager(){ plugins = new ArrayList<>(); cachedFunctions = new HashMap<>(); + cachedOperators = new HashMap<>(); } /** diff --git a/src/org/nwapw/abacus/tree/OpNode.java b/src/org/nwapw/abacus/tree/OpNode.java index e6565f7..475affc 100644 --- a/src/org/nwapw/abacus/tree/OpNode.java +++ b/src/org/nwapw/abacus/tree/OpNode.java @@ -95,13 +95,6 @@ public class OpNode extends TreeNode { String leftString = left != null ? left.toString() : "null"; String rightString = right != null ? right.toString() : "null"; - if(right != null && right instanceof OpNode){ - if(TreeNode.precedenceMap.get(((OpNode) right).getOperation()) > - TreeNode.precedenceMap.get(operation)) { - rightString = "(" + rightString + ")"; - } - } - - return leftString + operation + rightString; + return "(" + leftString + operation + rightString + ")"; } } diff --git a/src/org/nwapw/abacus/tree/TokenType.java b/src/org/nwapw/abacus/tree/TokenType.java index 06301ae..1d70998 100644 --- a/src/org/nwapw/abacus/tree/TokenType.java +++ b/src/org/nwapw/abacus/tree/TokenType.java @@ -6,7 +6,8 @@ package org.nwapw.abacus.tree; */ public enum TokenType { - INTERNAL_FUNCTION_END(-1), INTERNAL_FUNCTION_START(-1), ANY(0), COMMA(1), OP(2), NUM(3), WORD(4), OPEN_PARENTH(5), CLOSE_PARENTH(6); + INTERNAL_FUNCTION_END(-1), + ANY(0), COMMA(1), OP(2), NUM(3), FUNCTION(4), OPEN_PARENTH(5), CLOSE_PARENTH(6); /** * The priority by which this token gets sorted. diff --git a/src/org/nwapw/abacus/tree/TreeBuilder.java b/src/org/nwapw/abacus/tree/TreeBuilder.java new file mode 100644 index 0000000..d0560e1 --- /dev/null +++ b/src/org/nwapw/abacus/tree/TreeBuilder.java @@ -0,0 +1,142 @@ +package org.nwapw.abacus.tree; + +import org.nwapw.abacus.function.OperatorAssociativity; +import org.nwapw.abacus.lexing.Lexer; +import org.nwapw.abacus.lexing.pattern.Match; +import org.nwapw.abacus.plugin.PluginManager; + +import java.util.*; + +public class TreeBuilder { + + private Lexer lexer; + private HashMap precedenceMap; + private HashMap associativityMap; + + /** + * Comparator used to sort token types. + */ + protected static Comparator tokenSorter = Comparator.comparingInt(e -> e.priority); + + public TreeBuilder(){ + lexer = new Lexer(){{ + register(",", TokenType.COMMA); + register("[0-9]+(\\.[0-9]+)?", TokenType.NUM); + register("\\(", TokenType.OPEN_PARENTH); + register("\\)", TokenType.CLOSE_PARENTH); + }}; + precedenceMap = new HashMap<>(); + associativityMap = new HashMap<>(); + } + + /** + * Tokenizes a string, converting it into matches + * @param string the string to tokenize. + * @return the list of tokens produced. + */ + public ArrayList> tokenize(String string){ + return lexer.lexAll(string, 0, tokenSorter); + } + + /** + * Rearranges tokens into a postfix list, using Shunting Yard. + * @param source the source string. + * @param from the tokens to be rearranged. + * @return the resulting list of rearranged tokens. + */ + public ArrayList> intoPostfix(String source, ArrayList> from){ + ArrayList> output = new ArrayList<>(); + Stack> tokenStack = new Stack<>(); + while(!from.isEmpty()){ + Match match = from.remove(0); + TokenType matchType = match.getType(); + if(matchType == TokenType.NUM) { + output.add(match); + } else if(matchType == TokenType.FUNCTION) { + output.add(new Match<>(0, 0, TokenType.INTERNAL_FUNCTION_END)); + tokenStack.push(match); + } else if(matchType == TokenType.OP){ + String tokenString = source.substring(match.getFrom(), match.getTo()); + int precedence = precedenceMap.get(tokenString); + OperatorAssociativity associativity = associativityMap.get(tokenString); + + while(!tokenStack.empty()) { + Match otherMatch = tokenStack.peek(); + TokenType otherMatchType = otherMatch.getType(); + if(otherMatchType != TokenType.OP) break; + + int otherPrecdence = precedenceMap.get(source.substring(otherMatch.getFrom(), otherMatch.getTo())); + if(otherPrecdence < precedence || + (associativity == OperatorAssociativity.RIGHT && otherPrecdence == precedence)) { + break; + } + output.add(tokenStack.pop()); + } + tokenStack.push(match); + } else if(matchType == TokenType.OPEN_PARENTH){ + tokenStack.push(match); + } else if(matchType == TokenType.CLOSE_PARENTH || matchType == TokenType.COMMA){ + while(!tokenStack.empty() && tokenStack.peek().getType() != TokenType.OPEN_PARENTH){ + output.add(tokenStack.pop()); + } + if(tokenStack.empty()) return null; + if(matchType == TokenType.CLOSE_PARENTH){ + tokenStack.pop(); + } + } + } + while(!tokenStack.empty()){ + if(!(tokenStack.peek().getType() == TokenType.OP)) return null; + output.add(tokenStack.pop()); + } + return output; + } + + /** + * Constructs a tree recursively from a list of tokens. + * @param source the source string. + * @param matches the list of tokens from the source string. + * @return the construct tree expression. + */ + public TreeNode fromStringRecursive(String source, ArrayList> matches){ + if(matches.size() == 0) return null; + Match match = matches.remove(0); + TokenType matchType = match.getType(); + if(matchType == TokenType.OP){ + TreeNode right = fromStringRecursive(source, matches); + TreeNode left = fromStringRecursive(source, matches); + if(left == null || right == null) return null; + else return new OpNode(source.substring(match.getFrom(), match.getTo()), left, right); + } else if(matchType == TokenType.NUM){ + return new NumberNode(Double.parseDouble(source.substring(match.getFrom(), match.getTo()))); + } else if(matchType == TokenType.FUNCTION){ + String functionName = source.substring(match.getFrom(), match.getTo()); + FunctionNode node = new FunctionNode(functionName); + while(!matches.isEmpty() && matches.get(0).getType() != TokenType.INTERNAL_FUNCTION_END){ + TreeNode argument = fromStringRecursive(source, matches); + if(argument == null) return null; + node.addChild(argument); + } + if(matches.isEmpty()) return null; + matches.remove(0); + return node; + } + return null; + } + + /** + * Creates a tree node from a string. + * @param string the string to create a node from. + * @return the resulting tree. + */ + public TreeNode fromString(String string){ + ArrayList> matches = tokenize(string); + if(matches == null) return null; + matches = intoPostfix(string, matches); + if(matches == null) return null; + + Collections.reverse(matches); + return fromStringRecursive(string, matches); + } + +} diff --git a/src/org/nwapw/abacus/tree/TreeNode.java b/src/org/nwapw/abacus/tree/TreeNode.java index 93efc02..a3cae09 100644 --- a/src/org/nwapw/abacus/tree/TreeNode.java +++ b/src/org/nwapw/abacus/tree/TreeNode.java @@ -11,158 +11,6 @@ import java.util.*; */ public abstract class TreeNode { - /** - * The lexer used to lex tokens. - */ - protected static Lexer lexer = new Lexer(){{ - register(",", TokenType.COMMA); - register("\\+|-|\\*|/", TokenType.OP); - register("[0-9]+(\\.[0-9]+)?", TokenType.NUM); - register("[a-zA-Z]+", TokenType.WORD); - register("\\(", TokenType.OPEN_PARENTH); - register("\\)", TokenType.CLOSE_PARENTH); - }}; - /** - * A map that maps operations to their precedence. - */ - protected static HashMap precedenceMap = new HashMap(){{ - put("+", 0); - put("-", 0); - put("*", 1); - put("/", 1); - }}; - /** - * A map that maps operations to their associativity. - */ - protected static HashMap associativityMap = - new HashMap() {{ - put("+", OperatorAssociativity.LEFT); - put("-", OperatorAssociativity.LEFT); - put("*", OperatorAssociativity.LEFT); - put("/", OperatorAssociativity.LEFT); - }}; - - /** - * Comparator used to sort token types. - */ - protected static Comparator tokenSorter = Comparator.comparingInt(e -> e.priority); - - /** - * Tokenizes a string, converting it into matches - * @param string the string to tokenize. - * @return the list of tokens produced. - */ - public static ArrayList> tokenize(String string){ - return lexer.lexAll(string, 0, tokenSorter); - } - - /** - * Rearranges tokens into a postfix list, using Shunting Yard. - * @param source the source string. - * @param from the tokens to be rearranged. - * @return the resulting list of rearranged tokens. - */ - public static ArrayList> intoPostfix(String source, ArrayList> from){ - ArrayList> output = new ArrayList<>(); - Stack> tokenStack = new Stack<>(); - while(!from.isEmpty()){ - Match match = from.remove(0); - TokenType matchType = match.getType(); - if(matchType == TokenType.NUM || matchType == TokenType.WORD) { - output.add(match); - } else if(matchType == TokenType.OP){ - String tokenString = source.substring(match.getFrom(), match.getTo()); - int precedence = precedenceMap.get(tokenString); - OperatorAssociativity associativity = associativityMap.get(tokenString); - - while(!tokenStack.empty()) { - Match otherMatch = tokenStack.peek(); - if(otherMatch.getType() != TokenType.OP) break; - - int otherPrecdence = precedenceMap.get(source.substring(otherMatch.getFrom(), otherMatch.getTo())); - if(otherPrecdence < precedence || - (associativity == OperatorAssociativity.RIGHT && otherPrecdence == precedence)) { - break; - } - output.add(tokenStack.pop()); - } - tokenStack.push(match); - } else if(matchType == TokenType.OPEN_PARENTH){ - if(!output.isEmpty() && output.get(output.size() - 1).getType() == TokenType.WORD){ - tokenStack.push(output.remove(output.size() - 1)); - output.add(new Match<>(0, 0, TokenType.INTERNAL_FUNCTION_END)); - } - tokenStack.push(match); - } else if(matchType == TokenType.CLOSE_PARENTH || matchType == TokenType.COMMA){ - while(!tokenStack.empty() && tokenStack.peek().getType() != TokenType.OPEN_PARENTH){ - output.add(tokenStack.pop()); - } - if(tokenStack.empty()) return null; - if(matchType == TokenType.CLOSE_PARENTH){ - tokenStack.pop(); - if(!tokenStack.empty() && tokenStack.peek().getType() == TokenType.WORD) { - output.add(tokenStack.pop()); - output.add(new Match<>(0, 0, TokenType.INTERNAL_FUNCTION_START)); - } - } - } - } - while(!tokenStack.empty()){ - if(!(tokenStack.peek().getType() == TokenType.OP)) return null; - output.add(tokenStack.pop()); - } - return output; - } - - /** - * Constructs a tree recursively from a list of tokens. - * @param source the source string. - * @param matches the list of tokens from the source string. - * @return the construct tree expression. - */ - public static TreeNode fromStringRecursive(String source, ArrayList> matches){ - if(matches.size() == 0) return null; - Match match = matches.remove(0); - TokenType matchType = match.getType(); - if(matchType == TokenType.OP){ - TreeNode right = fromStringRecursive(source, matches); - TreeNode left = fromStringRecursive(source, matches); - if(left == null || right == null) return null; - else return new OpNode(source.substring(match.getFrom(), match.getTo()), left, right); - } else if(matchType == TokenType.NUM){ - return new NumberNode(Double.parseDouble(source.substring(match.getFrom(), match.getTo()))); - } else if(matchType == TokenType.INTERNAL_FUNCTION_START){ - if(matches.isEmpty() || matches.get(0).getType() != TokenType.WORD) return null; - Match stringName = matches.remove(0); - String functionName = source.substring(stringName.getFrom(), stringName.getTo()); - FunctionNode node = new FunctionNode(functionName); - while(!matches.isEmpty() && matches.get(0).getType() != TokenType.INTERNAL_FUNCTION_END){ - TreeNode argument = fromStringRecursive(source, matches); - if(argument == null) return null; - node.addChild(argument); - } - if(matches.isEmpty()) return null; - matches.remove(0); - return node; - } - return null; - } - - /** - * Creates a tree node from a string. - * @param string the string to create a node from. - * @return the resulting tree. - */ - public static TreeNode fromString(String string){ - ArrayList> matches = tokenize(string); - if(matches == null) return null; - matches = intoPostfix(string, matches); - if(matches == null) return null; - - Collections.reverse(matches); - return fromStringRecursive(string, matches); - } - public abstract T reduce(Reducer reducer); } diff --git a/src/org/nwapw/abacus/window/Window.java b/src/org/nwapw/abacus/window/Window.java index d2d4891..e419820 100644 --- a/src/org/nwapw/abacus/window/Window.java +++ b/src/org/nwapw/abacus/window/Window.java @@ -3,6 +3,7 @@ package org.nwapw.abacus.window; import org.nwapw.abacus.number.NumberInterface; import org.nwapw.abacus.plugin.PluginManager; import org.nwapw.abacus.tree.NumberReducer; +import org.nwapw.abacus.tree.TreeBuilder; import org.nwapw.abacus.tree.TreeNode; import javax.swing.*; @@ -47,6 +48,10 @@ public class Window extends JFrame { * The plugin manager used to retrieve functions. */ private PluginManager manager; + /** + * The builder used to construct the parse trees. + */ + private TreeBuilder builder; /** * The reducer used to evaluate the tree. */ @@ -123,7 +128,8 @@ public class Window extends JFrame { * Action listener that causes the input to be evaluated. */ private ActionListener evaluateListener = (event) -> { - TreeNode parsedExpression = TreeNode.fromString(inputField.getText()); + TreeBuilder builder = new TreeBuilder(); + TreeNode parsedExpression = builder.fromString(inputField.getText()); if(parsedExpression == null){ lastOutputArea.setText(SYNTAX_ERR_STRING); return;