1
0
mirror of https://github.com/DanilaFe/abacus synced 2024-12-23 07:50:09 -08:00

Change matches to store the string they matched.

This commit is contained in:
Danila Fedorin 2017-07-29 21:13:32 -07:00
parent 7b1b9aa01c
commit 65e8b7d15e
3 changed files with 28 additions and 43 deletions

View File

@ -102,7 +102,7 @@ public class Lexer<T> {
if(index < from.length() && node.matches(from.charAt(index))) { if(index < from.length() && node.matches(from.charAt(index))) {
node.addOutputsInto(futureSet); node.addOutputsInto(futureSet);
} else if(node instanceof EndNode){ } else if(node instanceof EndNode){
matches.add(new Match<>(startAt, index, ((EndNode<T>) node).getPatternId())); matches.add(new Match<>(from.substring(startAt, index), ((EndNode<T>) node).getPatternId()));
} }
} }
@ -115,7 +115,7 @@ public class Lexer<T> {
} }
matches.sort((a, b) -> compare.compare(a.getType(), b.getType())); matches.sort((a, b) -> compare.compare(a.getType(), b.getType()));
if(compare != null) { if(compare != null) {
matches.sort(Comparator.comparingInt(a -> a.getTo() - a.getFrom())); matches.sort(Comparator.comparingInt(a -> a.getContent().length()));
} }
return matches.isEmpty() ? null : matches.get(matches.size() - 1); return matches.isEmpty() ? null : matches.get(matches.size() - 1);
} }
@ -132,9 +132,10 @@ public class Lexer<T> {
ArrayList<Match<T>> matches = new ArrayList<>(); ArrayList<Match<T>> matches = new ArrayList<>();
Match<T> lastMatch = null; Match<T> lastMatch = null;
while(index < from.length() && (lastMatch = lexOne(from, index, compare)) != null){ while(index < from.length() && (lastMatch = lexOne(from, index, compare)) != null){
if(lastMatch.getTo() == lastMatch.getFrom()) return null; int length = lastMatch.getContent().length();
if(length == 0) return null;
matches.add(lastMatch); matches.add(lastMatch);
index += lastMatch.getTo() - lastMatch.getFrom(); index += length;
} }
if(lastMatch == null) return null; if(lastMatch == null) return null;
return matches; return matches;

View File

@ -7,13 +7,9 @@ package org.nwapw.abacus.lexing.pattern;
public class Match<T> { public class Match<T> {
/** /**
* The bottom range of the string, inclusive. * The content of this match.
*/ */
private int from; private String content;
/**
* The top range of the string, exclusive.
*/
private int to;
/** /**
* The pattern type this match matched. * The pattern type this match matched.
*/ */
@ -21,30 +17,20 @@ public class Match<T> {
/** /**
* Creates a new match with the given parameters. * Creates a new match with the given parameters.
* @param from the bottom range of the string. * @param content the content of this match.
* @param to the top range of the string.
* @param type the type of the match. * @param type the type of the match.
*/ */
public Match(int from, int to, T type){ public Match(String content, T type){
this.from = from; this.content = content;
this.to = to;
this.type = type; this.type = type;
} }
/** /**
* Gets the bottom range bound of the string. * Gets the content of this match.
* @return the bottom range bound of the string. * @return the content.
*/ */
public int getFrom() { public String getContent() {
return from; return content;
}
/**
* Gets the top range bound of the string.
* @return the top range bound of the string.
*/
public int getTo() {
return to;
} }
/** /**

View File

@ -91,11 +91,10 @@ public class TreeBuilder {
/** /**
* Rearranges tokens into a postfix list, using Shunting Yard. * Rearranges tokens into a postfix list, using Shunting Yard.
* @param source the source string.
* @param from the tokens to be rearranged. * @param from the tokens to be rearranged.
* @return the resulting list of rearranged tokens. * @return the resulting list of rearranged tokens.
*/ */
public List<Match<TokenType>> intoPostfix(String source, List<Match<TokenType>> from){ public List<Match<TokenType>> intoPostfix(List<Match<TokenType>> from){
ArrayList<Match<TokenType>> output = new ArrayList<>(); ArrayList<Match<TokenType>> output = new ArrayList<>();
Stack<Match<TokenType>> tokenStack = new Stack<>(); Stack<Match<TokenType>> tokenStack = new Stack<>();
while(!from.isEmpty()){ while(!from.isEmpty()){
@ -104,10 +103,10 @@ public class TreeBuilder {
if(matchType == TokenType.NUM) { if(matchType == TokenType.NUM) {
output.add(match); output.add(match);
} else if(matchType == TokenType.FUNCTION) { } else if(matchType == TokenType.FUNCTION) {
output.add(new Match<>(0, 0, TokenType.INTERNAL_FUNCTION_END)); output.add(new Match<>("" , TokenType.INTERNAL_FUNCTION_END));
tokenStack.push(match); tokenStack.push(match);
} else if(matchType == TokenType.OP){ } else if(matchType == TokenType.OP){
String tokenString = source.substring(match.getFrom(), match.getTo()); String tokenString = match.getContent();
OperatorType type = typeMap.get(tokenString); OperatorType type = typeMap.get(tokenString);
int precedence = precedenceMap.get(tokenString); int precedence = precedenceMap.get(tokenString);
OperatorAssociativity associativity = associativityMap.get(tokenString); OperatorAssociativity associativity = associativityMap.get(tokenString);
@ -123,7 +122,7 @@ public class TreeBuilder {
if(!(otherMatchType == TokenType.OP || otherMatchType == TokenType.FUNCTION)) break; if(!(otherMatchType == TokenType.OP || otherMatchType == TokenType.FUNCTION)) break;
if(otherMatchType == TokenType.OP){ if(otherMatchType == TokenType.OP){
int otherPrecedence = precedenceMap.get(source.substring(otherMatch.getFrom(), otherMatch.getTo())); int otherPrecedence = precedenceMap.get(match.getContent());
if(otherPrecedence < precedence || if(otherPrecedence < precedence ||
(associativity == OperatorAssociativity.RIGHT && otherPrecedence == precedence)) { (associativity == OperatorAssociativity.RIGHT && otherPrecedence == precedence)) {
break; break;
@ -155,34 +154,33 @@ public class TreeBuilder {
/** /**
* Constructs a tree recursively from a list of tokens. * Constructs a tree recursively from a list of tokens.
* @param source the source string.
* @param matches the list of tokens from the source string. * @param matches the list of tokens from the source string.
* @return the construct tree expression. * @return the construct tree expression.
*/ */
public TreeNode fromStringRecursive(String source, List<Match<TokenType>> matches){ public TreeNode fromStringRecursive(List<Match<TokenType>> matches){
if(matches.size() == 0) return null; if(matches.size() == 0) return null;
Match<TokenType> match = matches.remove(0); Match<TokenType> match = matches.remove(0);
TokenType matchType = match.getType(); TokenType matchType = match.getType();
if(matchType == TokenType.OP){ if(matchType == TokenType.OP){
String operator = source.substring(match.getFrom(), match.getTo()); String operator = match.getContent();
OperatorType type = typeMap.get(operator); OperatorType type = typeMap.get(operator);
if(type == OperatorType.BINARY_INFIX){ if(type == OperatorType.BINARY_INFIX){
TreeNode right = fromStringRecursive(source, matches); TreeNode right = fromStringRecursive(matches);
TreeNode left = fromStringRecursive(source, matches); TreeNode left = fromStringRecursive(matches);
if(left == null || right == null) return null; if(left == null || right == null) return null;
else return new BinaryInfixNode(operator, left, right); else return new BinaryInfixNode(operator, left, right);
} else { } else {
TreeNode applyTo = fromStringRecursive(source, matches); TreeNode applyTo = fromStringRecursive(matches);
if(applyTo == null) return null; if(applyTo == null) return null;
else return new UnaryPrefixNode(operator, applyTo); else return new UnaryPrefixNode(operator, applyTo);
} }
} else if(matchType == TokenType.NUM){ } else if(matchType == TokenType.NUM){
return new NumberNode(abacus.numberFromString(source.substring(match.getFrom(), match.getTo()))); return new NumberNode(abacus.numberFromString(match.getContent()));
} else if(matchType == TokenType.FUNCTION){ } else if(matchType == TokenType.FUNCTION){
String functionName = source.substring(match.getFrom(), match.getTo()); String functionName = match.getContent();
FunctionNode node = new FunctionNode(functionName); FunctionNode node = new FunctionNode(functionName);
while(!matches.isEmpty() && matches.get(0).getType() != TokenType.INTERNAL_FUNCTION_END){ while(!matches.isEmpty() && matches.get(0).getType() != TokenType.INTERNAL_FUNCTION_END){
TreeNode argument = fromStringRecursive(source, matches); TreeNode argument = fromStringRecursive(matches);
if(argument == null) return null; if(argument == null) return null;
node.prependChild(argument); node.prependChild(argument);
} }
@ -202,11 +200,11 @@ public class TreeBuilder {
List<Match<TokenType>> matches = tokenize(string); List<Match<TokenType>> matches = tokenize(string);
if(matches == null) return null; if(matches == null) return null;
matches.removeIf(m -> m.getType() == TokenType.WHITESPACE); matches.removeIf(m -> m.getType() == TokenType.WHITESPACE);
matches = intoPostfix(string, matches); matches = intoPostfix(matches);
if(matches == null) return null; if(matches == null) return null;
Collections.reverse(matches); Collections.reverse(matches);
return fromStringRecursive(string, matches); return fromStringRecursive(matches);
} }
} }