package org.nwapw.abacus.lexing; import org.nwapw.abacus.lexing.pattern.EndNode; import org.nwapw.abacus.lexing.pattern.Match; import org.nwapw.abacus.lexing.pattern.Pattern; import org.nwapw.abacus.lexing.pattern.PatternNode; import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; /** * A lexer that can generate tokens of a given type given a list of regular expressions * to operate on. * @param the type used to identify which match belongs to which pattern. */ public class Lexer { /** * The registered patterns. */ private ArrayList> patterns; /** * Creates a new lexer with no registered patterns. */ public Lexer(){ patterns = new ArrayList<>(); } /** * Registers a single pattern. * @param pattern the pattern regex * @param id the ID by which to identify the pattern. */ public void register(String pattern, T id){ Pattern compiledPattern = new Pattern<>(pattern, id); if(compiledPattern.getHead() != null) patterns.add(compiledPattern); } /** * Reads one token from the given string. * @param from the string to read from * @param startAt the index to start at * @param compare the comparator used to sort tokens by their ID. * @return the best match. */ public Match lexOne(String from, int startAt, Comparator compare){ ArrayList> matches = new ArrayList<>(); HashSet> currentSet = new HashSet<>(); HashSet> futureSet = new HashSet<>(); int index = startAt; for(Pattern pattern : patterns){ pattern.getHead().addInto(currentSet); } while(!currentSet.isEmpty()){ for(PatternNode node : currentSet){ if(index < from.length() && node.matches(from.charAt(index))) { node.addOutputsInto(futureSet); } else if(node instanceof EndNode){ matches.add(new Match<>(startAt, index, ((EndNode) node).getPatternId())); } } HashSet> tmp = currentSet; currentSet = futureSet; futureSet = tmp; futureSet.clear(); index++; } matches.sort((a, b) -> compare.compare(a.getType(), b.getType())); if(compare != null) { matches.sort(Comparator.comparingInt(a -> a.getTo() - a.getFrom())); } return matches.isEmpty() ? null : matches.get(matches.size() - 1); } /** * Reads all tokens from a string. * @param from the string to start from. * @param startAt the index to start at. * @param compare the comparator used to sort matches by their IDs. * @return the resulting list of matches, in order, or null on error. */ public ArrayList> lexAll(String from, int startAt, Comparator compare){ int index = startAt; ArrayList> matches = new ArrayList<>(); Match lastMatch = null; while(index < from.length() && (lastMatch = lexOne(from, index, compare)) != null){ if(lastMatch.getTo() == lastMatch.getFrom()) return null; matches.add(lastMatch); index += lastMatch.getTo() - lastMatch.getFrom(); } if(lastMatch == null) return null; return matches; } }