From ac19c7b23091ad064d0347d338e5600a379432d5 Mon Sep 17 00:00:00 2001
From: Danila Fedorin <danila.fedorin@gmail.com>
Date: Fri, 25 Aug 2017 01:03:12 -0700
Subject: [PATCH] Change lexer tokenizer to recognize tree value functions.

---
 .../main/java/org/nwapw/abacus/parsing/LexerTokenizer.java  | 6 ++++++
 core/src/main/java/org/nwapw/abacus/tree/TokenType.java     | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/nwapw/abacus/parsing/LexerTokenizer.java b/core/src/main/java/org/nwapw/abacus/parsing/LexerTokenizer.java
index b7bdd1f..14ecf95 100644
--- a/core/src/main/java/org/nwapw/abacus/parsing/LexerTokenizer.java
+++ b/core/src/main/java/org/nwapw/abacus/parsing/LexerTokenizer.java
@@ -53,6 +53,9 @@ public class LexerTokenizer implements Tokenizer<Match<TokenType>>, PluginListen
         for (String function : manager.getAllFunctions()) {
             lexer.register(Pattern.sanitize(function), TokenType.FUNCTION);
         }
+        for (String function : manager.getAllTreeValueFunctions()){
+            lexer.register(Pattern.sanitize(function), TokenType.TREE_VALUE_FUNCTION);
+        }
     }
 
     @Override
@@ -63,6 +66,9 @@ public class LexerTokenizer implements Tokenizer<Match<TokenType>>, PluginListen
         for (String function : manager.getAllFunctions()) {
             lexer.unregister(Pattern.sanitize(function), TokenType.FUNCTION);
         }
+        for (String function : manager.getAllTreeValueFunctions()){
+            lexer.unregister(Pattern.sanitize(function), TokenType.TREE_VALUE_FUNCTION);
+        }
     }
 
 }
diff --git a/core/src/main/java/org/nwapw/abacus/tree/TokenType.java b/core/src/main/java/org/nwapw/abacus/tree/TokenType.java
index aa1957a..857e9b4 100644
--- a/core/src/main/java/org/nwapw/abacus/tree/TokenType.java
+++ b/core/src/main/java/org/nwapw/abacus/tree/TokenType.java
@@ -7,7 +7,7 @@ package org.nwapw.abacus.tree;
 public enum TokenType {
 
     INTERNAL_FUNCTION_END(-1),
-    ANY(0), WHITESPACE(1), COMMA(2), OP(3), NUM(4), VARIABLE(5), FUNCTION(6), OPEN_PARENTH(7), CLOSE_PARENTH(8);
+    ANY(0), WHITESPACE(1), COMMA(2), OP(3), NUM(4), VARIABLE(5), FUNCTION(6), TREE_VALUE_FUNCTION(6), OPEN_PARENTH(7), CLOSE_PARENTH(8);
 
     /**
      * The priority by which this token gets sorted.