Implement lexing code.

include/lexer.h

@@ -17,6 +17,35 @@ struct lexer {
     eval_config config;
 };
 
+/**
+ * A token that is produced by the lexer.
+ */
+struct lexer_match {
+    /**
+     * The line that this token was found on.
+     */
+    size_t line;
+    /**
+     * The first index at which this token's string
+     * begins.
+     */
+    size_t from;
+    /**
+     * The index of the first character that is outside
+     * this token.
+     */
+    size_t to;
+    /**
+     * The index of the beginning of the line on which
+     * this token is found.
+     */
+    size_t line_from;
+    /**
+     * The type of token.
+     */
+    int type;
+};
+
 /**
  * The various tokens used by the lexer
  * in order to tag meaningful sequences
@@ -45,6 +74,7 @@ enum lexer_token {
 
 typedef struct lexer lexer;
 typedef enum lexer_token lexer_token;
+typedef struct lexer_match lexer_match;
 
 /**
  * Initializes the given lexer,
@@ -53,6 +83,14 @@ typedef enum lexer_token lexer_token;
  * @return the result of the operation (can be MALLOC on failed allocation.)
  */
 libab_result lexer_init(lexer* lexer);
+/**
+ * Turns the given input string into tokens.
+ * @param lexer the lexer to use to turn the string into tokens.
+ * @param string the string to turn into tokens.
+ * @param lex_into the list which should be populated with matches.
+ * @return the result of the operation.
+ */
+libab_result lexer_lex(lexer* lexer, const char* string, ll* lext_into);
 /**
  * Releases the memory associated with the given lexer,
  * removing all registered patterns from it.
@@ -60,5 +98,11 @@ libab_result lexer_init(lexer* lexer);
  * @return the result of the operation.
  */
 libab_result lexer_free(lexer* lexer);
+/**
+ * Function intended to be passed to "foreach" calls
+ * in libds. lexer_lex allocates matches, and passing this function
+ * to foreach will free the memory allocated for the matches.
+ */
+int lexer_foreach_match_free(void* data, va_list args);
 
 #endif