From 40858cee415643769ee5f6216b0cd4335996ff2f Mon Sep 17 00:00:00 2001
From: bjculkin <bjculkin@mix.wvu.edu>
Date: Tue, 11 Apr 2017 17:51:13 -0400
Subject: General cleanup and fixes

---
 dice-lang/src/bjc/dicelang/expr/Lexer.java | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

(limited to 'dice-lang/src/bjc/dicelang/expr/Lexer.java')

diff --git a/dice-lang/src/bjc/dicelang/expr/Lexer.java b/dice-lang/src/bjc/dicelang/expr/Lexer.java
index 7c15884..3147463 100644
--- a/dice-lang/src/bjc/dicelang/expr/Lexer.java
+++ b/dice-lang/src/bjc/dicelang/expr/Lexer.java
@@ -3,7 +3,8 @@ package bjc.dicelang.expr;
 import java.util.LinkedList;
 import java.util.List;
 
-import bjc.utils.parserutils.splitter.SimpleTokenSplitter;
+import bjc.utils.funcdata.IList;
+import bjc.utils.parserutils.splitter.ConfigurableTokenSplitter;
 
 /**
  * Implements the lexer for simple expression operations.
@@ -12,18 +13,18 @@ import bjc.utils.parserutils.splitter.SimpleTokenSplitter;
  */
 public class Lexer {
 	/*
-	 * Spliter we use
+	 * Splitter we use.
 	 */
-	private SimpleTokenSplitter split;
+	private ConfigurableTokenSplitter split;
 
 	/**
 	 * Create a new expression lexer.
 	 */
 	public Lexer() {
-		split = new SimpleTokenSplitter();
+		split = new ConfigurableTokenSplitter(true);
 
-		split.addDelimiter("(", ")");
-		split.addDelimiter("+", "-", "*", "/");
+		split.addSimpleDelimiters("(", ")");
+		split.addSimpleDelimiters("+", "-", "*", "/");
 	}
 
 	/**
@@ -31,8 +32,9 @@ public class Lexer {
 	 *
 	 * @param inp
 	 *                The input command.
+	 * 
 	 * @param tks
-	 *                The token state
+	 *                The token state.
 	 *
 	 * @return A series of infix tokens representing the command.
 	 */
@@ -42,11 +44,10 @@ public class Lexer {
 		List<Token> tokens = new LinkedList<>();
 
 		for(String spacedToken : spacedTokens) {
-			String[] rawTokens = split.split(spacedToken);
+			IList<String> splitTokens = split.split(spacedToken);
+			IList<Token> rawTokens = splitTokens.map(tok -> tks.lexToken(tok, spacedToken));
 
-			for(String tok : rawTokens) {
-				tokens.add(tks.lexToken(tok, spacedToken));
-			}
+			rawTokens.forEach(tokens::add);
 		}
 
 		return tokens.toArray(new Token[0]);
-- 
cgit v1.2.3