Lexer/shunter work

5 files changed, 344 insertions, 55 deletions

diff --git a/dice-lang/src/bjc/dicelang/v2/DiceBox.java b/dice-lang/src/bjc/dicelang/v2/DiceBox.java
index ac452e4..164530c 100644
--- a/dice-lang/src/bjc/dicelang/v2/DiceBox.java
+++ b/dice-lang/src/bjc/dicelang/v2/DiceBox.java
@@ -1,5 +1,6 @@
 package bjc.dicelang.v2;
 
+import java.util.Arrays;
 import java.util.Random;
 import java.util.regex.Pattern;
 
@@ -13,6 +14,42 @@ public class DiceBox {
 		int roll();
 	}
 
+	public interface DieList {
+		boolean canOptimize();
+		int[] optimize();
+
+		int[] roll();
+	}
+
+	public static class DieExpression {
+		public final boolean isList;
+
+		public Die     scalar;
+		public DieList list;
+
+		public DieExpression(Die scal) {
+			isList = false;
+
+			scalar = scal;
+		}
+
+		public DieExpression(DieList lst) {
+			isList = true;
+
+			list   = lst;
+		}
+
+		public String toString() {
+			if(isList) return list.toString();
+			else       return scalar.toString();
+		}
+
+		public String value() {
+			if(isList) return Arrays.toString(list.roll());
+			else       return Integer.toString(scalar.roll());
+		}
+	}
+
 	private static class ScalarDie implements Die {
 		private int val;
 
@@ -59,7 +96,7 @@ public class DiceBox {
 			int total = 0;
 
 			for(int i = 0; i < numDice; i++) {
-				total += rng.nextInt(i) + 1;
+				total += rng.nextInt(diceSize) + 1;
 			}
 
 			return total;
@@ -84,42 +121,120 @@ public class DiceBox {
 		}
 
 		public int optimize() {
-			return left.optimize() + "" + right.optimize();
+			return Integer.parseInt(left.optimize() + "" + right.optimize());
 		}
 
 		public int roll() {
 			return Integer.parseInt(left.roll() + "" + right.roll());
 		}
+
+		public String toString() {
+			return left.toString() + "c" + right.toString();
+		}
+	}
+
+	private static class SimpleDieList implements DieList {
+		private Die numDice;
+		private Die size;
+
+		public SimpleDieList(Die nDice, Die sze) {
+			numDice = nDice;
+			size = sze;
+		}
+
+		public boolean canOptimize() {
+			if(size.canOptimize() && size.optimize() == 1) {
+				return numDice.canOptimize();
+			} else {
+				return false;
+			}
+		}
+
+		public int[] optimize() {
+			int[] ret = new int[numDice.optimize()];
+
+			int optSize = size.optimize();
+
+			for(int i = 0; i < optSize; i++) {
+				ret[i] = 1;
+			}
+
+			return ret;
+		}
+
+		public int[] roll() {
+			int num = numDice.roll();
+
+			int[] ret = new int[num];
+
+			for(int i = 0; i < num; i++) {
+				ret[i] = size.roll();
+			}
+
+			return ret;
+		}
+
+		public String toString() {
+			return numDice.toString() + "dl" + size.toString();
+		}
 	}
 
-	public static Die parseExpression(String exp) {
+	public static DieExpression parseExpression(String exp) {
 		if(!isValidExpression(exp)) return null;
 
 		if(scalarDiePattern.matcher(exp).matches()) {
-			return new ScalarDie(Integer.parseInt(exp));
+			return new DieExpression(new ScalarDie(Integer.parseInt(exp)));
 		} else if(simpleDiePattern.matcher(exp).matches()) {
 			String[] dieParts = exp.split("d");
 
 			if(dieParts[0].equals("")) {
-				return new SimpleDie(1, Integer.parseInt(dieParts[1]));
+				return new DieExpression(new SimpleDie(1, Integer.parseInt(dieParts[1])));
 			} else {
-				return new SimpleDie(Integer.parseInt(dieParts[0]), Integer.parseInt(dieParts[1]));
+				return new DieExpression(new SimpleDie(Integer.parseInt(dieParts[0]), Integer.parseInt(dieParts[1])));
 			}
 		} else if(compoundDiePattern.matcher(exp).matches()) {
 			String[] dieParts = exp.split("c");
 
-			return new CompoundDie(parseExpression(dieParts[0]), parseExpression(dieParts[1]));
+			DieExpression left  = parseExpression(dieParts[0]);
+			DieExpression right = parseExpression(dieParts[1]);
+
+			if(left.isList || right.isList) {
+				// @TODO give a specific error message
+				return null;
+			}
+
+			return new DieExpression(new CompoundDie(left.scalar, right.scalar));
+		} else if(diceListPattern.matcher(exp).matches()) {
+			String[] dieParts = exp.split("dl");
+
+			DieExpression left  = parseExpression(dieParts[0]);
+			DieExpression right = parseExpression(dieParts[1]);
+
+			if(left.isList || right.isList) {
+				return null;
+			}
+
+			return new DieExpression(new SimpleDieList(left.scalar, right.scalar));
 		}
 
+		// @TODO give a specific error message
 		return null;
 	}
 
-	private static final Pattern scalarDiePattern   = Pattern.compile(
-			"[\\+\\-]?\\d+");
-	private static final Pattern simpleDiePattern   = Pattern.compile(
-			"(?:\\d+)?d\\d+");
-	private static final Pattern compoundDiePattern = Pattern.compile(
-			simpleDiePattern + "c(?:(?:" + simpleDiePattern + ")|(?:\\d+))";
+	private static final String  scalarDie          = "[\\+\\-]?\\d+";
+	private static final Pattern scalarDiePattern   = Pattern.compile("\\A" + scalarDie + "\\Z");
+
+	private static final String  simpleDie          = "(?:\\d+)?d\\d+";
+	private static final Pattern simpleDiePattern   = Pattern.compile("\\A" + simpleDie + "\\Z");
+
+	private static final String  compoundDie        = simpleDie + "c(?:(?:" + simpleDie + ")|(?:\\d+))";
+	private static final Pattern compoundDiePattern = Pattern.compile("\\A" + compoundDie + "\\Z");
+
+	private static final String  compoundGroup      = "(?:(?:" + scalarDie + ")|(?:" + simpleDie + ")|(?:"
+		+ compoundDie + "))";
+
+	private static final String  diceList           = compoundGroup + "dl" + compoundGroup;
+	private static final Pattern diceListPattern    = Pattern.compile("\\A" + diceList + "\\Z");
 
 	public static boolean isValidExpression(String exp) {
 		if(scalarDiePattern.matcher(exp).matches()) {
@@ -128,6 +243,8 @@ public class DiceBox {
 			return true;
 		} else if (compoundDiePattern.matcher(exp).matches()) {
 			return true;
+		} else if (diceListPattern.matcher(exp).matches()) {
+			return true;
 		} else {
 			return false;
 		}
diff --git a/dice-lang/src/bjc/dicelang/v2/DiceLangEngine.java b/dice-lang/src/bjc/dicelang/v2/DiceLangEngine.java
index d8a43c5..5a4ff26 100644
--- a/dice-lang/src/bjc/dicelang/v2/DiceLangEngine.java
+++ b/dice-lang/src/bjc/dicelang/v2/DiceLangEngine.java
@@ -8,10 +8,12 @@ import bjc.utils.funcdata.FunctionalStringTokenizer;
 import bjc.utils.funcdata.IList;
 import bjc.utils.funcdata.IMap;
 import bjc.utils.funcutils.ListUtils;
+import bjc.utils.funcutils.StringUtils;
 
 import java.util.Arrays;
 import java.util.Deque;
 import java.util.LinkedList;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import static bjc.dicelang.v2.Token.Type.*;
@@ -21,12 +23,19 @@ public class DiceLangEngine {
 	private Deque<IPair<String, String>> opExpansionTokens;
 	private Deque<IPair<String, String>> deaffixationTokens;
 
-	// ID for generation of string literal variables
+	// ID for generation
 	private int nextLiteral;
+	private int nextSym;
 
 	// Debug indicator
 	private boolean debugMode;
 
+	// Shunter for token postfixing
+	private Shunter shunt;
+
+	private IMap<Integer, String> symTable;
+	private IMap<Integer, String> stringLits;
+
 	private final int MATH_PREC = 20;
 	private final int DICE_PREC = 10;
 	private final int EXPR_PREC = 0;
@@ -53,6 +62,8 @@ public class DiceLangEngine {
 
 		// @TODO make configurable
 		debugMode = true;
+
+		shunt = new Shunter();
 	}
 
 	public boolean runCommand(String command) {
@@ -75,34 +86,31 @@ public class DiceLangEngine {
 		if(!success) return success;
 
 		if(debugMode) {
-			System.out.println("\tCommand after destringing: "
-					+ destringed.toString());
+			System.out.println("\tCommand after destringing: " + destringed.toString());
 
 			System.out.println("\tString literals in table");
+
 			stringLiterals.forEach((key, val) -> {
 				System.out.printf("\t\tName: (%s)\tValue: (%s)\n",
 					key, val);
 			});
 		}
 
-		IList<String> semiExpandedTokens = 
-			ListUtils.deAffixTokens(
-					destringed, deaffixationTokens);
+		IList<String> semiExpandedTokens  = ListUtils.deAffixTokens(destringed, deaffixationTokens);
+		IList<String> fullyExpandedTokens = ListUtils.splitTokens(semiExpandedTokens, opExpansionTokens);
 
-		IList<String> fullyExpandedTokens = 
-			ListUtils.splitTokens(
-					semiExpandedTokens, opExpansionTokens);
+		if(debugMode) {
+			System.out.printf("\tCommand after token deaffixation: " 
+					+ semiExpandedTokens.toString() + "\n");
 
-		if(debugMode)
-			System.out.printf("\tCommand after token"
-					+ " expansion: " 
-					+ fullyExpandedTokens.toString()
-					+ "\n");
+			System.out.printf("\tCommand after token expansion: " 
+					+ fullyExpandedTokens.toString() + "\n");
+		}
 
 		IList<Token> lexedTokens = new FunctionalList<>();
 
 		for(String token : fullyExpandedTokens.toIterable()) {
-			Token tk = lexToken(token);
+			Token tk = lexToken(token, stringLiterals);
 
 			if(tk == null) continue;
 			else if(tk == Token.NIL_TOKEN) return false;
@@ -112,10 +120,18 @@ public class DiceLangEngine {
 		if(debugMode)
 			System.out.printf("\tCommand after tokenization: %s\n", lexedTokens.toString());
 
+		IList<Token> shuntedTokens = new FunctionalList<>();
+		success                    = shunt.shuntTokens(lexedTokens, shuntedTokens);
+
+		if(!success) return false;
+
+		if(debugMode)
+			System.out.printf("\tCommand after shunting: %s\n", shuntedTokens.toString());
+
 		return true;
 	}
 
-	private Token lexToken(String token) {
+	private Token lexToken(String token, IMap<String, String> stringLts) {
 		if(token.equals("")) return null;
 
 		Token tk = Token.NIL_TOKEN;
@@ -136,43 +152,92 @@ public class DiceLangEngine {
 			case "//":
 				tk = new Token(IDIVIDE);
 				break;
-			case "(":
-				tk = new Token(OPAREN);
+			case "dg":
+				tk = new Token(DICEGROUP);
 				break;
-			case ")":
-				tk = new Token(CPAREN);
+			case "dc":
+				tk = new Token(DICECONCAT);
 				break;
-			case "[":
-				tk = new Token(OBRACKET);
+			case "dl":
+				tk = new Token(DICELIST);
+				break;
+			case "=>":
+				tk = new Token(LET);
 				break;
+			case ":=":
+			   tk = new Token(BIND);
+			   break;
+			case "(":
+			case ")":
+			case "[":
 			case "]":
-				tk = new Token(CBRACKET);
+				tk = tokenizeGrouping(token);
 				break;
 			default:
+				tk = tokenizeLiteral(token, stringLts);
+		}
+
+		return tk;
+	}
+
+	private Token tokenizeGrouping(String token) {
+		Token tk = Token.NIL_TOKEN;
 
-				tk = tokenizeLiteral(token);
+		if(StringUtils.containsOnly(token, "\\" + token.charAt(0))) {
+			switch(token) {
+				case "(":
+					tk = new Token(OPAREN, token.length());
+					break;
+				case ")":
+					tk = new Token(CPAREN, token.length());
+					break;
+				case "[":
+					tk = new Token(OBRACKET, token.length());
+					break;
+				case "]":
+					tk = new Token(CBRACKET, token.length());
+					break;
+			}
 		}
 
 		return tk;
 	}
 
-	private Pattern intMatcher = Pattern.compile(
-			"[\\-\\+]?\\d+");
+	private Pattern intMatcher = Pattern.compile("\\A[\\-\\+]?\\d+\\Z");
+	private Pattern stringLitMatcher = Pattern.compile("\\AstringLiteral\\d+\\Z");
 
-	private Token tokenizeLiteral(String token) {
+	private Token tokenizeLiteral(String token, IMap<String, String> stringLts) {
 		Token tk = Token.NIL_TOKEN;
 
-		if(DoubleMatcher.floatingLiteral.matcher(token).matches()) {
-			tk = new Token(FLOAT_LIT, Double.parseDouble(token));
-		} else if(intMatcher.matcher(token).matches()) {
+		if(intMatcher.matcher(token).matches()) {
 			tk = new Token(INT_LIT, Integer.parseInt(token));
+		} else if(DoubleMatcher.floatingLiteral.matcher(token).matches()) {
+			tk = new Token(FLOAT_LIT, Double.parseDouble(token));
 		} else if(DiceBox.isValidExpression(token)) {
 			tk = new Token(DICE_LIT, DiceBox.parseExpression(token));
+
+			if(debugMode)
+				System.out.println("\tDEBUG: Parsed dice expression"
+						+ ", evaluated as: " 
+						+ tk.diceValue.value());
 		} else {
-			System.out.printf("\tERROR: Unrecognized token:"
-					+ "%s\n", token);
+			Matcher stringLit = stringLitMatcher.matcher(token);
+
+			if(stringLit.matches()) {
+				int litNum = Integer.parseInt(stringLit.group());
+
+				stringLits.put(litNum, stringLts.get(token));
+				tk = new Token(STRING_LIT, litNum);
+			} else {
+				// @TODO define what a valid identifier is
+				symTable.put(nextSym++, token);
+
+				tk = new Token(VREF, nextSym - 1);
+			}
 
-			return tk;
+			// @TODO uncomment when we have a defn. for var names
+			// System.out.printf("\tERROR: Unrecognized token:"
+			// 		+ "%s\n", token);
 		}
 
 		return tk;
diff --git a/dice-lang/src/bjc/dicelang/v2/DoubleMatcher.java b/dice-lang/src/bjc/dicelang/v2/DoubleMatcher.java
index 3c6f6c5..5f6f0db 100644
--- a/dice-lang/src/bjc/dicelang/v2/DoubleMatcher.java
+++ b/dice-lang/src/bjc/dicelang/v2/DoubleMatcher.java
@@ -53,6 +53,6 @@ public class DoubleMatcher {
 		 "[fFdD]?))" +
 		 "[\\x00-\\x20]*");// Optional trailing "whitespace"
 
-	public static final Pattern floatingLiteral = Pattern.compile(fpRegex);
+	public static final Pattern floatingLiteral = Pattern.compile("\\A" + fpRegex + "\\Z");
 
 }
diff --git a/dice-lang/src/bjc/dicelang/v2/Shunter.java b/dice-lang/src/bjc/dicelang/v2/Shunter.java
index bca08eb..3edceeb 100644
--- a/dice-lang/src/bjc/dicelang/v2/Shunter.java
+++ b/dice-lang/src/bjc/dicelang/v2/Shunter.java
@@ -5,7 +5,9 @@ import bjc.utils.funcdata.FunctionalMap;
 import bjc.utils.funcdata.IList;
 import bjc.utils.funcdata.IMap;
 
+import java.util.Deque;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.Set;
 
 import static bjc.dicelang.v2.Token.Type.*;
@@ -42,18 +44,121 @@ public class Shunter {
 
 		ops.put(DICEGROUP,  0 + DICE_PREC);
 		ops.put(DICECONCAT, 1 + DICE_PREC);
+		ops.put(DICELIST,   2 + DICE_PREC);
 
 		ops.put(LET,  0 + EXPR_PREC);
 		ops.put(BIND, 1 + EXPR_PREC);
 	}
 
-	public IList<Token> shuntTokens(IList<Token> tks) {
-		IList<Token> returned = new FunctionalList<>();
+	public boolean shuntTokens(IList<Token> tks, IList<Token> returned) {
+		Deque<Token> opStack    = new LinkedList<>();
+
+		boolean unaryMode          = false;
+		Deque<Token> unaryOps = new LinkedList<>();
 
 		for(Token tk : tks.toIterable()) {
+			if(unaryMode) {
+				// @TODO finish unary mode
+				if(unaryAdjectives.contains(tk.type) || unaryAdverbs.contains(tk.type)) {
+					unaryOps.push(tk);
+					continue;
+				}
+
+				Token currOperator = unaryOps.peek();
+
+				if(unaryAdjectives.contains(currOperator.type)) {
+					boolean isOp = ops.containsKey(tk.type) 
+						|| unaryAdverbs.contains(tk.type)
+						|| unaryAdjectives.contains(tk.type);
+
+					if(isOp) {
+						System.out.printf("\tError: Unary operator %s is an"
+								+ " adjective, not an adverb (can't be applied"
+								+ " to operator %s)\n", currOperator, tk );
+						
+						return false;
+					}
+
+					returned.add(tk);
+					returned.add(unaryOps.pop());
+				} else if (unaryAdverbs.contains(currOperator.type)) {
+					if(opStack.size() < 1) {
+						System.out.printf("\tError: Unary operators %s is an adverb,"
+								+ " but there is no operator to apply it to\n");
+					}
+
+					Token currOperand = opStack.peek();
+
+					boolean isOp = ops.containsKey(currOperand.type) 
+						|| unaryAdverbs.contains(currOperand.type)
+						|| unaryAdjectives.contains(currOperand.type);
+
+					if(!isOp) {
+						System.out.printf("\tError: Unary operator %s is an adverb,"
+								+ " not an  adjective (can't be applied to operand %s)\n",
+								currOperator, tk);
+						
+						return false;
+					}
+
+					returned.add(tk);
+					returned.add(unaryOps.pop());
+				}
+
+				if(unaryOps.isEmpty()) unaryMode = false;
+			} else {
+				if(unaryAdjectives.contains(tk.type) || unaryAdverbs.contains(tk.type)) {
+					unaryMode = true;
+
+					unaryOps.add(tk);
+					continue;
+				} else if(ops.containsKey(tk.type)) {
+					while(!opStack.isEmpty() 
+							&& isHigherPrec(tk.type, opStack.peek().type)) {
+						returned.add(opStack.pop());
+					}
+
+					opStack.push(tk);
+				} else if(tk.type == OPAREN) {
+					opStack.push(tk);
+				} else if(tk.type == CPAREN) {
+					Token currTk = opStack.peek();
+
+					while(currTk.type != OPAREN && currTk.intValue != tk.intValue) {
+						if(opStack.isEmpty()) {
+							System.out.printf("\tError: Could not find matching parenthesis"
+								+ " with matching level %d", tk.intValue);
+
+							return false;
+						}
+
+						returned.add(opStack.pop());
+					}
+				} else {
+					returned.add(tk);
+				}
+			}
+		}
+
+		// Flush leftover operators
+		while(!opStack.isEmpty()) {
+			returned.add(opStack.pop());
+		}
 
+		return true;
+	}
+
+	private boolean isHigherPrec(Token.Type left, Token.Type right) {
+		boolean exists = ops.containsKey(right);
+
+		// If it doesn't, the left is higher precedence.
+		if (!exists) {
+			return false;
 		}
 
-		return returned;
+		int rightPrecedence = ops.get(right);
+		int leftPrecedence  = ops.get(left);
+
+		return rightPrecedence >= leftPrecedence;
 	}
 }
diff --git a/dice-lang/src/bjc/dicelang/v2/Token.java b/dice-lang/src/bjc/dicelang/v2/Token.java
index 471832e..e5fce00 100644
--- a/dice-lang/src/bjc/dicelang/v2/Token.java
+++ b/dice-lang/src/bjc/dicelang/v2/Token.java
@@ -1,7 +1,5 @@
 package bjc.dicelang.v2;
 
-import bjc.dicelang.IDiceExpression;
-
 /**
  * Lexer token
  */
@@ -17,7 +15,7 @@ public class Token {
 		DIVIDE, IDIVIDE,
 		INT_LIT, FLOAT_LIT, STRING_LIT,
 		VREF,
-		DICE_LIT, DICEGROUP, DICECONCAT,
+		DICE_LIT, DICEGROUP, DICECONCAT, DICELIST,
 		LET, BIND,
 		OPAREN, CPAREN,
 		OBRACKET, CBRACKET,
@@ -30,7 +28,7 @@ public class Token {
 	// based on the token type
 	public int intValue;
 	public double floatValue;
-	public DiceBox.Die diceValue;
+	public DiceBox.DieExpression diceValue;
 
 	public Token(Type typ) {
 		type = typ;
@@ -48,7 +46,7 @@ public class Token {
 		floatValue = val;
 	}
 
-	public Token(Type typ, DiceBox.Die val) {
+	public Token(Type typ, DiceBox.DieExpression val) {
 		this(typ);
 
 		diceValue = val;
@@ -59,6 +57,10 @@ public class Token {
 			case INT_LIT:
 			case STRING_LIT:
 			case VREF:
+			case OPAREN:
+			case CPAREN:
+			case OBRACKET:
+			case CBRACKET:
 				return type.toString() + "("
 					+ intValue + ")";
 			case FLOAT_LIT: