More tweaker work.

6 files changed, 208 insertions, 87 deletions

diff --git a/dice-lang/src/bjc/dicelang/CompilerTweaker.java b/dice-lang/src/bjc/dicelang/CompilerTweaker.java
index ffbfb7e..3779f2b 100644
--- a/dice-lang/src/bjc/dicelang/CompilerTweaker.java
+++ b/dice-lang/src/bjc/dicelang/CompilerTweaker.java
@@ -1,5 +1,7 @@
 package bjc.dicelang;

 

+import bjc.utils.funcutils.NeoTokenSplitter;

+

 /**

  * Contains methods for customizing the DiceLang and SCL compilers.

  * 

@@ -7,6 +9,77 @@ package bjc.dicelang;
  */

 public class CompilerTweaker {

 	/*

-	 * @TODO add things

+	 * Bits of the compiler necessary

+	 */

+	private DiceLangEngine eng;

+	private NeoTokenSplitter opExpander;

+

+	public CompilerTweaker(DiceLangEngine eng) {

+		this.eng = eng;

+

+		this.opExpander = eng.opExpander;

+	}

+

+	/**

+	 * Add a string literal to the compiler's internal banks.

+	 * 

+	 * @param val

+	 *                The string literal to add.

+	 * 

+	 * @return The key into the string literal table for this string.

 	 */

+	public int addStringLiteral(String val) {

+		eng.addStringLiteral(eng.nextLiteral, val);

+

+		eng.nextLiteral += 1;

+		return eng.nextLiteral;

+	}

+

+	/**

+	 * Add a line defn to the compiler.

+	 * 

+	 * @param dfn

+	 *                The defn to add.

+	 */

+	public void addLineDefine(Define dfn) {

+		eng.addLineDefine(dfn);

+	}

+

+	/**

+	 * Add a token defn to the compiler.

+	 * 

+	 * @param dfn

+	 *                The defn to add.

+	 */

+	public void addTokenDefine(Define dfn) {

+		eng.addTokenDefine(dfn);

+	}

+

+	/**

+	 * Adds a delimiter that is expanded from tokens.

+	 * 

+	 * @param delim

+	 *                The delimiter to expand on.

+	 */

+	public void addDelimiter(String delim) {

+		opExpander.addDelimiter(delim);

+	}

+

+	/**

+	 * Adds a multi-character delimiter that is expanded from tokens.

+	 * 

+	 * @param delim

+	 *                The multi-character delimiter to expand on.

+	 */

+	public void addMultiDelimiter(String delim) {

+		opExpander.addMultiDelimiter(delim);

+	}

+

+	/**

+	 * Make delimiter changes visible to the compiler.

+	 */

+	public void compile() {

+		opExpander.compile();

+	}

+

 }

diff --git a/dice-lang/src/bjc/dicelang/DiceLangEngine.java b/dice-lang/src/bjc/dicelang/DiceLangEngine.java
index 7b21aed..d52e242 100644
--- a/dice-lang/src/bjc/dicelang/DiceLangEngine.java
+++ b/dice-lang/src/bjc/dicelang/DiceLangEngine.java
@@ -40,7 +40,7 @@ public class DiceLangEngine {
 	/*
 	 * ID for generation.
 	 */
-	private int nextLiteral;
+	int nextLiteral;
 
 	/*
 	 * Debug indicator.
@@ -74,13 +74,14 @@ public class DiceLangEngine {
 	/*
 	 * Evaluator for evaluating.
 	 */
-	private Evaluator eval;
+	Evaluator eval;
 
 	/*
 	 * Tables for various things.
 	 */
 	public final IMap<Integer, String> symTable;
-	public final IMap<Integer, String> stringLits;
+	
+	private IMap<Integer, String> stringLits;
 	private IMap<String, String> stringLiterals;
 
 	/*
@@ -97,7 +98,7 @@ public class DiceLangEngine {
 	/*
 	 * Stream engine for processing streams.
 	 */
-	private StreamEngine streamEng;
+	StreamEngine streamEng;
 
 	public DiceLangEngine() {
 		/*
@@ -715,4 +716,12 @@ public class DiceLangEngine {
 		
 		return true;
 	}
+	
+	String getStringLiteral(int key) {
+		return stringLits.get(key);
+	}
+	
+	void addStringLiteral(int key, String val) {
+		stringLits.put(key, val);
+	}
 }
\ No newline at end of file
diff --git a/dice-lang/src/bjc/dicelang/Errors.java b/dice-lang/src/bjc/dicelang/Errors.java
index e066292..053ad9f 100644
--- a/dice-lang/src/bjc/dicelang/Errors.java
+++ b/dice-lang/src/bjc/dicelang/Errors.java
@@ -89,6 +89,8 @@ public class Errors {
 		EK_SHUNT_NOGROUP,
 		// No group for group seperator to attach to
 		EK_SHUNT_INVSEP,
+		// Attempted to chain non-associative operator
+		EK_SHUNT_NOTASSOC,
 
 		// Stream Errors
 		// Attempted to switch to a non-existant stream
@@ -261,6 +263,9 @@ public class Errors {
 			System.out.printf("\tERROR: Couldn't find matching grouping %s (expected %s)\n",
 					args[0], args[1]);
 			break;
+		case EK_SHUNT_NOTASSOC:
+			System.out.printf("\tERROR: Attempted to chain non-associative operator %s\n", args[0]);
+			break;
 		case EK_SHUNT_INVSEP:
 			System.out.printf("\tERROR: Couldn't find grouper for group seperator to attach to\n");
 			break;
diff --git a/dice-lang/src/bjc/dicelang/Evaluator.java b/dice-lang/src/bjc/dicelang/Evaluator.java
index bb5c345..286af5c 100644
--- a/dice-lang/src/bjc/dicelang/Evaluator.java
+++ b/dice-lang/src/bjc/dicelang/Evaluator.java
@@ -478,7 +478,7 @@ public class Evaluator {
 			res = new EvaluatorResult(DICE, tk.diceValue);
 			break;
 		case STRING_LIT:
-			res = new EvaluatorResult(STRING, eng.stringLits.get((int)(tk.intValue)));
+			res = new EvaluatorResult(STRING, eng.getStringLiteral((int)(tk.intValue)));
 			break;
 		default:
 			Errors.inst.printError(EK_EVAL_UNTOK, tk.type.toString());
diff --git a/dice-lang/src/bjc/dicelang/Shunter.java b/dice-lang/src/bjc/dicelang/Shunter.java
index 28ae117..900f72c 100644
--- a/dice-lang/src/bjc/dicelang/Shunter.java
+++ b/dice-lang/src/bjc/dicelang/Shunter.java
@@ -16,27 +16,40 @@ import java.util.Set;
 public class Shunter {
 	// The binary operators and their
 	// priorities
-	private IMap<Token.Type, Integer> ops;
+	IMap<Token.Type, Integer> ops;
 
+	/*
+	 * Operators that are right-associative
+	 */
+	Set<Token.Type> rightAssoc;
+	
+	/*
+	 * Operators that aren't associative
+	 */
+	Set<Token.Type> notAssoc;
+	
 	// Unary operators that can only be
 	// applied to non-operator tokens and yield operator tokens
-	private Set<Token.Type> unaryAdjectives;
+	Set<Token.Type> unaryAdjectives;
 	
 	// Unary operators that can only be
 	// applied to operator tokens and yield operator tokens
-	private Set<Token.Type> unaryAdverbs;
+	Set<Token.Type> unaryAdverbs;
 
 	// Unary operators that can only be
 	// applied to operator tokens and yield data tokens
-	private Set<Token.Type> unaryGerunds;
+	Set<Token.Type> unaryGerunds;
 
-	private final int MATH_PREC	 = 30;
-	private final int DICE_PREC	 = 20;
-	private final int STR_PREC       = 10;
-	private final int EXPR_PREC	 = 0;
+	public final int MATH_PREC = 30;
+	public final int DICE_PREC = 20;
+	public final int STR_PREC  = 10;
+	public final int EXPR_PREC = 0;
 
 	public Shunter() {
 		ops = new FunctionalMap<>();
+		
+		rightAssoc = new HashSet<>();
+		notAssoc   = new HashSet<>();
 
 		unaryAdjectives = new HashSet<>();
 		unaryAdverbs    = new HashSet<>();
@@ -60,30 +73,47 @@ public class Shunter {
 		ops.put(STRCAT,     0 + STR_PREC);
 		
 		ops.put(STRREP,     1 + STR_PREC);
+		
 		ops.put(LET,        0 + EXPR_PREC);
 		ops.put(BIND,       1 + EXPR_PREC);
 	}
 
-	private boolean isUnary(Token tk) {
-		Token.Type ty = tk.type;
+	public boolean shuntTokens(IList<Token> tks, IList<Token> returned) {
+		Deque<Token> opStack  = new LinkedList<>();
+		Deque<Token> unaryOps = new LinkedList<>();
 
-		if(unaryAdjectives.contains(ty)) return true;
-		if(unaryAdverbs.contains(ty))    return true;
-		if(unaryGerunds.contains(ty))    return true;
+		Deque<Token> currReturned = new LinkedList<>();
 
-		return false;
-	}
+		Deque<Token> feed = new LinkedList<>();
 
-	private boolean isOp(Token tk) {
-		Token.Type ty = tk.type;
+		for(Token tk : tks.toIterable()) {
+			boolean succ;
+			
+			while(feed.size() != 0) {
+				succ = shuntToken(feed.poll(), opStack, unaryOps, currReturned, feed);
+				
+				if(!succ) {
+					return false;
+				}
+			}
+			
+			succ = shuntToken(tk, opStack, unaryOps, currReturned, feed);
+			
+			if(!succ) {
+				return false;
+			}
+		}
 
-		if(ops.containsKey(ty))          return true;
-		if(unaryAdjectives.contains(ty)) return true;
-		if(unaryAdverbs.contains(ty))    return true;
-		if(unaryGerunds.contains(ty))    return true;
-		if(ty == TAGOPR)                 return true;
+		// Flush leftover operators
+		while(!opStack.isEmpty()) {
+			currReturned.addLast(opStack.pop());
+		}
 
-		return false;
+		for(Token tk : currReturned) {
+			returned.add(tk);
+		}
+
+		return true;
 	}
 
 	private boolean shuntToken(Token tk, Deque<Token> opStack,
@@ -94,66 +124,72 @@ public class Shunter {
 				unaryStack.add(tk);
 				return true;
 			}
-
+	
 			Token unaryOp = unaryStack.pop();
-
+	
 			Token.Type unaryType = unaryOp.type;
-
+	
 			if(unaryAdjectives.contains(unaryType)) {
 				if(isOp(tk)) {
 					Errors.inst.printError(EK_SHUNT_NOTADV, unaryOp.toString(), tk.toString());
 					return false;
 				}
-
+	
 				Token newTok = new Token(TAGOPR);
-
+	
 				if(tk.type == TAGOP) {
 					newTok.tokenValues = tk.tokenValues;
 				} else {
 					newTok.tokenValues = new FunctionalList<>(tk);
 				}
-
+	
 				newTok.tokenValues.add(unaryOp);
 				opStack.push(newTok);
-
+	
 				return true;
 			} else if(unaryAdverbs.contains(unaryType)) {
 				if(!isOp(tk)) {
 					Errors.inst.printError(EK_SHUNT_NOTADJ, unaryOp.toString(), tk.toString());
 					return false;
 				}
-
+	
 				Token newTok = new Token(TAGOPR);
-
+	
 				if(tk.type == TAGOP) {
 					newTok.tokenValues = tk.tokenValues;
 				} else {
 					newTok.tokenValues = new FunctionalList<>(tk);
 				}
-
+	
 				newTok.tokenValues.add(unaryOp);
 				opStack.push(newTok);
-
+	
 				return true;
 			}
 		}
-
+	
 		if(isUnary(tk)) {
 			unaryStack.add(tk);
 			return true;
 		} else if(isOp(tk)) {
 			while(!opStack.isEmpty() && isHigherPrec(tk, opStack.peek())) {
-				currReturned.addLast(opStack.pop());
+				Token newOp = opStack.pop();
+				
+				if(tk.type == newOp.type && notAssoc.contains(tk.type)) {
+					Errors.inst.printError(EK_SHUNT_NOTASSOC, tk.type.toString());
+				}
+				
+				currReturned.addLast(newOp);
 			}
-
+	
 			opStack.push(tk);
 		} else if(tk.type == OPAREN || tk.type == OBRACE) {
 			opStack.push(tk);
-
+	
 			if(tk.type == OBRACE) currReturned.addLast(tk);
 		} else if(tk.type == CPAREN || tk.type == CBRACE) {
 			Token matching = null;
-
+	
 			switch(tk.type) {
 				case CPAREN:
 					matching = new Token(OPAREN, tk.intValue);
@@ -164,68 +200,42 @@ public class Shunter {
 				default:
 					break;
 			}
-
+	
 			if(!opStack.contains(matching)) {
 				Errors.inst.printError(EK_SHUNT_NOGROUP, tk.toString(), matching.toString());
 				return false;
 			}
-
+	
 			while(!opStack.peek().equals(matching)) {
 				currReturned.addLast(opStack.pop());
 			}
-
+	
 			if(tk.type == CBRACE) {
 				currReturned.addLast(tk);
 			}
-
+	
 			opStack.pop();
 		} else if(tk.type == GROUPSEP) {
 			IList<Token> group = new FunctionalList<>();
-
+	
 			while(currReturned.size() != 0 && !currReturned.peek().isGrouper()) {
 				group.add(currReturned.pop());
 			}
-
+	
 			while(opStack.size() != 0 && !opStack.peek().isGrouper()) {
 				group.add(opStack.pop());
 			}
-
+	
 			if(currReturned.size() == 0) {
 				Errors.inst.printError(EK_SHUNT_INVSEP);
 				return false;
 			}
-
+	
 			currReturned.addLast(new Token(TOKGROUP, group));
 		} else {
 			currReturned.addLast(tk);
 		}
-
-		return true;
-	}
-
-	public boolean shuntTokens(IList<Token> tks, IList<Token> returned) {
-		Deque<Token> opStack  = new LinkedList<>();
-		Deque<Token> unaryOps = new LinkedList<>();
-
-		Deque<Token> currReturned = new LinkedList<>();
-
-		Deque<Token> feed = new LinkedList<>();
-
-		for(Token tk : tks.toIterable()) {
-			while(feed.size() != 0)
-				shuntToken(feed.poll(), opStack, unaryOps, currReturned, feed);
-			shuntToken(tk, opStack, unaryOps, currReturned, feed);
-		}
-
-		// Flush leftover operators
-		while(!opStack.isEmpty()) {
-			currReturned.addLast(opStack.pop());
-		}
-
-		for(Token tk : currReturned) {
-			returned.add(tk);
-		}
-
+	
 		return true;
 	}
 
@@ -256,7 +266,33 @@ public class Shunter {
 		} else {
 			leftPrecedence  = ops.get(left);
 		}
+		
+		if(rightAssoc.contains(left)) {
+			return rightPrecedence > leftPrecedence;
+		} else {
+			return rightPrecedence >= leftPrecedence;
+		}
+	}
+
+	private boolean isOp(Token tk) {
+		Token.Type ty = tk.type;
+	
+		if(ops.containsKey(ty))          return true;
+		if(unaryAdjectives.contains(ty)) return true;
+		if(unaryAdverbs.contains(ty))    return true;
+		if(unaryGerunds.contains(ty))    return true;
+		if(ty == TAGOPR)                 return true;
+	
+		return false;
+	}
 
-		return rightPrecedence >= leftPrecedence;
+	private boolean isUnary(Token tk) {
+		Token.Type ty = tk.type;
+	
+		if(unaryAdjectives.contains(ty)) return true;
+		if(unaryAdverbs.contains(ty))    return true;
+		if(unaryGerunds.contains(ty))    return true;
+	
+		return false;
 	}
 }
diff --git a/dice-lang/src/bjc/dicelang/Tokenizer.java b/dice-lang/src/bjc/dicelang/Tokenizer.java
index 11bffec..a4b72b9 100644
--- a/dice-lang/src/bjc/dicelang/Tokenizer.java
+++ b/dice-lang/src/bjc/dicelang/Tokenizer.java
@@ -140,18 +140,16 @@ public class Tokenizer {
 			if(stringLit.matches()) {

 				int litNum = Integer.parseInt(stringLit.group(1));

 

-				eng.stringLits.put(litNum, stringLts.get(token));

+				eng.addStringLiteral(litNum, stringLts.get(token));

 				tk = new Token(STRING_LIT, litNum);

 			} else {

-				// @TODO define what a valid identifier is

+				/*

+				 * Everything else is a symbol

+				 */

 				eng.symTable.put(nextSym++, token);

 

 				tk = new Token(VREF, nextSym - 1);

 			}

-

-			// @TODO uncomment when we have a defn. for var names

-			// System.out.printf("\tERROR: Unrecognized token:"

-			// 		+ "%s\n", token);

 		}

 

 		return tk;