Merge branch 'master' of https://github.com/bculkin2442/JPratt.git

15 files changed, 509 insertions, 470 deletions

diff --git a/JPratt/src/examples/java/bjc/pratt/examples/InputState.java b/JPratt/src/examples/java/bjc/pratt/examples/InputState.java
deleted file mode 100644
index 7fe74fb..0000000
--- a/JPratt/src/examples/java/bjc/pratt/examples/InputState.java
+++ /dev/null
@@ -1,150 +0,0 @@
-package bjc.pratt.examples;
-
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import bjc.pratt.Token;
-import bjc.utils.data.TransformIterator;
-import bjc.utils.parserutils.splitterv2.ChainTokenSplitter;
-import bjc.utils.parserutils.splitterv2.ConfigurableTokenSplitter;
-import bjc.utils.parserutils.splitterv2.ExcludingTokenSplitter;
-import bjc.utils.parserutils.splitterv2.TokenSplitter;
-
-/**
- * Packaged input state for the parser.
- * 
- * @author student
- *
- */
-public class InputState {
-	public Set<String> ops;
-	public Set<String> reserved;
-
-	public TokenSplitter splitter;
-
-	public TestContext ctx;
-
-	public String ln;
-
-	private InputState(Set<String> opps, TokenSplitter splt, String lin, Set<String> rserved, TestContext ctxt) {
-		ops = opps;
-		reserved = rserved;
-
-		splitter = splt;
-
-		ctx = ctxt;
-
-		ln = lin;
-	}
-
-	public static InputState createState() {
-		/*
-		 * Use a linked hash set to preserve insertion order.
-		 */
-		Set<String> ops = new LinkedHashSet<>();
-
-		ops.add("!!!");
-
-		ops.addAll(Arrays.asList("->", "=>"));
-		ops.add(":=");
-		ops.addAll(Arrays.asList("||", "&&"));
-		ops.addAll(Arrays.asList("<=", ">="));
-
-		ops.addAll(Arrays.asList("±"));
-		ops.addAll(Arrays.asList(".", ",", ";", ":"));
-		ops.addAll(Arrays.asList("=", "<", ">"));
-		ops.addAll(Arrays.asList("+", "-", "*", "/"));
-		ops.addAll(Arrays.asList("^", "!"));
-		ops.addAll(Arrays.asList("(", ")"));
-		ops.addAll(Arrays.asList("[", "]"));
-		ops.addAll(Arrays.asList("{", "}"));
-
-		/*
-		 * Reserved words that represent themselves, not literals.
-		 */
-		Set<String> reserved = new LinkedHashSet<>();
-
-		reserved.addAll(Arrays.asList("if", "then", "else"));
-		reserved.addAll(Arrays.asList("and", "or"));
-		reserved.addAll(Arrays.asList("begin", "end"));
-		reserved.addAll(Arrays.asList("switch", "case"));
-		reserved.addAll(Arrays.asList("sqrt", "cbrt", "root"));
-
-		reserved.add("var");
-
-		ChainTokenSplitter nsplit = new ChainTokenSplitter();
-
-		ConfigurableTokenSplitter hi = new ConfigurableTokenSplitter(true);
-		ConfigurableTokenSplitter lo = new ConfigurableTokenSplitter(true);
-
-		hi.addSimpleDelimiters("->");
-		hi.addSimpleDelimiters(":=");
-		hi.addSimpleDelimiters("||", "&&");
-		hi.addSimpleDelimiters("<=", ">=");
-
-		lo.addSimpleDelimiters("±");
-		lo.addSimpleDelimiters(".", ",", ";", ":");
-		lo.addSimpleDelimiters("=", "<", ">");
-		lo.addSimpleDelimiters("+", "-", "*", "/");
-		lo.addSimpleDelimiters("^");
-
-		lo.addMultiDelimiters("!");
-		lo.addMultiDelimiters("(", ")");
-		lo.addMultiDelimiters("[", "]");
-		lo.addMultiDelimiters("{", "}");
-
-		hi.compile();
-		lo.compile();
-
-		nsplit.appendSplitters(hi, lo);
-
-		ExcludingTokenSplitter excluder = new ExcludingTokenSplitter(nsplit);
-
-		excluder.addLiteralExclusions(reserved.toArray(new String[0]));
-
-		TestContext ctx = new TestContext();
-
-		InputState state = new InputState(ops, excluder, null, reserved, ctx);
-
-		return state;
-	}
-
-	public Iterator<Token<String, String>> preprocessInput() {
-		String[] rawTokens = ln.split("\\s+");
-	
-		List<String> splitTokens = new LinkedList<>();
-	
-		for (String raw : rawTokens) {
-			boolean doSplit = false;
-	
-			for (String op : ops) {
-				if (raw.contains(op)) {
-					doSplit = true;
-					break;
-				}
-			}
-	
-			if (doSplit) {
-				String[] strangs = splitter.split(raw).toArray(new String[0]);
-	
-				splitTokens.addAll(Arrays.asList(strangs));
-			} else {
-				splitTokens.add(raw);
-			}
-		}
-	
-		System.out.println("\nSplit string: " + splitTokens);
-	
-		Iterator<String> source = splitTokens.iterator();
-	
-		Tokenizer tokenzer = new Tokenizer(ops, reserved, ctx);
-	
-		Iterator<Token<String, String>> tokens = new TransformIterator<>(source, tokenzer);
-	
-		return tokens;
-	}
-}
\ No newline at end of file
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java b/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java
deleted file mode 100644
index 7b9795e..0000000
--- a/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java
+++ /dev/null
@@ -1,184 +0,0 @@
-package bjc.pratt.examples;
-
-import bjc.pratt.InitialCommand;
-import bjc.pratt.NonInitialCommand;
-import bjc.pratt.PrattParser;
-import bjc.pratt.Token;
-import bjc.pratt.tokens.StringToken;
-import bjc.pratt.tokens.StringTokenStream;
-import bjc.utils.data.ITree;
-import bjc.utils.parserutils.ParserException;
-
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Scanner;
-import java.util.function.UnaryOperator;
-
-import static bjc.pratt.commands.InitialCommands.*;
-import static bjc.pratt.commands.NonInitialCommands.*;
-import static bjc.pratt.tokens.StringToken.litToken;
-import static bjc.utils.functypes.ID.id;
-
-/**
- * Simple test for Pratt parser.
- * 
- * @author EVE
- *
- */
-public class PrattParserTest {
-	/**
-	 * Main method.
-	 * 
-	 * @param args
-	 *            Unused CLI arguments.
-	 */
-	public static void main(String[] args) {
-		InputState state = InputState.createState();
-
-		PrattParser<String, String, TestContext> parser = createParser();
-
-		Scanner scn = new Scanner(System.in);
-
-		System.out.print("Enter a command (blank line to exit): ");
-		String ln = scn.nextLine();
-
-		while (!ln.trim().equals("")) {
-			state.ln = ln;
-
-			Iterator<Token<String, String>> tokens = state.preprocessInput();
-
-			try {
-				StringTokenStream tokenStream = new StringTokenStream(tokens);
-
-				/*
-				 * Prime stream.
-				 */
-				tokenStream.next();
-
-				ITree<Token<String, String>> tree = parser.parseExpression(0, tokenStream, state.ctx, true);
-
-				if (!tokenStream.headIs("(end)")) {
-					System.out.println("\nMultiple expressions on line");
-				}
-
-				System.out.println("\nParsed expression:\n" + tree);
-			} catch (ParserException pex) {
-				pex.printStackTrace();
-			}
-
-			System.out.print("\nEnter a command (blank line to exit): ");
-			ln = scn.nextLine();
-		}
-
-		System.out.println();
-		System.out.println("\nContext is: " + state.ctx);
-
-		scn.close();
-	}
-
-	private static PrattParser<String, String, TestContext> createParser() {
-		/*
-		 * Set of which relational operators chain with each other.
-		 */
-		HashSet<String> relChain = new HashSet<>();
-		relChain.addAll(Arrays.asList("=", "<", ">", "<=", ">="));
-
-		/*
-		 * Token for marking chains.
-		 */
-		StringToken chainToken = litToken("and");
-
-		/*
-		 * ID function.
-		 */
-		UnaryOperator<TestContext> idfun = id();
-
-		PrattParser<String, String, TestContext> parser = new PrattParser<>();
-
-		parser.addNonInitialCommand("!!!", postfix(0));
-
-		parser.addNonInitialCommand(":", infixNon(3));
-
-		NonInitialCommand<String, String, TestContext> ifElse = ternary(5, 0, "else", litToken("cond"), false);
-		parser.addNonInitialCommand("if", ifElse);
-
-		parser.addNonInitialCommand(":=", new AssignCommand(10));
-
-		parser.addNonInitialCommand("->", infixRight(11));
-
-		NonInitialCommand<String, String, TestContext> nonSSRelJoin = infixLeft(13);
-		parser.addNonInitialCommand("and", nonSSRelJoin);
-		parser.addNonInitialCommand("or", nonSSRelJoin);
-
-		NonInitialCommand<String, String, TestContext> chainRelOp = chain(15, relChain, chainToken);
-		parser.addNonInitialCommand("=", chainRelOp);
-		parser.addNonInitialCommand("<", chainRelOp);
-		parser.addNonInitialCommand(">", chainRelOp);
-		parser.addNonInitialCommand("<=", chainRelOp);
-		parser.addNonInitialCommand(">=", chainRelOp);
-
-		NonInitialCommand<String, String, TestContext> ssRelJoin = infixRight(17);
-		parser.addNonInitialCommand("&&", ssRelJoin);
-		parser.addNonInitialCommand("||", ssRelJoin);
-
-		NonInitialCommand<String, String, TestContext> addSub = infixLeft(20);
-		parser.addNonInitialCommand("+", addSub);
-		parser.addNonInitialCommand("-", addSub);
-		parser.addNonInitialCommand("±", addSub);
-
-		NonInitialCommand<String, String, TestContext> mulDiv = infixLeft(30);
-		parser.addNonInitialCommand("*", mulDiv);
-		parser.addNonInitialCommand("/", mulDiv);
-
-		parser.addNonInitialCommand("!", postfix(40));
-
-		NonInitialCommand<String, String, TestContext> expon = infixRight(50);
-		parser.addNonInitialCommand("^", expon);
-		parser.addNonInitialCommand("root", expon);
-
-		NonInitialCommand<String, String, TestContext> superexpon = postfix(50);
-		parser.addNonInitialCommand("(superexp)", superexpon);
-
-		parser.addNonInitialCommand(".", infixLeft(60));
-
-		NonInitialCommand<String, String, TestContext> arrayIdx = postCircumfix(60, 0, "]", litToken("idx"));
-		parser.addNonInitialCommand("[", arrayIdx);
-
-		InitialCommand<String, String, TestContext> ifThenElse = preTernary(0, 0, 0, "then", "else",
-				litToken("ifelse"));
-		parser.addInitialCommand("if", ifThenElse);
-
-		InitialCommand<String, String, TestContext> parens = grouping(0, ")", litToken("parens"));
-		parser.addInitialCommand("(", parens);
-
-		InitialCommand<String, String, TestContext> scoper = delimited(0, ";", "end", litToken("block"),
-				new BlockEnter(), idfun, new BlockExit(), true);
-		parser.addInitialCommand("begin", scoper);
-
-		InitialCommand<String, String, TestContext> arrayLiteral = delimited(0, ",", "]", litToken("array"), idfun,
-				idfun, idfun, false);
-		parser.addInitialCommand("[", arrayLiteral);
-
-		InitialCommand<String, String, TestContext> jsonLiteral = delimited(0, ",", "}", litToken("json"), idfun, idfun,
-				idfun, false);
-		parser.addInitialCommand("{", jsonLiteral);
-
-		parser.addInitialCommand("case", unary(5));
-
-		parser.addInitialCommand("-", unary(30));
-
-		InitialCommand<String, String, TestContext> root = unary(50);
-		parser.addInitialCommand("sqrt", root);
-		parser.addInitialCommand("cbrt", root);
-
-		InitialCommand<String, String, TestContext> leaf = leaf();
-		parser.addInitialCommand("(literal)", leaf);
-
-		parser.addInitialCommand("var", new VarCommand());
-
-		parser.addInitialCommand("switch", new SwitchCommand());
-
-		return parser;
-	}
-}
\ No newline at end of file
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/SwitchCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/SwitchCommand.java
deleted file mode 100644
index 4a5bd22..0000000
--- a/JPratt/src/examples/java/bjc/pratt/examples/SwitchCommand.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package bjc.pratt.examples;
-
-import bjc.pratt.InitialCommand;
-import bjc.pratt.ParserContext;
-import bjc.pratt.Token;
-import bjc.pratt.tokens.StringToken;
-import bjc.utils.data.ITree;
-import bjc.utils.data.Tree;
-import bjc.utils.parserutils.ParserException;
-
-class SwitchCommand implements InitialCommand<String, String, TestContext> {
-	@Override
-	public ITree<Token<String, String>> denote(Token<String, String> operator,
-			ParserContext<String, String, TestContext> ctx) throws ParserException {
-		ITree<Token<String, String>> object = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
-
-		ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
-
-		return new Tree<>(new StringToken("switch", "switch"), object, body);
-	}
-}
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java b/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java
deleted file mode 100644
index cfaf3e2..0000000
--- a/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package bjc.pratt.examples;
-
-import bjc.pratt.Token;
-import bjc.pratt.tokens.StringToken;
-
-import java.util.Set;
-import java.util.function.Function;
-
-import static bjc.pratt.tokens.StringToken.litToken;
-
-final class Tokenizer implements Function<String, Token<String, String>> {
-	private Set<String> ops;
-	private Set<String> reserved;
-
-	@SuppressWarnings("unused")
-	private TestContext ctx;
-
-	public Tokenizer(Set<String> operators, Set<String> reservedWords, TestContext context) {
-		ops = operators;
-		reserved = reservedWords;
-		ctx = context;
-	}
-
-	@Override
-	public Token<String, String> apply(String strang) {
-		if (ops.contains(strang) || reserved.contains(strang)) {
-			return litToken(strang);
-		} else if (strang.matches("(?:[\\u00B2\\u00B3\\u00B9\\u2070]|[\\u2074-\\u2079])+")) {
-			/*
-			 * This regular expression matches series of unicode super-scripts
-			 * 1-9.
-			 */
-			return new StringToken("(superexp)", strang);
-		} else {
-			return new StringToken("(literal)", strang);
-		}
-	}
-}
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/TreeEvaluator.java b/JPratt/src/examples/java/bjc/pratt/examples/TreeEvaluator.java
deleted file mode 100644
index e4325b2..0000000
--- a/JPratt/src/examples/java/bjc/pratt/examples/TreeEvaluator.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package bjc.pratt.examples;
-
-import bjc.utils.data.ITree;
-import bjc.utils.data.TopDownTransformResult;
-
-/**
- * Evaluate a tree to a result.
- * 
- * @author student
- *
- */
-public class TreeEvaluator {
-	/**
-	 * The result of evaluating a tree.
-	 * 
-	 * @author student
-	 *
-	 */
-	public static final class EvaluationResult {
-		public static EvaluationResult fromToken(EvaluationToken tok) {
-			return null;
-		}
-	}
-
-	/**
-	 * The token type for trees to evaluate.
-	 * 
-	 * @author student
-	 *
-	 */
-	public static final class EvaluationToken {
-
-	}
-
-	public static EvaluationResult evaluateTree(ITree<EvaluationToken> tree, TestContext ctx) {
-		ITree<EvaluationToken> result = tree.topDownTransform(TreeEvaluator::pickNodeEvaluation,
-				TreeEvaluator::evaluateNode);
-
-		return EvaluationResult.fromToken(result.getHead());
-	}
-
-	private static TopDownTransformResult pickNodeEvaluation(EvaluationToken node) {
-		return TopDownTransformResult.PUSHDOWN;
-	}
-
-	private static ITree<EvaluationToken> evaluateNode(ITree<EvaluationToken> tree) {
-		return tree;
-	}
-}
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/AssignCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/AssignCommand.java
index 45aac1a..8b96850 100644
--- a/JPratt/src/examples/java/bjc/pratt/examples/AssignCommand.java
+++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/AssignCommand.java
@@ -1,22 +1,23 @@
-package bjc.pratt.examples;
+package bjc.pratt.examples.lang;
 
 import bjc.pratt.ParserContext;
-import bjc.pratt.Token;
-import bjc.pratt.commands.NonBinaryCommand;
+import bjc.pratt.commands.impls.NonBinaryCommand;
 import bjc.pratt.tokens.StringToken;
+import bjc.pratt.tokens.Token;
 import bjc.utils.data.ITree;
 import bjc.utils.data.Tree;
 import bjc.utils.parserutils.ParserException;
 
 class AssignCommand extends NonBinaryCommand<String, String, TestContext> {
-	public AssignCommand(int prec) {
+	public AssignCommand(final int prec) {
 		super(prec);
 	}
 
 	@Override
-	public ITree<Token<String, String>> denote(ITree<Token<String, String>> operand, Token<String, String> operator,
-			ParserContext<String, String, TestContext> ctx) throws ParserException {
-		Token<String, String> name = operand.getHead();
+	public ITree<Token<String, String>> denote(final ITree<Token<String, String>> operand,
+			final Token<String, String> operator, final ParserContext<String, String, TestContext> ctx)
+			throws ParserException {
+		final Token<String, String> name = operand.getHead();
 
 		switch (name.getKey()) {
 		case "(literal)":
@@ -26,7 +27,7 @@ class AssignCommand extends NonBinaryCommand<String, String, TestContext> {
 			throw new ParserException("Variable name must be simple");
 		}
 
-		ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
+		final ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
 
 		ctx.state.scopes.top().putKey(name.getValue(), body);
 
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/BlockEnter.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/BlockEnter.java
index 67e560c..dedf010 100644
--- a/JPratt/src/examples/java/bjc/pratt/examples/BlockEnter.java
+++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/BlockEnter.java
@@ -1,19 +1,19 @@
-package bjc.pratt.examples;
+package bjc.pratt.examples.lang;
 
-import bjc.pratt.Token;
+import java.util.function.UnaryOperator;
+
+import bjc.pratt.tokens.Token;
 import bjc.utils.data.ITree;
 import bjc.utils.esodata.Directory;
 import bjc.utils.esodata.Stack;
 
-import java.util.function.UnaryOperator;
-
 final class BlockEnter implements UnaryOperator<TestContext> {
 	@Override
-	public TestContext apply(TestContext state) {
-		Directory<String, ITree<Token<String, String>>> enclosing = state.scopes.top();
-		Stack<Integer> blockCount = state.blockCount;
+	public TestContext apply(final TestContext state) {
+		final Directory<String, ITree<Token<String, String>>> enclosing = state.scopes.top();
+		final Stack<Integer> blockCount = state.blockCount;
 
-		int currBlockNumber = blockCount.pop();
+		final int currBlockNumber = blockCount.pop();
 
 		state.scopes.push(enclosing.newSubdirectory("block" + currBlockNumber));
 
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/BlockExit.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/BlockExit.java
index 1cca6d5..15b2a5e 100644
--- a/JPratt/src/examples/java/bjc/pratt/examples/BlockExit.java
+++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/BlockExit.java
@@ -1,10 +1,10 @@
-package bjc.pratt.examples;
+package bjc.pratt.examples.lang;
 
 import java.util.function.UnaryOperator;
 
 final class BlockExit implements UnaryOperator<TestContext> {
 	@Override
-	public TestContext apply(TestContext state) {
+	public TestContext apply(final TestContext state) {
 		state.scopes.pop();
 
 		state.blockCount.pop();
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/PrattParserTest.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/PrattParserTest.java
new file mode 100644
index 0000000..7af16a0
--- /dev/null
+++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/PrattParserTest.java
@@ -0,0 +1,317 @@
+package bjc.pratt.examples.lang;
+
+import static bjc.pratt.commands.impls.InitialCommands.delimited;
+import static bjc.pratt.commands.impls.InitialCommands.grouping;
+import static bjc.pratt.commands.impls.InitialCommands.leaf;
+import static bjc.pratt.commands.impls.InitialCommands.preTernary;
+import static bjc.pratt.commands.impls.InitialCommands.unary;
+import static bjc.pratt.commands.impls.NonInitialCommands.chain;
+import static bjc.pratt.commands.impls.NonInitialCommands.infixLeft;
+import static bjc.pratt.commands.impls.NonInitialCommands.infixNon;
+import static bjc.pratt.commands.impls.NonInitialCommands.infixRight;
+import static bjc.pratt.commands.impls.NonInitialCommands.postCircumfix;
+import static bjc.pratt.commands.impls.NonInitialCommands.postfix;
+import static bjc.pratt.commands.impls.NonInitialCommands.ternary;
+import static bjc.pratt.tokens.StringToken.litToken;
+import static bjc.utils.functypes.ID.id;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Scanner;
+import java.util.Set;
+import java.util.function.UnaryOperator;
+
+import bjc.pratt.PrattParser;
+import bjc.pratt.commands.InitialCommand;
+import bjc.pratt.commands.NonInitialCommand;
+import bjc.pratt.tokens.StringToken;
+import bjc.pratt.tokens.StringTokenStream;
+import bjc.pratt.tokens.Token;
+import bjc.utils.data.ITree;
+import bjc.utils.data.TransformIterator;
+import bjc.utils.funcdata.IList;
+import bjc.utils.parserutils.ParserException;
+import bjc.utils.parserutils.splitter.ChainTokenSplitter;
+import bjc.utils.parserutils.splitter.ConfigurableTokenSplitter;
+import bjc.utils.parserutils.splitter.ExcludingTokenSplitter;
+import bjc.utils.parserutils.splitter.FilteredTokenSplitter;
+import bjc.utils.parserutils.splitter.TokenSplitter;
+
+/**
+ * Simple test for Pratt parser.
+ *
+ * @author EVE
+ *
+ */
+public class PrattParserTest {
+	/**
+	 * Main method.
+	 *
+	 * @param args
+	 *                Unused CLI arguments.
+	 */
+	public static void main(final String[] args) {
+		/*
+		 * Use a linked hash set to preserve insertion order.
+		 */
+		final Set<String> ops = new LinkedHashSet<>();
+
+		ops.add("!!!");
+
+		ops.addAll(Arrays.asList("->", "=>"));
+		ops.add(":=");
+		ops.addAll(Arrays.asList("||", "&&"));
+		ops.addAll(Arrays.asList("<=", ">="));
+
+		ops.addAll(Arrays.asList("\u00B1")); // Unicode plus/minus
+		ops.addAll(Arrays.asList(".", ",", ";", ":"));
+		ops.addAll(Arrays.asList("=", "<", ">"));
+		ops.addAll(Arrays.asList("+", "-", "*", "/"));
+		ops.addAll(Arrays.asList("^", "!"));
+		ops.addAll(Arrays.asList("(", ")"));
+		ops.addAll(Arrays.asList("[", "]"));
+		ops.addAll(Arrays.asList("{", "}"));
+
+		/*
+		 * Reserved words that represent themselves, not literals.
+		 */
+		final Set<String> reserved = new LinkedHashSet<>();
+		reserved.addAll(Arrays.asList("if", "then", "else"));
+		reserved.addAll(Arrays.asList("and", "or"));
+		reserved.addAll(Arrays.asList("begin", "end"));
+		reserved.addAll(Arrays.asList("switch", "case"));
+		reserved.addAll(Arrays.asList("sqrt", "cbrt", "root"));
+		reserved.addAll(Arrays.asList("try", "catch", "finally"));
+		reserved.add("var");
+
+		final ChainTokenSplitter nsplit = new ChainTokenSplitter();
+
+		final ConfigurableTokenSplitter hi = new ConfigurableTokenSplitter(true);
+		final ConfigurableTokenSplitter lo = new ConfigurableTokenSplitter(true);
+
+		hi.addSimpleDelimiters("->");
+		hi.addSimpleDelimiters(":=");
+		hi.addSimpleDelimiters("||", "&&");
+		hi.addSimpleDelimiters("<=", ">=");
+
+		lo.addSimpleDelimiters("\u00B1"); // Unicode plus/minus
+		lo.addSimpleDelimiters(".", ",", ";", ":");
+		lo.addSimpleDelimiters("=", "<", ">");
+		lo.addSimpleDelimiters("+", "-", "*", "/");
+		lo.addSimpleDelimiters("^");
+
+		lo.addMultiDelimiters("!");
+		lo.addMultiDelimiters("(", ")");
+		lo.addMultiDelimiters("[", "]");
+		lo.addMultiDelimiters("{", "}");
+
+		hi.compile();
+		lo.compile();
+
+		nsplit.appendSplitters(hi, lo);
+
+		final ExcludingTokenSplitter excluder = new ExcludingTokenSplitter(nsplit);
+
+		excluder.addLiteralExclusions(reserved.toArray(new String[0]));
+
+		final FilteredTokenSplitter filtered = new FilteredTokenSplitter(excluder, (tok) -> !tok.equals(""));
+
+		final PrattParser<String, String, TestContext> parser = createParser();
+
+		final TestContext ctx = new TestContext();
+
+		final Scanner scn = new Scanner(System.in);
+
+		System.out.print("Enter a command (blank line to exit): ");
+		String ln = scn.nextLine();
+
+		while (!ln.trim().equals("")) {
+			final Iterator<Token<String, String>> tokens = preprocessInput(ops, filtered, ln, reserved,
+					ctx);
+
+			try {
+				final StringTokenStream tokenStream = new StringTokenStream(tokens);
+
+				/*
+				 * Prime stream.
+				 */
+				tokenStream.next();
+
+				final ITree<Token<String, String>> tree = parser.parseExpression(0, tokenStream, ctx,
+						true);
+
+				if (!tokenStream.headIs("(end)")) {
+					System.out.println("\nMultiple expressions on line");
+				}
+
+				System.out.println("\nParsed expression:\n" + tree);
+			} catch (final ParserException pex) {
+				pex.printStackTrace();
+			}
+
+			System.out.print("\nEnter a command (blank line to exit): ");
+			ln = scn.nextLine();
+		}
+
+		System.out.println();
+		System.out.println("\nContext is: " + ctx);
+
+		scn.close();
+	}
+
+	private static Iterator<Token<String, String>> preprocessInput(final Set<String> ops, final TokenSplitter split,
+			final String ln, final Set<String> reserved, final TestContext ctx) {
+		final String[] rawTokens = ln.split("\\s+");
+
+		final List<String> splitTokens = new LinkedList<>();
+
+		for (final String raw : rawTokens) {
+			if (raw.equals("")) continue;
+
+			boolean doSplit = false;
+
+			for (final String op : ops) {
+				if (raw.contains(op)) {
+					doSplit = true;
+					break;
+				}
+			}
+
+			if (doSplit) {
+				IList<String> splitStrangs = split.split(raw);
+				splitStrangs.removeMatching("");
+
+				splitStrangs.forEach(splitTokens::add);
+			} else {
+				splitTokens.add(raw);
+			}
+		}
+
+		System.out.println("\nSplit string: " + splitTokens);
+
+		final Iterator<String> source = splitTokens.iterator();
+
+		final Tokenizer tokenzer = new Tokenizer(ops, reserved, ctx);
+
+		final Iterator<Token<String, String>> tokens = new TransformIterator<>(source, tokenzer);
+
+		return tokens;
+	}
+
+	private static PrattParser<String, String, TestContext> createParser() {
+		/*
+		 * Set of which relational operators chain with each other.
+		 */
+		final HashSet<String> relChain = new HashSet<>();
+		relChain.addAll(Arrays.asList("=", "<", ">", "<=", ">="));
+
+		/*
+		 * Token for marking chains.
+		 */
+		final StringToken chainToken = litToken("and");
+
+		/*
+		 * ID function.
+		 */
+		final UnaryOperator<TestContext> idfun = id();
+
+		final PrattParser<String, String, TestContext> parser = new PrattParser<>();
+
+		parser.addNonInitialCommand("!!!", postfix(0));
+
+		parser.addNonInitialCommand(":", infixNon(3));
+
+		parser.addNonInitialCommand("finally", infixLeft(4));
+
+		parser.addNonInitialCommand("catch", infixLeft(5));
+
+		final NonInitialCommand<String, String, TestContext> ifElse = ternary(6, 0, "else", litToken("cond"),
+				false);
+		parser.addNonInitialCommand("if", ifElse);
+
+		parser.addNonInitialCommand(":=", new AssignCommand(10));
+
+		parser.addNonInitialCommand("->", infixRight(11));
+
+		final NonInitialCommand<String, String, TestContext> nonSSRelJoin = infixLeft(13);
+		parser.addNonInitialCommand("and", nonSSRelJoin);
+		parser.addNonInitialCommand("or", nonSSRelJoin);
+
+		final NonInitialCommand<String, String, TestContext> chainRelOp = chain(15, relChain, chainToken);
+		parser.addNonInitialCommand("=", chainRelOp);
+		parser.addNonInitialCommand("<", chainRelOp);
+		parser.addNonInitialCommand(">", chainRelOp);
+		parser.addNonInitialCommand("<=", chainRelOp);
+		parser.addNonInitialCommand(">=", chainRelOp);
+
+		final NonInitialCommand<String, String, TestContext> ssRelJoin = infixRight(17);
+		parser.addNonInitialCommand("&&", ssRelJoin);
+		parser.addNonInitialCommand("||", ssRelJoin);
+
+		final NonInitialCommand<String, String, TestContext> addSub = infixLeft(20);
+		parser.addNonInitialCommand("+", addSub);
+		parser.addNonInitialCommand("-", addSub);
+		parser.addNonInitialCommand("\u00B1", addSub); // Unicode plus/minus
+
+		final NonInitialCommand<String, String, TestContext> mulDiv = infixLeft(30);
+		parser.addNonInitialCommand("*", mulDiv);
+		parser.addNonInitialCommand("/", mulDiv);
+
+		parser.addNonInitialCommand("!", postfix(40));
+
+		final NonInitialCommand<String, String, TestContext> expon = infixRight(50);
+		parser.addNonInitialCommand("^", expon);
+		parser.addNonInitialCommand("root", expon);
+
+		final NonInitialCommand<String, String, TestContext> superexpon = postfix(50);
+		parser.addNonInitialCommand("(superexp)", superexpon);
+
+		parser.addNonInitialCommand(".", infixLeft(60));
+
+		final NonInitialCommand<String, String, TestContext> arrayIdx = postCircumfix(60, 0, "]",
+				litToken("idx"));
+		parser.addNonInitialCommand("[", arrayIdx);
+
+		final InitialCommand<String, String, TestContext> ifThenElse = preTernary(0, 0, 0, "then", "else",
+				litToken("ifelse"));
+		parser.addInitialCommand("if", ifThenElse);
+
+		final InitialCommand<String, String, TestContext> parens = grouping(0, ")", litToken("parens"));
+		parser.addInitialCommand("(", parens);
+
+		final InitialCommand<String, String, TestContext> scoper = delimited(0, ";", "end", litToken("block"),
+				new BlockEnter(), idfun, new BlockExit(), true);
+		parser.addInitialCommand("begin", scoper);
+
+		final InitialCommand<String, String, TestContext> arrayLiteral = delimited(0, ",", "]",
+				litToken("array"), idfun, idfun, idfun, false);
+		parser.addInitialCommand("[", arrayLiteral);
+
+		final InitialCommand<String, String, TestContext> jsonLiteral = delimited(0, ",", "}", litToken("json"),
+				idfun, idfun, idfun, false);
+		parser.addInitialCommand("{", jsonLiteral);
+
+		parser.addInitialCommand("try", unary(3));
+
+		parser.addInitialCommand("case", unary(5));
+
+		parser.addInitialCommand("-", unary(30));
+
+		final InitialCommand<String, String, TestContext> root = unary(50);
+		parser.addInitialCommand("sqrt", root);
+		parser.addInitialCommand("cbrt", root);
+
+		final InitialCommand<String, String, TestContext> leaf = leaf();
+		parser.addInitialCommand("(literal)", leaf);
+
+		parser.addInitialCommand("var", new VarCommand());
+
+		parser.addInitialCommand("switch", new SwitchCommand());
+
+		return parser;
+	}
+}
\ No newline at end of file
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/SwitchCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/SwitchCommand.java
new file mode 100644
index 0000000..e824375
--- /dev/null
+++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/SwitchCommand.java
@@ -0,0 +1,21 @@
+package bjc.pratt.examples.lang;
+
+import bjc.pratt.ParserContext;
+import bjc.pratt.commands.InitialCommand;
+import bjc.pratt.tokens.StringToken;
+import bjc.pratt.tokens.Token;
+import bjc.utils.data.ITree;
+import bjc.utils.data.Tree;
+import bjc.utils.parserutils.ParserException;
+
+class SwitchCommand implements InitialCommand<String, String, TestContext> {
+	@Override
+	public ITree<Token<String, String>> denote(final Token<String, String> operator,
+			final ParserContext<String, String, TestContext> ctx) throws ParserException {
+		final ITree<Token<String, String>> object = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
+
+		final ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
+
+		return new Tree<>(new StringToken("switch", "switch"), object, body);
+	}
+}
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/TestContext.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/TestContext.java
index b5e8448..340c83d 100644
--- a/JPratt/src/examples/java/bjc/pratt/examples/TestContext.java
+++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/TestContext.java
@@ -1,6 +1,6 @@
-package bjc.pratt.examples;
+package bjc.pratt.examples.lang;
 
-import bjc.pratt.Token;
+import bjc.pratt.tokens.Token;
 import bjc.utils.data.ITree;
 import bjc.utils.esodata.Directory;
 import bjc.utils.esodata.SimpleDirectory;
@@ -9,7 +9,7 @@ import bjc.utils.esodata.Stack;
 
 /**
  * Simple context for the parser.
- * 
+ *
  * @author EVE
  *
  */
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/Tokenizer.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/Tokenizer.java
new file mode 100644
index 0000000..7a84d93
--- /dev/null
+++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/Tokenizer.java
@@ -0,0 +1,38 @@
+package bjc.pratt.examples.lang;
+
+import static bjc.pratt.tokens.StringToken.litToken;
+
+import java.util.Set;
+import java.util.function.Function;
+
+import bjc.pratt.tokens.StringToken;
+import bjc.pratt.tokens.Token;
+
+final class Tokenizer implements Function<String, Token<String, String>> {
+	private final Set<String>	ops;
+	private final Set<String>	reserved;
+
+	@SuppressWarnings("unused")
+	private final TestContext ctx;
+
+	public Tokenizer(final Set<String> operators, final Set<String> reservedWords, final TestContext context) {
+		ops = operators;
+		reserved = reservedWords;
+		ctx = context;
+	}
+
+	@Override
+	public Token<String, String> apply(final String strang) {
+		if (ops.contains(strang) || reserved.contains(strang))
+			return litToken(strang);
+		else if (strang.matches("(?:[\\u00B2\\u00B3\\u00B9\\u2070]|[\\u2074-\\u2079])+")) {
+			/*
+			 * This regular expression matches series of unicode
+			 * super - scripts 1 - 9.
+			 */
+			String subscript = strang;
+
+			return new StringToken("(superexp)", subscript);
+		} else return new StringToken("(literal)", strang);
+	}
+}
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/VarCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/VarCommand.java
index dd04a59..48fc49c 100644
--- a/JPratt/src/examples/java/bjc/pratt/examples/VarCommand.java
+++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/VarCommand.java
@@ -1,9 +1,9 @@
-package bjc.pratt.examples;
+package bjc.pratt.examples.lang;
 
 import bjc.pratt.ParserContext;
-import bjc.pratt.Token;
 import bjc.pratt.commands.AbstractInitialCommand;
 import bjc.pratt.tokens.StringToken;
+import bjc.pratt.tokens.Token;
 import bjc.utils.data.ITree;
 import bjc.utils.data.Tree;
 import bjc.utils.parserutils.ParserException;
@@ -11,11 +11,11 @@ import bjc.utils.parserutils.ParserException;
 class VarCommand extends AbstractInitialCommand<String, String, TestContext> {
 
 	@Override
-	protected ITree<Token<String, String>> intNullDenotation(Token<String, String> operator,
-			ParserContext<String, String, TestContext> ctx) throws ParserException {
-		Token<String, String> name = ctx.tokens.current();
+	protected ITree<Token<String, String>> intNullDenotation(final Token<String, String> operator,
+			final ParserContext<String, String, TestContext> ctx) throws ParserException {
+		final Token<String, String> name = ctx.tokens.current();
 
-		switch(name.getKey()) {
+		switch (name.getKey()) {
 		case "(literal)":
 		case "(vref)":
 			ctx.tokens.next();
@@ -26,7 +26,7 @@ class VarCommand extends AbstractInitialCommand<String, String, TestContext> {
 
 		ctx.tokens.expect("=");
 
-		ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
+		final ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
 
 		ctx.state.scopes.top().putKey(name.getValue(), body);
 
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/regex/Destringer.java b/JPratt/src/examples/java/bjc/pratt/examples/regex/Destringer.java
new file mode 100644
index 0000000..7ba2d5b
--- /dev/null
+++ b/JPratt/src/examples/java/bjc/pratt/examples/regex/Destringer.java
@@ -0,0 +1,31 @@
+package bjc.pratt.examples.regex;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.function.UnaryOperator;
+
+import bjc.utils.data.GeneratingIterator;
+
+final class Destringer implements UnaryOperator<String> {
+	private final Iterator<Integer>		numbers;
+	public final Map<String, String>	stringLiterals;
+
+	public Destringer(final Map<String, String> literals) {
+		numbers = new GeneratingIterator<>(0, (num) -> num + 1, (val) -> true);
+		stringLiterals = literals;
+	}
+
+	@Override
+	public String apply(final String token) {
+		if (token.startsWith("\"") && token.endsWith("\"")) {
+			final String symName = "stringLiteral" + Integer.toString(numbers.next());
+
+			final String dequotedString = token.substring(1, token.length() - 1);
+			stringLiterals.put(symName, dequotedString);
+
+			return symName;
+		}
+
+		return token;
+	}
+}
\ No newline at end of file
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/regex/RegexGrammar.java b/JPratt/src/examples/java/bjc/pratt/examples/regex/RegexGrammar.java
new file mode 100644
index 0000000..5bb606e
--- /dev/null
+++ b/JPratt/src/examples/java/bjc/pratt/examples/regex/RegexGrammar.java
@@ -0,0 +1,73 @@
+package bjc.pratt.examples.regex;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Scanner;
+
+import bjc.utils.funcdata.IList;
+import bjc.utils.parserutils.TokenUtils.StringTokenSplitter;
+import bjc.utils.parserutils.splitter.ChainTokenSplitter;
+import bjc.utils.parserutils.splitter.ConfigurableTokenSplitter;
+import bjc.utils.parserutils.splitter.TokenSplitter;
+import bjc.utils.parserutils.splitter.TransformTokenSplitter;
+
+/**
+ * Grammar test for regular expressions.
+ *
+ * @author bjculkin
+ *
+ */
+public class RegexGrammar {
+	/**
+	 * Main method.
+	 *
+	 * @param args
+	 *                Unused CLI arguments.
+	 */
+	public static void main(final String[] args) {
+		final Scanner scn = new Scanner(System.in);
+
+		System.out.print("Enter text to parse (blank line to exit): ");
+		String ln = scn.nextLine().trim();
+
+		final Map<String, String> stringLiterals = new HashMap<>();
+
+		/*
+		 * Build the token splitter
+		 */
+		final ChainTokenSplitter splitter = buildSplitter(stringLiterals);
+
+		while (!ln.equals("")) {
+			final IList<String> quotelessTokens = splitter.split(ln);
+
+			System.out.println("\nSplit tokens: " + quotelessTokens);
+
+			System.out.print("\nEnter text to parse (blank line to exit): ");
+			ln = scn.nextLine().trim();
+		}
+
+		System.out.println("\nString table: ");
+		for (final Entry<String, String> entry : stringLiterals.entrySet()) {
+			System.out.printf("\t%s\t'%s'\n", entry.getKey(), entry.getValue());
+		}
+
+		scn.close();
+	}
+
+	private static ChainTokenSplitter buildSplitter(final Map<String, String> stringLiterals) {
+		final Destringer destringer = new Destringer(stringLiterals);
+
+		final TokenSplitter dquoteSplitter = new StringTokenSplitter();
+		final TokenSplitter dquoteRemover = new TransformTokenSplitter(dquoteSplitter, destringer);
+
+		final ConfigurableTokenSplitter regexSplitter = new ConfigurableTokenSplitter(true);
+		regexSplitter.addSimpleDelimiters("+", "|");
+		regexSplitter.compile();
+
+		final ChainTokenSplitter splitter = new ChainTokenSplitter();
+		splitter.appendSplitters(dquoteRemover, regexSplitter);
+
+		return splitter;
+	}
+}