Update Pratt Parser

5 files changed, 141 insertions, 14 deletions

diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/AssignCommand.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/AssignCommand.java
new file mode 100644
index 0000000..9cd30a8
--- /dev/null
+++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/AssignCommand.java
@@ -0,0 +1,35 @@
+package bjc.utils.examples.parsing;
+
+import bjc.utils.data.ITree;
+import bjc.utils.data.Tree;
+import bjc.utils.parserutils.ParserException;
+import bjc.utils.parserutils.pratt.ParserContext;
+import bjc.utils.parserutils.pratt.StringToken;
+import bjc.utils.parserutils.pratt.Token;
+import bjc.utils.parserutils.pratt.commands.NonBinaryCommand;
+
+class AssignCommand extends NonBinaryCommand<String, String, TestContext> {
+	public AssignCommand() {
+		super(10);
+	}
+
+	@Override
+	public ITree<Token<String, String>> denote(ITree<Token<String, String>> operand, Token<String, String> operator,
+			ParserContext<String, String, TestContext> ctx) throws ParserException {
+		Token<String, String> name = operand.getHead();
+
+		switch(name.getKey()) {
+		case "(literal)":
+		case "(vref)":
+			break;
+		default:
+			throw new ParserException("Variable name must be simple");
+		}
+
+		ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
+
+		ctx.state.scopes.top().putKey(name.getValue(), body);
+
+		return new Tree<>(new StringToken("assign", "assign"), operand, body);
+	}
+}
diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java
index 3462d74..110d331 100644
--- a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java
+++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java
@@ -25,7 +25,7 @@ import static bjc.utils.parserutils.pratt.commands.NonInitialCommands.*;
 import static bjc.utils.parserutils.pratt.commands.InitialCommands.*;
 
 /**
- * Simple test for pratt parser.
+ * Simple test for Pratt parser.
  * 
  * @author EVE
  *
@@ -45,7 +45,7 @@ public class PrattParserTest {
 	private static final class BlockEnter implements UnaryOperator<TestContext> {
 		@Override
 		public TestContext apply(TestContext state) {
-			Directory<String, String> enclosing = state.scopes.top();
+			Directory<String, ITree<Token<String, String>>> enclosing = state.scopes.top();
 			int currBlockNumber = state.blockCount.pop();
 
 			state.scopes.push(enclosing.newSubdirectory("block" + currBlockNumber));
@@ -73,8 +73,7 @@ public class PrattParserTest {
 		ops.addAll(Arrays.asList("||", "&&"));
 		ops.addAll(Arrays.asList("<=", ">="));
 
-		ops.add(".");
-		ops.add(";");
+		ops.addAll(Arrays.asList(".", ",", ";", ":"));
 		ops.addAll(Arrays.asList("=", "<", ">"));
 		ops.addAll(Arrays.asList("+", "-", "*", "/"));
 		ops.addAll(Arrays.asList("^", "!"));
@@ -89,6 +88,8 @@ public class PrattParserTest {
 		reserved.addAll(Arrays.asList("if", "then", "else"));
 		reserved.addAll(Arrays.asList("and", "or"));
 		reserved.addAll(Arrays.asList("begin", "end"));
+		reserved.addAll(Arrays.asList("switch", "case"));
+		reserved.add("var");
 
 		TwoLevelSplitter split = new TwoLevelSplitter();
 
@@ -96,15 +97,17 @@ public class PrattParserTest {
 		split.addCompoundDelim("||", "&&");
 		split.addCompoundDelim("<=", ">=");
 
-		split.addSimpleDelim(".");
-		split.addSimpleDelim(";");
+		split.addSimpleDelim(".", ",", ";", ":");
 		split.addSimpleDelim("=", "<", ">");
 		split.addSimpleDelim("+", "-", "*", "/");
 		split.addSimpleDelim("^", "!");
+
 		split.addSimpleMulti("\\(", "\\)");
 		split.addSimpleMulti("\\[", "\\]");
 		split.addSimpleMulti("\\{", "\\}");
 
+		split.exclude(reserved.toArray(new String[0]));
+
 		split.compile();
 
 		PrattParser<String, String, TestContext> parser = createParser();
@@ -130,7 +133,7 @@ public class PrattParserTest {
 				ITree<Token<String, String>> tree = parser.parseExpression(0, tokenStream, ctx, true);
 
 				if(!tokenStream.current().getKey().equals("(end)")) {
-					System.out.println("Multipe expressions on line");
+					System.out.println("Multiple expressions on line");
 				}
 
 				System.out.println("Parsed expression:\n" + tree);
@@ -143,7 +146,7 @@ public class PrattParserTest {
 		}
 
 		System.out.println();
-		System.out.println("Context is:\n" + ctx);
+		System.out.println("Context is: " + ctx);
 
 		scn.close();
 	}
@@ -155,9 +158,22 @@ public class PrattParserTest {
 		List<String> splitTokens = new LinkedList<>();
 
 		for(String raw : rawTokens) {
-			String[] strangs = split.split(raw);
+			boolean doSplit = false;
 
-			splitTokens.addAll(Arrays.asList(strangs));
+			for(String op : ops) {
+				if(raw.contains(op)) {
+					doSplit = true;
+					break;
+				}
+			}
+
+			if(doSplit) {
+				String[] strangs = split.split(raw);
+
+				splitTokens.addAll(Arrays.asList(strangs));
+			} else {
+				splitTokens.add(raw);
+			}
 		}
 
 		System.out.println("Split string: " + splitTokens);
@@ -167,6 +183,8 @@ public class PrattParserTest {
 		Iterator<Token<String, String>> tokens = new TransformIterator<>(source, (String strang) -> {
 			if(ops.contains(strang) || reserved.contains(strang)) {
 				return new StringToken(strang, strang);
+			} else if(ctx.scopes.top().containsKey(strang)) {
+				return new StringToken("(vref)", strang);
 			} else {
 				return new StringToken("(literal)", strang);
 			}
@@ -188,9 +206,11 @@ public class PrattParserTest {
 
 		PrattParser<String, String, TestContext> parser = new PrattParser<>();
 
+		parser.addNonInitialCommand(":", infixNon(3));
+
 		parser.addNonInitialCommand("if", ternary(5, 0, "else", new StringToken("cond", "cond"), false));
 
-		parser.addNonInitialCommand(":=", infixNon(10));
+		parser.addNonInitialCommand(":=", new AssignCommand());
 
 		parser.addNonInitialCommand("and", infixLeft(13));
 		parser.addNonInitialCommand("or", infixLeft(13));
@@ -223,12 +243,25 @@ public class PrattParserTest {
 
 		parser.addInitialCommand("(", grouping(0, ")", new StringToken("parens", "parens")));
 
-		parser.addInitialCommand("{", delimited(0, ";", "}", new StringToken("block", "block"),
+		parser.addInitialCommand("begin", delimited(0, ";", "end", new StringToken("block", "block"),
 				new BlockEnter(), (state) -> state, new BlockExit(), true));
 
+		parser.addInitialCommand("[", delimited(0, ",", "]", new StringToken("array", "array"),
+				(state) -> state, (state) -> state, (state) -> state, false));
+
+		parser.addInitialCommand("{", delimited(0, ",", "}", new StringToken("json", "json"), (state) -> state,
+				(state) -> state, (state) -> state, false));
+
+		parser.addInitialCommand("case", unary(5));
+
 		parser.addInitialCommand("-", unary(30));
 
 		parser.addInitialCommand("(literal)", leaf());
+		parser.addInitialCommand("(vref)", leaf());
+
+		parser.addInitialCommand("var", new VarCommand());
+
+		parser.addInitialCommand("switch", new SwitchCommand());
 
 		return parser;
 	}
diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/SwitchCommand.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/SwitchCommand.java
new file mode 100644
index 0000000..7464709
--- /dev/null
+++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/SwitchCommand.java
@@ -0,0 +1,21 @@
+package bjc.utils.examples.parsing;
+
+import bjc.utils.data.ITree;
+import bjc.utils.data.Tree;
+import bjc.utils.parserutils.ParserException;
+import bjc.utils.parserutils.pratt.InitialCommand;
+import bjc.utils.parserutils.pratt.ParserContext;
+import bjc.utils.parserutils.pratt.StringToken;
+import bjc.utils.parserutils.pratt.Token;
+
+class SwitchCommand implements InitialCommand<String, String, TestContext> {
+	@Override
+	public ITree<Token<String, String>> denote(Token<String, String> operator,
+			ParserContext<String, String, TestContext> ctx) throws ParserException {
+		ITree<Token<String, String>> object = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
+
+		ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
+
+		return new Tree<>(new StringToken("switch", "switch"), object, body);
+	}
+}
diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java
index cc1be17..7f9986e 100644
--- a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java
+++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java
@@ -1,9 +1,11 @@
 package bjc.utils.examples.parsing;
 
+import bjc.utils.data.ITree;
 import bjc.utils.esodata.Directory;
 import bjc.utils.esodata.SimpleDirectory;
 import bjc.utils.esodata.SimpleStack;
 import bjc.utils.esodata.Stack;
+import bjc.utils.parserutils.pratt.Token;
 
 /**
  * Simple context for the parser.
@@ -15,7 +17,7 @@ public class TestContext {
 	/**
 	 * The variable scoping information.
 	 */
-	public Stack<Directory<String, String>> scopes;
+	public Stack<Directory<String, ITree<Token<String, String>>>> scopes;
 
 	/**
 	 * The current number of scopes inside this scope.
@@ -35,6 +37,6 @@ public class TestContext {
 
 	@Override
 	public String toString() {
-		return String.format("TestContext [scopes=%s, blockCount=%s]", scopes, blockCount);
+		return String.format("TestContext [scopes=%s\n, blockCount=%s]", scopes, blockCount);
 	}
 }
diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/VarCommand.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/VarCommand.java
new file mode 100644
index 0000000..b702f0e
--- /dev/null
+++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/VarCommand.java
@@ -0,0 +1,36 @@
+package bjc.utils.examples.parsing;
+
+import bjc.utils.data.ITree;
+import bjc.utils.data.Tree;
+import bjc.utils.parserutils.ParserException;
+import bjc.utils.parserutils.pratt.ParserContext;
+import bjc.utils.parserutils.pratt.StringToken;
+import bjc.utils.parserutils.pratt.Token;
+import bjc.utils.parserutils.pratt.commands.AbstractInitialCommand;
+
+class VarCommand extends AbstractInitialCommand<String, String, TestContext> {
+
+	@Override
+	protected ITree<Token<String, String>> intNullDenotation(Token<String, String> operator,
+			ParserContext<String, String, TestContext> ctx) throws ParserException {
+		Token<String, String> name = ctx.tokens.current();
+
+		switch(name.getKey()) {
+		case "(literal)":
+		case "(vref)":
+			ctx.tokens.next();
+			break;
+		default:
+			throw new ParserException("Variable name must be simple");
+		}
+
+		ctx.tokens.expect("=");
+
+		ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false);
+
+		ctx.state.scopes.top().putKey(name.getValue(), body);
+
+		return new Tree<>(new StringToken("var-bind", "var-bind"), new Tree<>(name), body);
+	}
+
+}