Basic parser + evaluator skeletonHEADtrunk

This fleshes out a basic parser w/ a calculator grammar and the skeleton
for an evaluator

5 files changed, 276 insertions, 33 deletions

diff --git a/winot/winot-core/src/main/java/com/ashardalon/winot/core/TokenType.java b/winot/winot-core/src/main/java/com/ashardalon/winot/core/TokenType.java
index 5387328..1757be5 100644
--- a/winot/winot-core/src/main/java/com/ashardalon/winot/core/TokenType.java
+++ b/winot/winot-core/src/main/java/com/ashardalon/winot/core/TokenType.java
@@ -12,6 +12,10 @@ public enum TokenType {
 	 */
 	TERMINAL,
 	/**
+	 * Result of evaluation
+	 */
+	RESULT,
+	/**
 	 * Raw token. Hasn't been fully processed yet
 	 */
 	RAW,
@@ -20,6 +24,19 @@ public enum TokenType {
 	 * 'def' special-object initiator
 	 */
 	DEF,
+	/**
+	 * + operator
+	 */
+	ADD,
+	/**
+	 * - operator
+	 */
+	MINUS,
+	/** / operator */
+	DIVIDE,
+	/** * operator */
+	MULTIPLY,
+	
 	// Represent literals of various types
 	/**
 	 * Double-quoted string.
diff --git a/winot/winot-core/src/main/java/com/ashardalon/winot/core/WinotApp.java b/winot/winot-core/src/main/java/com/ashardalon/winot/core/WinotApp.java
index 019ce49..fdf0e59 100644
--- a/winot/winot-core/src/main/java/com/ashardalon/winot/core/WinotApp.java
+++ b/winot/winot-core/src/main/java/com/ashardalon/winot/core/WinotApp.java
@@ -2,14 +2,14 @@ package com.ashardalon.winot.core;
 
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
+import java.util.HashMap;
+import java.util.Map;
 
 import bjc.data.*;
 import bjc.pratt.*;
-import bjc.pratt.commands.InitialCommand;
 import bjc.pratt.commands.impls.InitialCommands;
-import bjc.pratt.tokens.*;
+import bjc.pratt.commands.impls.NonInitialCommands;
 import bjc.utils.cli.StreamTerminal;
-import bjc.utils.parserutils.*;
 import bjc.utils.parserutils.splitter.*;
 
 /**
@@ -24,17 +24,30 @@ public class WinotApp {
 	 * @param args Currently unused CLI args
 	 */
 	public static void main(String[] args) {
+		Map<String, TokenType> operators = new HashMap<>();
+		
+		operators.put("def", TokenType.DEF);
+		
+		operators.put("+", TokenType.ADD);
+		operators.put("*", TokenType.MULTIPLY);
+		operators.put("-", TokenType.MINUS);
+		operators.put("/", TokenType.DIVIDE);
+		
 		ConfigurableTokenSplitter baseSplitter = new ConfigurableTokenSplitter(true);
-		baseSplitter.addMultiDelimiters(" ");
+		
+		//baseSplitter.addMultiDelimiters(" ");
+		
+		baseSplitter.addSimpleDelimiters("+", "-", "*", "/");
+		
 		baseSplitter.compile();
 
 		ExcludingTokenSplitter exclSplitter = new ExcludingTokenSplitter(baseSplitter);
 
-		FilteredTokenSplitter filtSplitter = new FilteredTokenSplitter(exclSplitter, (tok) -> !tok.equals(""));
+		FilteredTokenSplitter filtSplitter = new FilteredTokenSplitter(exclSplitter, (tok) -> !tok.equals("") || !tok.equals(" "));
 
 		WinotContext state = new WinotContext();
 
-		PrattParser<TokenType, String, WinotContext> parser = new PrattParser<>();
+		PrattParser<TokenType, WinotValue, WinotContext> parser = new PrattParser<>();
 
 		parser.addInitialCommand(TokenType.LITERAL, InitialCommands.leaf());
 		parser.addInitialCommand(TokenType.DQ_STR, InitialCommands.leaf());
@@ -42,14 +55,20 @@ public class WinotApp {
 		
 		// using 'var' because otherwise the type is some 40+ characters involving repeating the command type twice
 		// Also, have to specify args to 'branch' since the compiler can't guess them otherwise
-		var defBuilder = InitialCommands.<TokenType, String, WinotContext>branch();
-		InitialCommand<TokenType, String, WinotContext> defCommand = defBuilder.add(TokenType.DEF, null).build();
+		var defBuilder = InitialCommands.<TokenType, WinotValue, WinotContext>branch();
+		var defCommand = defBuilder.add(TokenType.DEF, null).build();
 		parser.addInitialCommand(TokenType.DEF, defCommand);
+		
+		parser.addNonInitialCommand(TokenType.ADD, NonInitialCommands.infixLeft(10));
+		parser.addNonInitialCommand(TokenType.MINUS, NonInitialCommands.infixLeft(10));
+		parser.addNonInitialCommand(TokenType.MULTIPLY, NonInitialCommands.infixLeft(11));
+		parser.addNonInitialCommand(TokenType.DIVIDE, NonInitialCommands.infixLeft(11));
 
 		StreamTerminal terminal = new StreamTerminal(new InputStreamReader(System.in),
 				new OutputStreamWriter(System.out), "/", null);
 		IntHolder comNo = new IntHolder();
-		terminal.setMode(new WinotMainMode(terminal, comNo, parser, state, filtSplitter));
+		WinotMainMode mode = new WinotMainMode(terminal, comNo, parser, state, filtSplitter, operators);
+		terminal.setMode(mode);
 
 		terminal.addOutput("0 > ");
 		terminal.run();
diff --git a/winot/winot-core/src/main/java/com/ashardalon/winot/core/WinotMainMode.java b/winot/winot-core/src/main/java/com/ashardalon/winot/core/WinotMainMode.java
index ce8b9ef..1b520df 100644
--- a/winot/winot-core/src/main/java/com/ashardalon/winot/core/WinotMainMode.java
+++ b/winot/winot-core/src/main/java/com/ashardalon/winot/core/WinotMainMode.java
@@ -2,12 +2,15 @@ package com.ashardalon.winot.core;
 
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.function.Consumer;
 import java.util.function.Function;
 
 import bjc.data.*;
+import bjc.funcdata.ListEx;
 import bjc.pratt.PrattParser;
 import bjc.pratt.commands.CommandResult;
+import bjc.pratt.commands.CommandResult.Status;
 import bjc.pratt.tokens.*;
 import bjc.utils.cli.StreamTerminal;
 import bjc.utils.parserutils.ParserException;
@@ -17,19 +20,21 @@ import bjc.utils.parserutils.splitter.FilteredTokenSplitter;
 final class WinotMainMode implements Consumer<String> {
 	private final StreamTerminal terminal;
 	private final IntHolder comNo;
-	private final PrattParser<TokenType, String, WinotContext> parser;
+	private final PrattParser<TokenType, WinotValue, WinotContext> parser;
 	private final WinotContext state;
 	private final FilteredTokenSplitter filtSplitter;
+	private final Map<String, TokenType> operators;
 
-	WinotMainMode(StreamTerminal terminal, IntHolder comNo, PrattParser<TokenType, String, WinotContext> parser,
-			WinotContext state, FilteredTokenSplitter filtSplitter) {
+	WinotMainMode(StreamTerminal terminal, IntHolder comNo, PrattParser<TokenType, WinotValue, WinotContext> parser,
+			WinotContext state, FilteredTokenSplitter filtSplitter, Map<String, TokenType> operators) {
 		this.terminal = terminal;
 		this.comNo = comNo;
 		this.parser = parser;
 		this.state = state;
 		this.filtSplitter = filtSplitter;
+		this.operators = operators;
 	}
-
+	
 	@Override
 	public void accept(String str) {
 		List<String> rawTokens = TokenUtils.removeDQuotedStrings(str);
@@ -37,31 +42,65 @@ final class WinotMainMode implements Consumer<String> {
 		Function<String, Token<TokenType, String>> tokenzer = (strang) -> {
 			return new SimpleToken<>(strToggle.get(), strang);
 		};
-		TransformIterator<String,
-				Token<TokenType, String>> rawItr = new TransformIterator<>(rawTokens.iterator(), tokenzer);
-		FlatMapIterator<Token<TokenType, String>,
-				Token<TokenType, String>> mapItr = new FlatMapIterator<>(rawItr, (tok) -> {
-					if (tok.getKey() == TokenType.RAW) {
-						Iterator<String> tknItr = filtSplitter.split(tok.getValue()).toIterable().iterator();
-						Function<String, Token<TokenType, String>> literizer = (tokn) -> {
-							return new SimpleToken<>(TokenType.LITERAL, tokn);
-						};
-						TransformIterator<String,
-								Token<TokenType, String>> transf = new TransformIterator<>(tknItr, literizer);
-						return transf;
-					}
-					return new SingleIterator<>(tok);
-				});
-		SimpleTokenStream<TokenType,
-				String> sts = new SimpleTokenStream<>(mapItr, new SimpleToken<>(TokenType.TERMINAL, ""));
+		var rawItr = new TransformIterator<String, Token<TokenType, String>>(rawTokens.iterator(), tokenzer);
+		var mapItr = new FlatMapIterator<Token<TokenType, String>, Token<TokenType, WinotValue>>(rawItr, this::toLiteral);
+		var sts = new SimpleTokenStream<TokenType, WinotValue>(mapItr, new SimpleToken<>(TokenType.TERMINAL, WinotValue.EMPTY));
 		try {
 			// Prime token stream
 			sts.next();
-			CommandResult<TokenType, String> result = parser.parseExpression(0, sts, state, true);
-			terminal.addOutput(result.toString() + "\n");
+			CommandResult<TokenType, WinotValue> result = parser.parseExpression(0, sts, state, true);
+			terminal.addOutput("Parsed:\n" + result.toString() + "\n");
+			if (result.status == Status.SUCCESS) {
+				var evalRes = result.success().topDownTransform((tok) -> {
+					// Default to evaluating bottom up
+					return TopDownTransformResult.PUSHDOWN;
+				}, (tree) -> {
+					Token<TokenType,WinotValue> head = tree.getHead();
+					switch (head.getKey()) {
+					case LITERAL:
+						SimpleToken<TokenType, WinotValue> res = new SimpleToken<>(TokenType.RESULT, head.getValue());
+						return new SimpleTree<>(res);
+					default:
+						throw new UnsupportedOperationException("Unsupport eval token: " + head.getKey());
+					}
+				});
+				terminal.addOutput("Evaluated to: " + evalRes);
+			}
+			if (sts.hasNext()) {
+				terminal.addOutput("Remaining tokens:\n");
+				sts.forEachRemaining((tok) -> {
+					terminal.addOutput("\t" + tok.toString() + "\n");
+				});
+			}
 		} catch (ParserException pex) {
 			terminal.addOutput(pex.toString() + "\n");
 		}
 		terminal.addOutput(String.format("%d > ", comNo.incr()));
 	}
+
+	private Iterator<Token<TokenType, WinotValue>> toLiteral(Token<TokenType, String> tok) {
+		if (tok.getKey() == TokenType.RAW) {
+			ListEx<String> splist = filtSplitter.split(tok.getValue());
+			Iterator<String> tknItr = splist.toIterable().iterator();
+			TransformIterator<String, String> splitr;
+			splitr = new TransformIterator<String, String>(tknItr, String::trim);
+			
+			Function<String, Token<TokenType, WinotValue>> literizer;
+			literizer = (tokn) -> {
+				if (operators.containsKey(tokn)) {
+					return new SimpleToken<>(operators.get(tokn), WinotValue.EMPTY);
+				} else if (tokn.matches("\\d+")) {
+					WinotValue intVal = WinotValue.integer(Integer.parseInt(tokn));
+					return new SimpleToken<>(TokenType.LITERAL, intVal);
+				} else if (TokenUtils.isDouble(tokn)) {
+					WinotValue doubleVal = WinotValue.fpLit(Double.parseDouble(tokn));
+					return new SimpleToken<>(TokenType.LITERAL, doubleVal);
+				}
+				return new SimpleToken<>(TokenType.LITERAL, WinotValue.string(tokn));
+			};
+			var transf = new TransformIterator<String, Token<TokenType, WinotValue>>(splitr, literizer);
+			return transf;
+		}
+		return new SingleIterator<>(new SimpleToken<>(tok.getKey(), WinotValue.string(tok.getValue())));
+	}
 }
\ No newline at end of file
diff --git a/winot/winot-core/src/main/java/com/ashardalon/winot/core/WinotValue.java b/winot/winot-core/src/main/java/com/ashardalon/winot/core/WinotValue.java
new file mode 100644
index 0000000..79b57e9
--- /dev/null
+++ b/winot/winot-core/src/main/java/com/ashardalon/winot/core/WinotValue.java
@@ -0,0 +1,168 @@
+package com.ashardalon.winot.core;
+
+import java.util.Objects;
+
+public class WinotValue {
+	public enum Type {
+		/**
+		 * A value that contains nothing
+		 */
+		EMPTY,
+		/**
+		 * A string
+		 */
+		STRING,
+		/**
+		 * An int
+		 */
+		INT,
+		/**
+		 * A double
+		 */
+		DOUBLE
+	}
+
+	public static final class IntegerWinotValue extends WinotValue {
+		public int val;
+
+		public IntegerWinotValue(int val) {
+			super(Type.INT);
+			this.val = val;
+		}
+
+		@Override
+		public int hashCode() {
+			return Objects.hash(val);
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			if (this == obj)
+				return true;
+			if (obj == null)
+				return false;
+			if (getClass() != obj.getClass())
+				return false;
+			IntegerWinotValue other = (IntegerWinotValue) obj;
+			return val == other.val;
+		}
+
+		@Override
+		public String toString() {
+			return "IntegerWinotValue [val=" + val + "]";
+		}
+	}
+	
+	public static final class DoubleWinotValue extends WinotValue {
+		public double val;
+
+		public DoubleWinotValue(double val) {
+			super(Type.DOUBLE);
+			this.val = val;
+		}
+
+		@Override
+		public int hashCode() {
+			return Objects.hash(val);
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			if (this == obj)
+				return true;
+			if (obj == null)
+				return false;
+			if (getClass() != obj.getClass())
+				return false;
+			DoubleWinotValue other = (DoubleWinotValue) obj;
+			return Double.doubleToLongBits(val) == Double.doubleToLongBits(other.val);
+		}
+
+		@Override
+		public String toString() {
+			return "DoubleWinotValue [val=" + val + "]";
+		}
+	}
+
+	public static class TerminalWinotValue extends WinotValue {
+		public TerminalWinotValue() {
+			super(Type.EMPTY);
+		}
+
+		@Override
+		public String toString() {
+			return "TerminalWinotValue []";
+		}
+	}
+	
+	public static class StringWinotValue extends WinotValue {
+		private String val;
+
+		public StringWinotValue(String val) {
+			super(Type.STRING);
+			this.val = val;		
+		}
+
+		@Override
+		public int hashCode() {
+			return Objects.hash(val);
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			if (this == obj)
+				return true;
+			if (obj == null)
+				return false;
+			if (getClass() != obj.getClass())
+				return false;
+			StringWinotValue other = (StringWinotValue) obj;
+			return Objects.equals(val, other.val);
+		}
+
+		@Override
+		public String toString() {
+			return "StringWinotValue [val=" + val + "]";
+		}
+	}
+	
+	public final Type type;
+	
+	protected WinotValue(Type typ) {
+		this.type = typ;
+	}
+	
+	public static WinotValue EMPTY = new TerminalWinotValue();
+	
+	
+	public static WinotValue string(String val) {
+		return new StringWinotValue(val);
+	}
+
+
+	public static WinotValue integer(int val) {
+		return new IntegerWinotValue(val);
+	}
+
+	public static WinotValue fpLit(double val) {
+		return new DoubleWinotValue(val);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(type);
+	}
+
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		WinotValue other = (WinotValue) obj;
+		return type == other.type;
+	}
+}
\ No newline at end of file
diff --git a/winot/winot-core/src/main/java/module-info.java b/winot/winot-core/src/main/java/module-info.java
index 96194a7..c0addf7 100644
--- a/winot/winot-core/src/main/java/module-info.java
+++ b/winot/winot-core/src/main/java/module-info.java
@@ -5,6 +5,6 @@
  */
 module com.ashardalon.winot.core {
 	requires bjc.utils;
-	requires jpratt;
 	requires esodata;
+	requires jpratt;
 }
\ No newline at end of file