diff options
10 files changed, 305 insertions, 72 deletions
diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java index 6b97c19..522bf6f 100644 --- a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java +++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java @@ -4,6 +4,8 @@ import bjc.utils.data.ITree; import bjc.utils.data.TransformIterator; import bjc.utils.esodata.Directory; import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.InitialCommand; +import bjc.utils.parserutils.pratt.NonInitialCommand; import bjc.utils.parserutils.pratt.PrattParser; import bjc.utils.parserutils.pratt.Token; import bjc.utils.parserutils.pratt.tokens.StringToken; @@ -19,11 +21,16 @@ import java.util.LinkedList; import java.util.List; import java.util.Scanner; import java.util.Set; +import java.util.function.Function; import java.util.function.UnaryOperator; import static bjc.utils.parserutils.pratt.commands.NonInitialCommands.*; import static bjc.utils.parserutils.pratt.commands.InitialCommands.*; +import static bjc.utils.parserutils.pratt.tokens.StringToken.litToken; + +import static bjc.utils.functypes.ID.id; + /** * Simple test for Pratt parser. * @@ -31,6 +38,29 @@ import static bjc.utils.parserutils.pratt.commands.InitialCommands.*; * */ public class PrattParserTest { + private static final class Tokenizer implements Function<String, Token<String, String>> { + private Set<String> ops; + private Set<String> reserved; + private TestContext ctx; + + public Tokenizer(Set<String> operators, Set<String> reservedWords, TestContext context) { + ops = operators; + reserved = reservedWords; + ctx = context; + } + + @Override + public Token<String, String> apply(String strang) { + if (ops.contains(strang) || reserved.contains(strang)) { + return litToken(strang); + } else if (ctx.scopes.top().containsKey(strang)) { + return new StringToken("(vref)", strang); + } else { + return new StringToken("(literal)", strang); + } + } + } + private static final class BlockExit implements UnaryOperator<TestContext> { @Override public TestContext apply(TestContext state) { @@ -119,7 +149,7 @@ public class PrattParserTest { System.out.print("Enter a command (blank line to exit): "); String ln = scn.nextLine(); - while(!ln.trim().equals("")) { + while (!ln.trim().equals("")) { Iterator<Token<String, String>> tokens = preprocessInput(ops, split, ln, reserved, ctx); try { @@ -132,12 +162,12 @@ public class PrattParserTest { ITree<Token<String, String>> tree = parser.parseExpression(0, tokenStream, ctx, true); - if(!tokenStream.current().getKey().equals("(end)")) { + if (!tokenStream.headIs("(end)")) { System.out.println("Multiple expressions on line"); } System.out.println("Parsed expression:\n" + tree); - } catch(ParserException pex) { + } catch (ParserException pex) { pex.printStackTrace(); } @@ -157,17 +187,17 @@ public class PrattParserTest { List<String> splitTokens = new LinkedList<>(); - for(String raw : rawTokens) { + for (String raw : rawTokens) { boolean doSplit = false; - for(String op : ops) { - if(raw.contains(op)) { + for (String op : ops) { + if (raw.contains(op)) { doSplit = true; break; } } - if(doSplit) { + if (doSplit) { String[] strangs = split.split(raw); splitTokens.addAll(Arrays.asList(strangs)); @@ -180,15 +210,8 @@ public class PrattParserTest { Iterator<String> source = splitTokens.iterator(); - Iterator<Token<String, String>> tokens = new TransformIterator<>(source, (String strang) -> { - if(ops.contains(strang) || reserved.contains(strang)) { - return new StringToken(strang, strang); - } else if(ctx.scopes.top().containsKey(strang)) { - return new StringToken("(vref)", strang); - } else { - return new StringToken("(literal)", strang); - } - }); + Iterator<Token<String, String>> tokens = new TransformIterator<>(source, + new Tokenizer(ops, reserved, ctx)); return tokens; } @@ -202,33 +225,43 @@ public class PrattParserTest { /* * Token for marking chains. */ - StringToken chainToken = new StringToken("and", "and"); + StringToken chainToken = litToken("and"); + + /* + * ID function. + */ + UnaryOperator<TestContext> idfun = id(); PrattParser<String, String, TestContext> parser = new PrattParser<>(); parser.addNonInitialCommand(":", infixNon(3)); - parser.addNonInitialCommand("if", ternary(5, 0, "else", new StringToken("cond", "cond"), false)); + parser.addNonInitialCommand("if", ternary(5, 0, "else", litToken("cond"), false)); parser.addNonInitialCommand(":=", new AssignCommand()); - parser.addNonInitialCommand("and", infixLeft(13)); - parser.addNonInitialCommand("or", infixLeft(13)); + NonInitialCommand<String, String, TestContext> nonSSRelJoin = infixLeft(13); + parser.addNonInitialCommand("and", nonSSRelJoin); + parser.addNonInitialCommand("or", nonSSRelJoin); - parser.addNonInitialCommand("=", chain(15, relChain, chainToken)); - parser.addNonInitialCommand("<", chain(15, relChain, chainToken)); - parser.addNonInitialCommand(">", chain(15, relChain, chainToken)); - parser.addNonInitialCommand("<=", chain(15, relChain, chainToken)); - parser.addNonInitialCommand(">=", chain(15, relChain, chainToken)); + NonInitialCommand<String, String, TestContext> chainRelOp = chain(15, relChain, chainToken); + parser.addNonInitialCommand("=", chainRelOp); + parser.addNonInitialCommand("<", chainRelOp); + parser.addNonInitialCommand(">", chainRelOp); + parser.addNonInitialCommand("<=", chainRelOp); + parser.addNonInitialCommand(">=", chainRelOp); - parser.addNonInitialCommand("&&", infixRight(17)); - parser.addNonInitialCommand("||", infixRight(17)); + NonInitialCommand<String, String, TestContext> ssRelJoin = infixRight(17); + parser.addNonInitialCommand("&&", ssRelJoin); + parser.addNonInitialCommand("||", ssRelJoin); - parser.addNonInitialCommand("+", infixLeft(20)); - parser.addNonInitialCommand("-", infixLeft(20)); + NonInitialCommand<String, String, TestContext> addSub = infixLeft(20); + parser.addNonInitialCommand("+", addSub); + parser.addNonInitialCommand("-", addSub); - parser.addNonInitialCommand("*", infixLeft(30)); - parser.addNonInitialCommand("/", infixLeft(30)); + NonInitialCommand<String, String, TestContext> mulDiv = infixLeft(30); + parser.addNonInitialCommand("*", mulDiv); + parser.addNonInitialCommand("/", mulDiv); parser.addNonInitialCommand("!", postfix(40)); @@ -236,28 +269,26 @@ public class PrattParserTest { parser.addNonInitialCommand(".", infixLeft(60)); - parser.addNonInitialCommand("[", postCircumfix(60, 0, "]", new StringToken("idx", "idx"))); + parser.addNonInitialCommand("[", postCircumfix(60, 0, "]", litToken("idx"))); - parser.addInitialCommand("if", - preTernary(0, 0, 0, "then", "else", new StringToken("ifelse", "ifelse"))); + parser.addInitialCommand("if", preTernary(0, 0, 0, "then", "else", litToken("ifelse"))); - parser.addInitialCommand("(", grouping(0, ")", new StringToken("parens", "parens"))); + parser.addInitialCommand("(", grouping(0, ")", litToken("parens"))); - parser.addInitialCommand("begin", delimited(0, ";", "end", new StringToken("block", "block"), - new BlockEnter(), (state) -> state, new BlockExit(), true)); + parser.addInitialCommand("begin", delimited(0, ";", "end", litToken("block"), new BlockEnter(), idfun, + new BlockExit(), true)); - parser.addInitialCommand("[", delimited(0, ",", "]", new StringToken("array", "array"), - (state) -> state, (state) -> state, (state) -> state, false)); + parser.addInitialCommand("[", delimited(0, ",", "]", litToken("array"), idfun, idfun, idfun, false)); - parser.addInitialCommand("{", delimited(0, ",", "}", new StringToken("json", "json"), (state) -> state, - (state) -> state, (state) -> state, false)); + parser.addInitialCommand("{", delimited(0, ",", "}", litToken("json"), idfun, idfun, idfun, false)); parser.addInitialCommand("case", unary(5)); parser.addInitialCommand("-", unary(30)); - parser.addInitialCommand("(literal)", leaf()); - parser.addInitialCommand("(vref)", leaf()); + InitialCommand<String, String, TestContext> leaf = leaf(); + parser.addInitialCommand("(literal)", leaf); + parser.addInitialCommand("(vref)", leaf); parser.addInitialCommand("var", new VarCommand()); @@ -265,4 +296,4 @@ public class PrattParserTest { return parser; } -} +}
\ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/functypes/ID.java b/BJC-Utils2/src/main/java/bjc/utils/functypes/ID.java new file mode 100644 index 0000000..af72d1a --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/functypes/ID.java @@ -0,0 +1,9 @@ +package bjc.utils.functypes;
+
+import java.util.function.UnaryOperator;
+
+public class ID {
+ public static <A> UnaryOperator<A> id() {
+ return (x) -> x;
+ }
+}
diff --git a/BJC-Utils2/src/main/java/bjc/utils/ioutils/BlockReaders.java b/BJC-Utils2/src/main/java/bjc/utils/ioutils/BlockReaders.java new file mode 100644 index 0000000..ec8dcfb --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/ioutils/BlockReaders.java @@ -0,0 +1,26 @@ +package bjc.utils.ioutils;
+
+import java.io.Reader;
+
+/**
+ * Utility methods for constructing instances of {@link BlockReader}
+ *
+ * @author bjculkin
+ *
+ */
+public class BlockReaders {
+ /**
+ * Create a new simple block reader that works off a regex.
+ *
+ * @param blockDelim
+ * The regex that seperates blocks.
+ *
+ * @param source
+ * The reader to get blocks from.
+ *
+ * @return A configured simple reader.
+ */
+ public static BlockReader simple(String blockDelim, Reader source) {
+ return new SimpleBlockReader(blockDelim, source);
+ }
+}
diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java index 20572e8..c53b4e1 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java @@ -26,18 +26,37 @@ import java.util.Map; * */ public class PrattParser<K, V, C> { + /* + * Default commands that error when used. + */ private final NonInitialCommand<K, V, C> DEFAULT_LEFT_COMMAND = new DefaultNonInitialCommand<>(); - private final InitialCommand<K, V, C> DEFAULT_NULL_COMMAND = new DefaultInitialCommand<>(); + private final InitialCommand<K, V, C> DEFAULT_NULL_COMMAND = new DefaultInitialCommand<>(); + + /* + * Left-commands that depend on what the null command was. + */ + private Map<K, Map<K, NonInitialCommand<K, V, C>>> dependantLeftCommands; - private Map<K, NonInitialCommand<K, V, C>> leftCommands; - private Map<K, InitialCommand<K, V, C>> nullCommands; - private Map<K, InitialCommand<K, V, C>> statementCommands; + /* + * The left commands. + */ + private Map<K, NonInitialCommand<K, V, C>> leftCommands; + /* + * The initial commands. + */ + private Map<K, InitialCommand<K, V, C>> nullCommands; + /* + * Initial commands only checked for statements. + */ + private Map<K, InitialCommand<K, V, C>> statementCommands; /** * Create a new Pratt parser. * */ public PrattParser() { + dependantLeftCommands = new HashMap<>(); + leftCommands = new HashMap<>(); nullCommands = new HashMap<>(); statementCommands = new HashMap<>(); @@ -65,34 +84,41 @@ public class PrattParser<K, V, C> { */ public ITree<Token<K, V>> parseExpression(int precedence, TokenStream<K, V> tokens, C state, boolean isStatement) throws ParserException { - if(precedence < 0) { + if (precedence < 0) { throw new IllegalArgumentException("Precedence must be greater than zero"); } Token<K, V> initToken = tokens.current(); tokens.next(); + K initKey = initToken.getKey(); + ITree<Token<K, V>> ast; - if(isStatement && statementCommands.containsKey(initToken.getKey())) { - ast = statementCommands.getOrDefault(initToken.getKey(), DEFAULT_NULL_COMMAND) - .denote(initToken, new ParserContext<>(tokens, this, state)); + if (isStatement && statementCommands.containsKey(initKey)) { + ast = statementCommands.getOrDefault(initKey, DEFAULT_NULL_COMMAND).denote(initToken, + new ParserContext<>(tokens, this, state)); } else { - ast = nullCommands.getOrDefault(initToken.getKey(), DEFAULT_NULL_COMMAND) - .denote(initToken, new ParserContext<>(tokens, this, state)); + ast = nullCommands.getOrDefault(initKey, DEFAULT_NULL_COMMAND).denote(initToken, + new ParserContext<>(tokens, this, state)); } int rightPrec = Integer.MAX_VALUE; - while(true) { + while (true) { Token<K, V> tok = tokens.current(); K key = tok.getKey(); NonInitialCommand<K, V, C> command = leftCommands.getOrDefault(key, DEFAULT_LEFT_COMMAND); + + if (dependantLeftCommands.containsKey(initKey)) { + command = dependantLeftCommands.get(initKey).getOrDefault(key, command); + } + int leftBind = command.leftBinding(); - if(NumberUtils.between(precedence, rightPrec, leftBind)) { + if (NumberUtils.between(precedence, rightPrec, leftBind)) { tokens.next(); ast = command.denote(ast, tok, new ParserContext<>(tokens, this, state)); @@ -146,4 +172,19 @@ public class PrattParser<K, V, C> { public void addStatementCommand(K marker, InitialCommand<K, V, C> comm) { statementCommands.put(marker, comm); } -} + + /** + * Add a dependant non-initial command to this parser. + */ + public void addDependantCommand(K dependant, K marker, NonInitialCommand<K, V, C> comm) { + if (dependantLeftCommands.containsKey(dependant)) { + dependantLeftCommands.get(dependant).put(marker, comm); + } else { + Map<K, NonInitialCommand<K, V, C>> comms = new HashMap<>(); + + comms.put(marker, comm); + + dependantLeftCommands.put(dependant, comms); + } + } +}
\ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/TokenStream.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/TokenStream.java index 71482b4..db7d1e0 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/TokenStream.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/TokenStream.java @@ -86,4 +86,8 @@ public abstract class TokenStream<K, V> implements Iterator<Token<K, V>> { public final void expect(K... expectedKeys) throws ExpectationException { expect(new HashSet<>(Arrays.asList(expectedKeys))); } + + public boolean headIs(K val) { + return current().getKey().equals(val); + } } diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/blocks/ParseBlocks.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/blocks/ParseBlocks.java index 43163e6..9df8355 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/blocks/ParseBlocks.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/blocks/ParseBlocks.java @@ -1,6 +1,11 @@ package bjc.utils.parserutils.pratt.blocks; +import java.util.function.Predicate; +import java.util.function.UnaryOperator; + +import bjc.utils.data.ITree; import bjc.utils.parserutils.pratt.ParseBlock; +import bjc.utils.parserutils.pratt.Token; /** * Utility class for creating common implementations of {@link ParseBlock} @@ -9,5 +14,71 @@ import bjc.utils.parserutils.pratt.ParseBlock; * */ public class ParseBlocks { + /** + * Create a new repeating parse block. + * + * @param inner + * The parse block to repeat. + * + * @param delim + * The token type that seperates repetitions. + * + * @param term + * The token type that terminates repititions. + * + * @param mark + * The token to use as the node in the AST. + * + * @param action + * The action to perform on the state after every + * repitition. + * + * @return A configured repeating parse block. + */ + public static <K, V, C> ParseBlock<K, V, C> repeating(ParseBlock<K, V, C> inner, K delim, K term, + Token<K, V> mark, UnaryOperator<C> action) { + return new RepeatingParseBlock<>(inner, delim, term, mark, action); + } + + /** + * Create a new triggered parse block. + * + * @param source + * The block to trigger around. + * + * @param onEnter + * The action to perform upon the state before entering + * the block. + * + * @param onExit + * The action to perform upon the state after exiting the + * block. + * + * @return A configured trigger parse block. + */ + public static <K, V, C> ParseBlock<K, V, C> trigger(ParseBlock<K, V, C> source, UnaryOperator<C> onEnter, + UnaryOperator<C> onExit) { + return new TriggeredParseBlock<>(onEnter, onExit, source); + } -} + /** + * Create a new simple parse block. + * + * @param precedence + * The precedence of the expression inside the block. + * + * @param terminator + * The key type of the token expected after this block, + * or null if none is expected. + * + * @param validator + * The predicate to use to validate parsed expressions, + * or null if none is used. + * + * @return A configured simple parse block. + */ + public static <K, V, C> ParseBlock<K, V, C> simple(int precedence, K terminator, + Predicate<ITree<Token<K, V>>> validator) { + return new SimpleParseBlock<>(precedence, terminator, validator); + } +}
\ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DenestingCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DenestingCommand.java new file mode 100644 index 0000000..d2d7829 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DenestingCommand.java @@ -0,0 +1,31 @@ +package bjc.utils.parserutils.pratt.commands;
+
+import bjc.utils.data.ITree;
+import bjc.utils.parserutils.ParserException;
+import bjc.utils.parserutils.pratt.InitialCommand;
+import bjc.utils.parserutils.pratt.ParserContext;
+import bjc.utils.parserutils.pratt.Token;
+
+public class DenestingCommand<C, V, K> extends AbstractInitialCommand<K, V, C> {
+ private InitialCommand<K, V, C> internal;
+
+ /**
+ * Create a new transforming initial command.
+ *
+ * @param internal
+ * The initial command to delegate to.
+ *
+ * @param transform
+ * The transform to apply to the returned tree.
+ */
+ public DenestingCommand(InitialCommand<K, V, C> internal) {
+ super();
+ this.internal = internal;
+ }
+
+ @Override
+ protected ITree<Token<K, V>> intNullDenotation(Token<K, V> operator, ParserContext<K, V, C> ctx)
+ throws ParserException {
+ return internal.denote(operator, ctx).getChild(0);
+ }
+}
\ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java index bad5964..eac357a 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java @@ -4,12 +4,11 @@ import bjc.utils.data.ITree; import bjc.utils.parserutils.pratt.InitialCommand; import bjc.utils.parserutils.pratt.ParseBlock; import bjc.utils.parserutils.pratt.Token; -import bjc.utils.parserutils.pratt.blocks.RepeatingParseBlock; -import bjc.utils.parserutils.pratt.blocks.SimpleParseBlock; -import bjc.utils.parserutils.pratt.blocks.TriggeredParseBlock; import java.util.function.UnaryOperator; +import static bjc.utils.parserutils.pratt.blocks.ParseBlocks.*; + /** * * Contains factory methods for producing common implementations of * {@link InitialCommand} @@ -45,7 +44,7 @@ public class InitialCommands { * @return A command implementing the operator. */ public static <K, V, C> InitialCommand<K, V, C> grouping(int precedence, K term, Token<K, V> mark) { - ParseBlock<K, V, C> innerBlock = new SimpleParseBlock<>(precedence, term, null); + ParseBlock<K, V, C> innerBlock = simple(precedence, term, null); return new GroupingCommand<>(innerBlock, mark); } @@ -84,9 +83,9 @@ public class InitialCommands { */ public static <K, V, C> InitialCommand<K, V, C> preTernary(int cond1, int block1, int block2, K mark1, K mark2, Token<K, V> term) { - ParseBlock<K, V, C> condBlock = new SimpleParseBlock<>(cond1, mark1, null); - ParseBlock<K, V, C> opblock1 = new SimpleParseBlock<>(block1, mark2, null); - ParseBlock<K, V, C> opblock2 = new SimpleParseBlock<>(block2, null, null); + ParseBlock<K, V, C> condBlock = simple(cond1, mark1, null); + ParseBlock<K, V, C> opblock1 = simple(block1, mark2, null); + ParseBlock<K, V, C> opblock2 = simple(block2, null, null); return new PreTernaryCommand<>(condBlock, opblock1, opblock2, term); } @@ -139,9 +138,9 @@ public class InitialCommands { public static <K, V, C> InitialCommand<K, V, C> delimited(int inner, K delim, K mark, Token<K, V> term, UnaryOperator<C> onEnter, UnaryOperator<C> onDelim, UnaryOperator<C> onExit, boolean statement) { - ParseBlock<K, V, C> innerBlock = new SimpleParseBlock<>(inner, null, null); - ParseBlock<K, V, C> delimsBlock = new RepeatingParseBlock<>(innerBlock, delim, mark, term, onDelim); - ParseBlock<K, V, C> scopedBlock = new TriggeredParseBlock<>(onEnter, onExit, delimsBlock); + ParseBlock<K, V, C> innerBlock = simple(inner, null, null); + ParseBlock<K, V, C> delimsBlock = repeating(innerBlock, delim, mark, term, onDelim); + ParseBlock<K, V, C> scopedBlock = trigger(delimsBlock, onEnter, onExit); GroupingCommand<K, V, C> command = new GroupingCommand<>(scopedBlock, term); @@ -149,6 +148,21 @@ public class InitialCommands { * Remove the wrapper layer from grouping-command on top of * RepeatingParseBlock. */ - return new TransformingInitialCommand<>(command, (tree) -> tree.getChild(0)); + return denest(command); + } + + /** + * Create a new denesting command. + * + * This removes one tree-level, and is useful when combining complex + * parse blocks with commands. + * + * @param comm + * The command to denest. + * + * @return A command that denests the result of the provided command. + */ + public static <K, V, C> InitialCommand<K, V, C> denest(InitialCommand<K, V, C> comm) { + return new DenestingCommand<>(comm); } } diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringToken.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringToken.java index 7e779aa..f156f02 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringToken.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringToken.java @@ -77,4 +77,8 @@ public class StringToken implements Token<String, String> { public String toString() { return String.format("StringToken [key='%s', val='%s']", key, val); } + + public static StringToken litToken(String val) { + return new StringToken(val, val); + } } diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringTokenStream.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringTokenStream.java index d7d4a66..75e86c4 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringTokenStream.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringTokenStream.java @@ -5,11 +5,13 @@ import java.util.Iterator; import bjc.utils.parserutils.pratt.Token; import bjc.utils.parserutils.pratt.TokenStream; +import static bjc.utils.parserutils.pratt.tokens.StringToken.litToken; + /** * Simple implementation of token stream for strings. * - * The terminal token here is represented by a token with type '(end)' and null - * value. + * The terminal token here is represented by a token with type and value + * '(end)'. * * @author EVE * @@ -41,7 +43,7 @@ public class StringTokenStream extends TokenStream<String, String> { if (iter.hasNext()) { curr = iter.next(); } else { - curr = new StringToken("(end)", null); + curr = litToken("(end)"); } return curr; |