From 572551b78e7f36b65185cb258bea31114d9992f6 Mon Sep 17 00:00:00 2001 From: bjculkin Date: Mon, 27 Mar 2017 10:51:28 -0400 Subject: Simplifications --- .../utils/examples/parsing/PrattParserTest.java | 119 +++++++++++++-------- .../src/main/java/bjc/utils/functypes/ID.java | 9 ++ .../main/java/bjc/utils/ioutils/BlockReaders.java | 26 +++++ .../bjc/utils/parserutils/pratt/PrattParser.java | 67 +++++++++--- .../bjc/utils/parserutils/pratt/TokenStream.java | 4 + .../parserutils/pratt/blocks/ParseBlocks.java | 73 ++++++++++++- .../pratt/commands/DenestingCommand.java | 31 ++++++ .../pratt/commands/InitialCommands.java | 36 +++++-- .../parserutils/pratt/tokens/StringToken.java | 4 + .../pratt/tokens/StringTokenStream.java | 8 +- 10 files changed, 305 insertions(+), 72 deletions(-) create mode 100644 BJC-Utils2/src/main/java/bjc/utils/functypes/ID.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/ioutils/BlockReaders.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DenestingCommand.java diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java index 6b97c19..522bf6f 100644 --- a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java +++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java @@ -4,6 +4,8 @@ import bjc.utils.data.ITree; import bjc.utils.data.TransformIterator; import bjc.utils.esodata.Directory; import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.InitialCommand; +import bjc.utils.parserutils.pratt.NonInitialCommand; import bjc.utils.parserutils.pratt.PrattParser; import bjc.utils.parserutils.pratt.Token; import bjc.utils.parserutils.pratt.tokens.StringToken; @@ -19,11 +21,16 @@ import java.util.LinkedList; import java.util.List; import java.util.Scanner; import java.util.Set; +import java.util.function.Function; import java.util.function.UnaryOperator; import static bjc.utils.parserutils.pratt.commands.NonInitialCommands.*; import static bjc.utils.parserutils.pratt.commands.InitialCommands.*; +import static bjc.utils.parserutils.pratt.tokens.StringToken.litToken; + +import static bjc.utils.functypes.ID.id; + /** * Simple test for Pratt parser. * @@ -31,6 +38,29 @@ import static bjc.utils.parserutils.pratt.commands.InitialCommands.*; * */ public class PrattParserTest { + private static final class Tokenizer implements Function> { + private Set ops; + private Set reserved; + private TestContext ctx; + + public Tokenizer(Set operators, Set reservedWords, TestContext context) { + ops = operators; + reserved = reservedWords; + ctx = context; + } + + @Override + public Token apply(String strang) { + if (ops.contains(strang) || reserved.contains(strang)) { + return litToken(strang); + } else if (ctx.scopes.top().containsKey(strang)) { + return new StringToken("(vref)", strang); + } else { + return new StringToken("(literal)", strang); + } + } + } + private static final class BlockExit implements UnaryOperator { @Override public TestContext apply(TestContext state) { @@ -119,7 +149,7 @@ public class PrattParserTest { System.out.print("Enter a command (blank line to exit): "); String ln = scn.nextLine(); - while(!ln.trim().equals("")) { + while (!ln.trim().equals("")) { Iterator> tokens = preprocessInput(ops, split, ln, reserved, ctx); try { @@ -132,12 +162,12 @@ public class PrattParserTest { ITree> tree = parser.parseExpression(0, tokenStream, ctx, true); - if(!tokenStream.current().getKey().equals("(end)")) { + if (!tokenStream.headIs("(end)")) { System.out.println("Multiple expressions on line"); } System.out.println("Parsed expression:\n" + tree); - } catch(ParserException pex) { + } catch (ParserException pex) { pex.printStackTrace(); } @@ -157,17 +187,17 @@ public class PrattParserTest { List splitTokens = new LinkedList<>(); - for(String raw : rawTokens) { + for (String raw : rawTokens) { boolean doSplit = false; - for(String op : ops) { - if(raw.contains(op)) { + for (String op : ops) { + if (raw.contains(op)) { doSplit = true; break; } } - if(doSplit) { + if (doSplit) { String[] strangs = split.split(raw); splitTokens.addAll(Arrays.asList(strangs)); @@ -180,15 +210,8 @@ public class PrattParserTest { Iterator source = splitTokens.iterator(); - Iterator> tokens = new TransformIterator<>(source, (String strang) -> { - if(ops.contains(strang) || reserved.contains(strang)) { - return new StringToken(strang, strang); - } else if(ctx.scopes.top().containsKey(strang)) { - return new StringToken("(vref)", strang); - } else { - return new StringToken("(literal)", strang); - } - }); + Iterator> tokens = new TransformIterator<>(source, + new Tokenizer(ops, reserved, ctx)); return tokens; } @@ -202,33 +225,43 @@ public class PrattParserTest { /* * Token for marking chains. */ - StringToken chainToken = new StringToken("and", "and"); + StringToken chainToken = litToken("and"); + + /* + * ID function. + */ + UnaryOperator idfun = id(); PrattParser parser = new PrattParser<>(); parser.addNonInitialCommand(":", infixNon(3)); - parser.addNonInitialCommand("if", ternary(5, 0, "else", new StringToken("cond", "cond"), false)); + parser.addNonInitialCommand("if", ternary(5, 0, "else", litToken("cond"), false)); parser.addNonInitialCommand(":=", new AssignCommand()); - parser.addNonInitialCommand("and", infixLeft(13)); - parser.addNonInitialCommand("or", infixLeft(13)); + NonInitialCommand nonSSRelJoin = infixLeft(13); + parser.addNonInitialCommand("and", nonSSRelJoin); + parser.addNonInitialCommand("or", nonSSRelJoin); - parser.addNonInitialCommand("=", chain(15, relChain, chainToken)); - parser.addNonInitialCommand("<", chain(15, relChain, chainToken)); - parser.addNonInitialCommand(">", chain(15, relChain, chainToken)); - parser.addNonInitialCommand("<=", chain(15, relChain, chainToken)); - parser.addNonInitialCommand(">=", chain(15, relChain, chainToken)); + NonInitialCommand chainRelOp = chain(15, relChain, chainToken); + parser.addNonInitialCommand("=", chainRelOp); + parser.addNonInitialCommand("<", chainRelOp); + parser.addNonInitialCommand(">", chainRelOp); + parser.addNonInitialCommand("<=", chainRelOp); + parser.addNonInitialCommand(">=", chainRelOp); - parser.addNonInitialCommand("&&", infixRight(17)); - parser.addNonInitialCommand("||", infixRight(17)); + NonInitialCommand ssRelJoin = infixRight(17); + parser.addNonInitialCommand("&&", ssRelJoin); + parser.addNonInitialCommand("||", ssRelJoin); - parser.addNonInitialCommand("+", infixLeft(20)); - parser.addNonInitialCommand("-", infixLeft(20)); + NonInitialCommand addSub = infixLeft(20); + parser.addNonInitialCommand("+", addSub); + parser.addNonInitialCommand("-", addSub); - parser.addNonInitialCommand("*", infixLeft(30)); - parser.addNonInitialCommand("/", infixLeft(30)); + NonInitialCommand mulDiv = infixLeft(30); + parser.addNonInitialCommand("*", mulDiv); + parser.addNonInitialCommand("/", mulDiv); parser.addNonInitialCommand("!", postfix(40)); @@ -236,28 +269,26 @@ public class PrattParserTest { parser.addNonInitialCommand(".", infixLeft(60)); - parser.addNonInitialCommand("[", postCircumfix(60, 0, "]", new StringToken("idx", "idx"))); + parser.addNonInitialCommand("[", postCircumfix(60, 0, "]", litToken("idx"))); - parser.addInitialCommand("if", - preTernary(0, 0, 0, "then", "else", new StringToken("ifelse", "ifelse"))); + parser.addInitialCommand("if", preTernary(0, 0, 0, "then", "else", litToken("ifelse"))); - parser.addInitialCommand("(", grouping(0, ")", new StringToken("parens", "parens"))); + parser.addInitialCommand("(", grouping(0, ")", litToken("parens"))); - parser.addInitialCommand("begin", delimited(0, ";", "end", new StringToken("block", "block"), - new BlockEnter(), (state) -> state, new BlockExit(), true)); + parser.addInitialCommand("begin", delimited(0, ";", "end", litToken("block"), new BlockEnter(), idfun, + new BlockExit(), true)); - parser.addInitialCommand("[", delimited(0, ",", "]", new StringToken("array", "array"), - (state) -> state, (state) -> state, (state) -> state, false)); + parser.addInitialCommand("[", delimited(0, ",", "]", litToken("array"), idfun, idfun, idfun, false)); - parser.addInitialCommand("{", delimited(0, ",", "}", new StringToken("json", "json"), (state) -> state, - (state) -> state, (state) -> state, false)); + parser.addInitialCommand("{", delimited(0, ",", "}", litToken("json"), idfun, idfun, idfun, false)); parser.addInitialCommand("case", unary(5)); parser.addInitialCommand("-", unary(30)); - parser.addInitialCommand("(literal)", leaf()); - parser.addInitialCommand("(vref)", leaf()); + InitialCommand leaf = leaf(); + parser.addInitialCommand("(literal)", leaf); + parser.addInitialCommand("(vref)", leaf); parser.addInitialCommand("var", new VarCommand()); @@ -265,4 +296,4 @@ public class PrattParserTest { return parser; } -} +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/functypes/ID.java b/BJC-Utils2/src/main/java/bjc/utils/functypes/ID.java new file mode 100644 index 0000000..af72d1a --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/functypes/ID.java @@ -0,0 +1,9 @@ +package bjc.utils.functypes; + +import java.util.function.UnaryOperator; + +public class ID { + public static UnaryOperator id() { + return (x) -> x; + } +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/ioutils/BlockReaders.java b/BJC-Utils2/src/main/java/bjc/utils/ioutils/BlockReaders.java new file mode 100644 index 0000000..ec8dcfb --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/ioutils/BlockReaders.java @@ -0,0 +1,26 @@ +package bjc.utils.ioutils; + +import java.io.Reader; + +/** + * Utility methods for constructing instances of {@link BlockReader} + * + * @author bjculkin + * + */ +public class BlockReaders { + /** + * Create a new simple block reader that works off a regex. + * + * @param blockDelim + * The regex that seperates blocks. + * + * @param source + * The reader to get blocks from. + * + * @return A configured simple reader. + */ + public static BlockReader simple(String blockDelim, Reader source) { + return new SimpleBlockReader(blockDelim, source); + } +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java index 20572e8..c53b4e1 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java @@ -26,18 +26,37 @@ import java.util.Map; * */ public class PrattParser { + /* + * Default commands that error when used. + */ private final NonInitialCommand DEFAULT_LEFT_COMMAND = new DefaultNonInitialCommand<>(); - private final InitialCommand DEFAULT_NULL_COMMAND = new DefaultInitialCommand<>(); + private final InitialCommand DEFAULT_NULL_COMMAND = new DefaultInitialCommand<>(); + + /* + * Left-commands that depend on what the null command was. + */ + private Map>> dependantLeftCommands; - private Map> leftCommands; - private Map> nullCommands; - private Map> statementCommands; + /* + * The left commands. + */ + private Map> leftCommands; + /* + * The initial commands. + */ + private Map> nullCommands; + /* + * Initial commands only checked for statements. + */ + private Map> statementCommands; /** * Create a new Pratt parser. * */ public PrattParser() { + dependantLeftCommands = new HashMap<>(); + leftCommands = new HashMap<>(); nullCommands = new HashMap<>(); statementCommands = new HashMap<>(); @@ -65,34 +84,41 @@ public class PrattParser { */ public ITree> parseExpression(int precedence, TokenStream tokens, C state, boolean isStatement) throws ParserException { - if(precedence < 0) { + if (precedence < 0) { throw new IllegalArgumentException("Precedence must be greater than zero"); } Token initToken = tokens.current(); tokens.next(); + K initKey = initToken.getKey(); + ITree> ast; - if(isStatement && statementCommands.containsKey(initToken.getKey())) { - ast = statementCommands.getOrDefault(initToken.getKey(), DEFAULT_NULL_COMMAND) - .denote(initToken, new ParserContext<>(tokens, this, state)); + if (isStatement && statementCommands.containsKey(initKey)) { + ast = statementCommands.getOrDefault(initKey, DEFAULT_NULL_COMMAND).denote(initToken, + new ParserContext<>(tokens, this, state)); } else { - ast = nullCommands.getOrDefault(initToken.getKey(), DEFAULT_NULL_COMMAND) - .denote(initToken, new ParserContext<>(tokens, this, state)); + ast = nullCommands.getOrDefault(initKey, DEFAULT_NULL_COMMAND).denote(initToken, + new ParserContext<>(tokens, this, state)); } int rightPrec = Integer.MAX_VALUE; - while(true) { + while (true) { Token tok = tokens.current(); K key = tok.getKey(); NonInitialCommand command = leftCommands.getOrDefault(key, DEFAULT_LEFT_COMMAND); + + if (dependantLeftCommands.containsKey(initKey)) { + command = dependantLeftCommands.get(initKey).getOrDefault(key, command); + } + int leftBind = command.leftBinding(); - if(NumberUtils.between(precedence, rightPrec, leftBind)) { + if (NumberUtils.between(precedence, rightPrec, leftBind)) { tokens.next(); ast = command.denote(ast, tok, new ParserContext<>(tokens, this, state)); @@ -146,4 +172,19 @@ public class PrattParser { public void addStatementCommand(K marker, InitialCommand comm) { statementCommands.put(marker, comm); } -} + + /** + * Add a dependant non-initial command to this parser. + */ + public void addDependantCommand(K dependant, K marker, NonInitialCommand comm) { + if (dependantLeftCommands.containsKey(dependant)) { + dependantLeftCommands.get(dependant).put(marker, comm); + } else { + Map> comms = new HashMap<>(); + + comms.put(marker, comm); + + dependantLeftCommands.put(dependant, comms); + } + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/TokenStream.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/TokenStream.java index 71482b4..db7d1e0 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/TokenStream.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/TokenStream.java @@ -86,4 +86,8 @@ public abstract class TokenStream implements Iterator> { public final void expect(K... expectedKeys) throws ExpectationException { expect(new HashSet<>(Arrays.asList(expectedKeys))); } + + public boolean headIs(K val) { + return current().getKey().equals(val); + } } diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/blocks/ParseBlocks.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/blocks/ParseBlocks.java index 43163e6..9df8355 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/blocks/ParseBlocks.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/blocks/ParseBlocks.java @@ -1,6 +1,11 @@ package bjc.utils.parserutils.pratt.blocks; +import java.util.function.Predicate; +import java.util.function.UnaryOperator; + +import bjc.utils.data.ITree; import bjc.utils.parserutils.pratt.ParseBlock; +import bjc.utils.parserutils.pratt.Token; /** * Utility class for creating common implementations of {@link ParseBlock} @@ -9,5 +14,71 @@ import bjc.utils.parserutils.pratt.ParseBlock; * */ public class ParseBlocks { + /** + * Create a new repeating parse block. + * + * @param inner + * The parse block to repeat. + * + * @param delim + * The token type that seperates repetitions. + * + * @param term + * The token type that terminates repititions. + * + * @param mark + * The token to use as the node in the AST. + * + * @param action + * The action to perform on the state after every + * repitition. + * + * @return A configured repeating parse block. + */ + public static ParseBlock repeating(ParseBlock inner, K delim, K term, + Token mark, UnaryOperator action) { + return new RepeatingParseBlock<>(inner, delim, term, mark, action); + } + + /** + * Create a new triggered parse block. + * + * @param source + * The block to trigger around. + * + * @param onEnter + * The action to perform upon the state before entering + * the block. + * + * @param onExit + * The action to perform upon the state after exiting the + * block. + * + * @return A configured trigger parse block. + */ + public static ParseBlock trigger(ParseBlock source, UnaryOperator onEnter, + UnaryOperator onExit) { + return new TriggeredParseBlock<>(onEnter, onExit, source); + } -} + /** + * Create a new simple parse block. + * + * @param precedence + * The precedence of the expression inside the block. + * + * @param terminator + * The key type of the token expected after this block, + * or null if none is expected. + * + * @param validator + * The predicate to use to validate parsed expressions, + * or null if none is used. + * + * @return A configured simple parse block. + */ + public static ParseBlock simple(int precedence, K terminator, + Predicate>> validator) { + return new SimpleParseBlock<>(precedence, terminator, validator); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DenestingCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DenestingCommand.java new file mode 100644 index 0000000..d2d7829 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DenestingCommand.java @@ -0,0 +1,31 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.InitialCommand; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +public class DenestingCommand extends AbstractInitialCommand { + private InitialCommand internal; + + /** + * Create a new transforming initial command. + * + * @param internal + * The initial command to delegate to. + * + * @param transform + * The transform to apply to the returned tree. + */ + public DenestingCommand(InitialCommand internal) { + super(); + this.internal = internal; + } + + @Override + protected ITree> intNullDenotation(Token operator, ParserContext ctx) + throws ParserException { + return internal.denote(operator, ctx).getChild(0); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java index bad5964..eac357a 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java @@ -4,12 +4,11 @@ import bjc.utils.data.ITree; import bjc.utils.parserutils.pratt.InitialCommand; import bjc.utils.parserutils.pratt.ParseBlock; import bjc.utils.parserutils.pratt.Token; -import bjc.utils.parserutils.pratt.blocks.RepeatingParseBlock; -import bjc.utils.parserutils.pratt.blocks.SimpleParseBlock; -import bjc.utils.parserutils.pratt.blocks.TriggeredParseBlock; import java.util.function.UnaryOperator; +import static bjc.utils.parserutils.pratt.blocks.ParseBlocks.*; + /** * * Contains factory methods for producing common implementations of * {@link InitialCommand} @@ -45,7 +44,7 @@ public class InitialCommands { * @return A command implementing the operator. */ public static InitialCommand grouping(int precedence, K term, Token mark) { - ParseBlock innerBlock = new SimpleParseBlock<>(precedence, term, null); + ParseBlock innerBlock = simple(precedence, term, null); return new GroupingCommand<>(innerBlock, mark); } @@ -84,9 +83,9 @@ public class InitialCommands { */ public static InitialCommand preTernary(int cond1, int block1, int block2, K mark1, K mark2, Token term) { - ParseBlock condBlock = new SimpleParseBlock<>(cond1, mark1, null); - ParseBlock opblock1 = new SimpleParseBlock<>(block1, mark2, null); - ParseBlock opblock2 = new SimpleParseBlock<>(block2, null, null); + ParseBlock condBlock = simple(cond1, mark1, null); + ParseBlock opblock1 = simple(block1, mark2, null); + ParseBlock opblock2 = simple(block2, null, null); return new PreTernaryCommand<>(condBlock, opblock1, opblock2, term); } @@ -139,9 +138,9 @@ public class InitialCommands { public static InitialCommand delimited(int inner, K delim, K mark, Token term, UnaryOperator onEnter, UnaryOperator onDelim, UnaryOperator onExit, boolean statement) { - ParseBlock innerBlock = new SimpleParseBlock<>(inner, null, null); - ParseBlock delimsBlock = new RepeatingParseBlock<>(innerBlock, delim, mark, term, onDelim); - ParseBlock scopedBlock = new TriggeredParseBlock<>(onEnter, onExit, delimsBlock); + ParseBlock innerBlock = simple(inner, null, null); + ParseBlock delimsBlock = repeating(innerBlock, delim, mark, term, onDelim); + ParseBlock scopedBlock = trigger(delimsBlock, onEnter, onExit); GroupingCommand command = new GroupingCommand<>(scopedBlock, term); @@ -149,6 +148,21 @@ public class InitialCommands { * Remove the wrapper layer from grouping-command on top of * RepeatingParseBlock. */ - return new TransformingInitialCommand<>(command, (tree) -> tree.getChild(0)); + return denest(command); + } + + /** + * Create a new denesting command. + * + * This removes one tree-level, and is useful when combining complex + * parse blocks with commands. + * + * @param comm + * The command to denest. + * + * @return A command that denests the result of the provided command. + */ + public static InitialCommand denest(InitialCommand comm) { + return new DenestingCommand<>(comm); } } diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringToken.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringToken.java index 7e779aa..f156f02 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringToken.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringToken.java @@ -77,4 +77,8 @@ public class StringToken implements Token { public String toString() { return String.format("StringToken [key='%s', val='%s']", key, val); } + + public static StringToken litToken(String val) { + return new StringToken(val, val); + } } diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringTokenStream.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringTokenStream.java index d7d4a66..75e86c4 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringTokenStream.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/tokens/StringTokenStream.java @@ -5,11 +5,13 @@ import java.util.Iterator; import bjc.utils.parserutils.pratt.Token; import bjc.utils.parserutils.pratt.TokenStream; +import static bjc.utils.parserutils.pratt.tokens.StringToken.litToken; + /** * Simple implementation of token stream for strings. * - * The terminal token here is represented by a token with type '(end)' and null - * value. + * The terminal token here is represented by a token with type and value + * '(end)'. * * @author EVE * @@ -41,7 +43,7 @@ public class StringTokenStream extends TokenStream { if (iter.hasNext()) { curr = iter.next(); } else { - curr = new StringToken("(end)", null); + curr = litToken("(end)"); } return curr; -- cgit v1.2.3