From 0f6565687e03968abd2e508fa8183f50f04f1cc7 Mon Sep 17 00:00:00 2001 From: bjculkin Date: Fri, 24 Mar 2017 16:21:07 -0400 Subject: Update Pratt Parser --- .../java/bjc/utils/examples/DelimSplitterTest.java | 10 +- .../utils/examples/parsing/PrattParserTest.java | 99 +++++-- .../bjc/utils/examples/parsing/TestContext.java | 40 +++ BJC-Utils2/src/main/java/bjc/utils/data/Tree.java | 75 +++-- .../main/java/bjc/utils/funcutils/ListUtils.java | 4 +- .../java/bjc/utils/parserutils/TokenSplitter.java | 249 ---------------- .../parserutils/pratt/DefaultLeftCommand.java | 17 -- .../parserutils/pratt/DefaultNullCommand.java | 25 -- .../utils/parserutils/pratt/InitialCommand.java | 38 +++ .../bjc/utils/parserutils/pratt/LeftCommand.java | 63 ---- .../bjc/utils/parserutils/pratt/LeftCommands.java | 320 --------------------- .../utils/parserutils/pratt/NonInitialCommand.java | 63 ++++ .../bjc/utils/parserutils/pratt/NullCommand.java | 38 --- .../bjc/utils/parserutils/pratt/NullCommands.java | 207 ------------- .../bjc/utils/parserutils/pratt/PrattParser.java | 83 ++++-- .../pratt/commands/AbstractInitialCommand.java | 19 ++ .../parserutils/pratt/commands/BinaryCommand.java | 23 ++ .../pratt/commands/BinaryPostCommand.java | 19 ++ .../parserutils/pratt/commands/ChainCommand.java | 47 +++ .../pratt/commands/ConstantCommand.java | 21 ++ .../pratt/commands/DefaultInitialCommand.java | 28 ++ .../pratt/commands/DefaultNonInitialCommand.java | 34 +++ .../pratt/commands/DelimitedCommand.java | 65 +++++ .../pratt/commands/GroupingCommand.java | 30 ++ .../pratt/commands/InitialCommands.java | 134 +++++++++ .../parserutils/pratt/commands/LeafCommand.java | 17 ++ .../pratt/commands/LeftBinaryCommand.java | 12 + .../pratt/commands/NonBinaryCommand.java | 17 ++ .../pratt/commands/NonInitialCommands.java | 134 +++++++++ .../pratt/commands/PostCircumfixCommand.java | 32 +++ .../parserutils/pratt/commands/PostfixCommand.java | 19 ++ .../pratt/commands/PreTernaryCommand.java | 45 +++ .../pratt/commands/RightBinaryCommand.java | 12 + .../parserutils/pratt/commands/TernaryCommand.java | 47 +++ .../parserutils/pratt/commands/UnaryCommand.java | 23 ++ .../parserutils/splitter/SimpleTokenSplitter.java | 235 +++++++++++++++ .../utils/parserutils/splitter/TokenSplitter.java | 26 ++ .../parserutils/splitter/TwoLevelSplitter.java | 110 +++++++ 38 files changed, 1458 insertions(+), 1022 deletions(-) create mode 100644 BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java delete mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java delete mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/DefaultLeftCommand.java delete mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/DefaultNullCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/InitialCommand.java delete mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/LeftCommand.java delete mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/LeftCommands.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NonInitialCommand.java delete mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NullCommand.java delete mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NullCommands.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/AbstractInitialCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/BinaryCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/BinaryPostCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/ChainCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/ConstantCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DefaultInitialCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DefaultNonInitialCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DelimitedCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/GroupingCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/LeafCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/LeftBinaryCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/NonBinaryCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/NonInitialCommands.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PostCircumfixCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PostfixCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PreTernaryCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/RightBinaryCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/TernaryCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/UnaryCommand.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TokenSplitter.java create mode 100644 BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TwoLevelSplitter.java (limited to 'BJC-Utils2/src') diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/DelimSplitterTest.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/DelimSplitterTest.java index 4262ad2..5f4ef92 100644 --- a/BJC-Utils2/src/examples/java/bjc/utils/examples/DelimSplitterTest.java +++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/DelimSplitterTest.java @@ -2,13 +2,13 @@ package bjc.utils.examples; import bjc.utils.data.ITree; import bjc.utils.funcutils.StringUtils; -import bjc.utils.parserutils.TokenSplitter; import bjc.utils.parserutils.delims.DelimiterException; import bjc.utils.parserutils.delims.DelimiterGroup; import bjc.utils.parserutils.delims.RegexCloser; import bjc.utils.parserutils.delims.RegexOpener; import bjc.utils.parserutils.delims.SequenceDelimiter; import bjc.utils.parserutils.delims.StringDelimiter; +import bjc.utils.parserutils.splitter.SimpleTokenSplitter; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -22,13 +22,13 @@ import java.util.Map; import java.util.Scanner; /** - * Test for {@link SequenceDelimiter} as well as {@link TokenSplitter} + * Test for {@link SequenceDelimiter} as well as {@link SimpleTokenSplitter} * * @author EVE * */ public class DelimSplitterTest { - private TokenSplitter split; + private SimpleTokenSplitter split; private StringDelimiter dlm; @@ -46,7 +46,7 @@ public class DelimSplitterTest { groups = new HashMap<>(); - split = new TokenSplitter(); + split = new SimpleTokenSplitter(); dlm = new StringDelimiter(); @@ -172,7 +172,7 @@ public class DelimSplitterTest { System.out.println(split.toString()); break; case "splitter-reset": - split = new TokenSplitter(); + split = new SimpleTokenSplitter(); if(verbose) { System.out.println("Reset splitter"); } diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java index b4d40b2..3462d74 100644 --- a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java +++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java @@ -2,12 +2,14 @@ package bjc.utils.examples.parsing; import bjc.utils.data.ITree; import bjc.utils.data.TransformIterator; +import bjc.utils.esodata.Directory; import bjc.utils.parserutils.ParserException; -import bjc.utils.parserutils.TokenSplitter; import bjc.utils.parserutils.pratt.PrattParser; import bjc.utils.parserutils.pratt.StringToken; import bjc.utils.parserutils.pratt.StringTokenStream; import bjc.utils.parserutils.pratt.Token; +import bjc.utils.parserutils.splitter.TokenSplitter; +import bjc.utils.parserutils.splitter.TwoLevelSplitter; import java.util.Arrays; import java.util.HashSet; @@ -17,9 +19,10 @@ import java.util.LinkedList; import java.util.List; import java.util.Scanner; import java.util.Set; +import java.util.function.UnaryOperator; -import static bjc.utils.parserutils.pratt.LeftCommands.*; -import static bjc.utils.parserutils.pratt.NullCommands.*; +import static bjc.utils.parserutils.pratt.commands.NonInitialCommands.*; +import static bjc.utils.parserutils.pratt.commands.InitialCommands.*; /** * Simple test for pratt parser. @@ -28,11 +31,37 @@ import static bjc.utils.parserutils.pratt.NullCommands.*; * */ public class PrattParserTest { + private static final class BlockExit implements UnaryOperator { + @Override + public TestContext apply(TestContext state) { + state.scopes.pop(); + + state.blockCount.pop(); + + return state; + } + } + + private static final class BlockEnter implements UnaryOperator { + @Override + public TestContext apply(TestContext state) { + Directory enclosing = state.scopes.top(); + int currBlockNumber = state.blockCount.pop(); + + state.scopes.push(enclosing.newSubdirectory("block" + currBlockNumber)); + + state.blockCount.push(currBlockNumber + 1); + state.blockCount.push(0); + + return state; + } + } + /** * Main method. * * @param args - * Unused CLI arguments. + * Unused CLI arguments. */ public static void main(String[] args) { /* @@ -45,11 +74,13 @@ public class PrattParserTest { ops.addAll(Arrays.asList("<=", ">=")); ops.add("."); + ops.add(";"); ops.addAll(Arrays.asList("=", "<", ">")); ops.addAll(Arrays.asList("+", "-", "*", "/")); ops.addAll(Arrays.asList("^", "!")); ops.addAll(Arrays.asList("(", ")")); ops.addAll(Arrays.asList("[", "]")); + ops.addAll(Arrays.asList("{", "}")); /* * Reserved words that represent themselves, not literals. @@ -57,23 +88,36 @@ public class PrattParserTest { Set reserved = new LinkedHashSet<>(); reserved.addAll(Arrays.asList("if", "then", "else")); reserved.addAll(Arrays.asList("and", "or")); - - TokenSplitter split = new TokenSplitter(); - ops.forEach(split::addDelimiter); + reserved.addAll(Arrays.asList("begin", "end")); + + TwoLevelSplitter split = new TwoLevelSplitter(); + + split.addCompoundDelim(":="); + split.addCompoundDelim("||", "&&"); + split.addCompoundDelim("<=", ">="); - split.addNonMatcher("<=", ">="); + split.addSimpleDelim("."); + split.addSimpleDelim(";"); + split.addSimpleDelim("=", "<", ">"); + split.addSimpleDelim("+", "-", "*", "/"); + split.addSimpleDelim("^", "!"); + split.addSimpleMulti("\\(", "\\)"); + split.addSimpleMulti("\\[", "\\]"); + split.addSimpleMulti("\\{", "\\}"); split.compile(); - PrattParser parser = createParser(); + PrattParser parser = createParser(); + + TestContext ctx = new TestContext(); Scanner scn = new Scanner(System.in); System.out.print("Enter a command (blank line to exit): "); String ln = scn.nextLine(); - while (!ln.trim().equals("")) { - Iterator> tokens = preprocessInput(ops, split, ln, reserved); + while(!ln.trim().equals("")) { + Iterator> tokens = preprocessInput(ops, split, ln, reserved, ctx); try { StringTokenStream tokenStream = new StringTokenStream(tokens); @@ -83,14 +127,14 @@ public class PrattParserTest { */ tokenStream.next(); - ITree> tree = parser.parseExpression(0, tokenStream, null, true); + ITree> tree = parser.parseExpression(0, tokenStream, ctx, true); - if (!tokenStream.current().getKey().equals("(end)")) { + if(!tokenStream.current().getKey().equals("(end)")) { System.out.println("Multipe expressions on line"); } System.out.println("Parsed expression:\n" + tree); - } catch (ParserException pex) { + } catch(ParserException pex) { pex.printStackTrace(); } @@ -98,16 +142,19 @@ public class PrattParserTest { ln = scn.nextLine(); } + System.out.println(); + System.out.println("Context is:\n" + ctx); + scn.close(); } private static Iterator> preprocessInput(Set ops, TokenSplitter split, String ln, - Set reserved) { + Set reserved, TestContext ctx) { String[] rawTokens = ln.split("\\s+"); List splitTokens = new LinkedList<>(); - for (String raw : rawTokens) { + for(String raw : rawTokens) { String[] strangs = split.split(raw); splitTokens.addAll(Arrays.asList(strangs)); @@ -118,7 +165,7 @@ public class PrattParserTest { Iterator source = splitTokens.iterator(); Iterator> tokens = new TransformIterator<>(source, (String strang) -> { - if (ops.contains(strang) || reserved.contains(strang)) { + if(ops.contains(strang) || reserved.contains(strang)) { return new StringToken(strang, strang); } else { return new StringToken("(literal)", strang); @@ -127,7 +174,7 @@ public class PrattParserTest { return tokens; } - private static PrattParser createParser() { + private static PrattParser createParser() { /* * Set of which relational operators chain with each other. */ @@ -139,7 +186,7 @@ public class PrattParserTest { */ StringToken chainToken = new StringToken("and", "and"); - PrattParser parser = new PrattParser<>(); + PrattParser parser = new PrattParser<>(); parser.addNonInitialCommand("if", ternary(5, 0, "else", new StringToken("cond", "cond"), false)); @@ -147,7 +194,7 @@ public class PrattParserTest { parser.addNonInitialCommand("and", infixLeft(13)); parser.addNonInitialCommand("or", infixLeft(13)); - + parser.addNonInitialCommand("=", chain(15, relChain, chainToken)); parser.addNonInitialCommand("<", chain(15, relChain, chainToken)); parser.addNonInitialCommand(">", chain(15, relChain, chainToken)); @@ -156,7 +203,7 @@ public class PrattParserTest { parser.addNonInitialCommand("&&", infixRight(17)); parser.addNonInitialCommand("||", infixRight(17)); - + parser.addNonInitialCommand("+", infixLeft(20)); parser.addNonInitialCommand("-", infixLeft(20)); @@ -168,12 +215,16 @@ public class PrattParserTest { parser.addNonInitialCommand("^", infixRight(50)); parser.addNonInitialCommand(".", infixLeft(60)); - + parser.addNonInitialCommand("[", postCircumfix(60, 0, "]", new StringToken("idx", "idx"))); - parser.addInitialCommand("if", preTernary(0, 0, 0, "then", "else", new StringToken("ifelse", "ifelse"))); + parser.addInitialCommand("if", + preTernary(0, 0, 0, "then", "else", new StringToken("ifelse", "ifelse"))); + + parser.addInitialCommand("(", grouping(0, ")", new StringToken("parens", "parens"))); - parser.addInitialCommand("(", grouping(0, ")", new StringToken("()", "()"))); + parser.addInitialCommand("{", delimited(0, ";", "}", new StringToken("block", "block"), + new BlockEnter(), (state) -> state, new BlockExit(), true)); parser.addInitialCommand("-", unary(30)); diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java new file mode 100644 index 0000000..cc1be17 --- /dev/null +++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java @@ -0,0 +1,40 @@ +package bjc.utils.examples.parsing; + +import bjc.utils.esodata.Directory; +import bjc.utils.esodata.SimpleDirectory; +import bjc.utils.esodata.SimpleStack; +import bjc.utils.esodata.Stack; + +/** + * Simple context for the parser. + * + * @author EVE + * + */ +public class TestContext { + /** + * The variable scoping information. + */ + public Stack> scopes; + + /** + * The current number of scopes inside this scope. + */ + public Stack blockCount; + + /** + * Create a new test context. + */ + public TestContext() { + scopes = new SimpleStack<>(); + blockCount = new SimpleStack<>(); + + scopes.push(new SimpleDirectory<>()); + blockCount.push(0); + } + + @Override + public String toString() { + return String.format("TestContext [scopes=%s, blockCount=%s]", scopes, blockCount); + } +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/data/Tree.java b/BJC-Utils2/src/main/java/bjc/utils/data/Tree.java index cef9d51..3f8d258 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/data/Tree.java +++ b/BJC-Utils2/src/main/java/bjc/utils/data/Tree.java @@ -51,6 +51,8 @@ public class Tree implements ITree { public Tree(ContainedType leaf, IList> childrn) { this(leaf); + hasChildren = true; + childCount = childrn.getSize(); children = childrn; @@ -74,7 +76,7 @@ public class Tree implements ITree { children = new FunctionalList<>(); - for (ITree child : childrn) { + for(ITree child : childrn) { children.add(child); childCount++; @@ -83,7 +85,7 @@ public class Tree implements ITree { @Override public void addChild(ITree child) { - if (hasChildren == false) { + if(hasChildren == false) { hasChildren = true; children = new FunctionalList<>(); @@ -104,14 +106,14 @@ public class Tree implements ITree { @Override public void doForChildren(Consumer> action) { - if (childCount > 0) { + if(childCount > 0) { children.forEach(action); } } @Override public ITree flatMapTree(Function> mapper) { - if (hasChildren) { + if(hasChildren) { ITree flatMappedData = mapper.apply(data); children.map((child) -> child.flatMapTree(mapper)) @@ -130,7 +132,7 @@ public class Tree implements ITree { protected NewType internalCollapse(Function leafTransform, Function, NewType>> nodeCollapser) { - if (hasChildren) { + if(hasChildren) { Function, NewType> nodeTransformer = nodeCollapser.apply(data); IList collapsedChildren = children.map((child) -> { @@ -147,7 +149,7 @@ public class Tree implements ITree { } protected void internalToString(StringBuilder builder, int indentLevel, boolean initial) { - for (int i = 0; i < indentLevel; i++) { + for(int i = 0; i < indentLevel; i++) { builder.append(">\t"); } @@ -157,7 +159,7 @@ public class Tree implements ITree { builder.append(data == null ? "(null)" : data.toString()); builder.append("\n"); - if (hasChildren) { + if(hasChildren) { children.forEach((child) -> { ((Tree) child).internalToString(builder, indentLevel + 1, false); }); @@ -167,7 +169,7 @@ public class Tree implements ITree { @Override public ITree rebuildTree(Function leafTransformer, Function operatorTransformer) { - if (hasChildren) { + if(hasChildren) { IList> mappedChildren = children.map((child) -> { return child.rebuildTree(leafTransformer, operatorTransformer); }); @@ -180,7 +182,7 @@ public class Tree implements ITree { @Override public void selectiveTransform(Predicate nodePicker, UnaryOperator transformer) { - if (hasChildren) { + if(hasChildren) { children.forEach((child) -> child.selectiveTransform(nodePicker, transformer)); } else { data = transformer.apply(data); @@ -192,11 +194,11 @@ public class Tree implements ITree { UnaryOperator> transformer) { TopDownTransformResult transformResult = transformPicker.apply(data); - switch (transformResult) { + switch(transformResult) { case PASSTHROUGH: ITree result = new Tree<>(data); - if (hasChildren) { + if(hasChildren) { children.forEach((child) -> { result.addChild(child.topDownTransform(transformPicker, transformer)); }); @@ -212,7 +214,7 @@ public class Tree implements ITree { case PUSHDOWN: result = new Tree<>(data); - if (hasChildren) { + if(hasChildren) { children.forEach((child) -> { result.addChild(child.topDownTransform(transformPicker, transformer)); }); @@ -249,7 +251,7 @@ public class Tree implements ITree { @Override public TransformedType transformChild(int childNo, Function, TransformedType> transformer) { - if (childNo < 0 || childNo > childCount - 1) + if(childNo < 0 || childNo > childCount - 1) throw new IllegalArgumentException("Child index #" + childNo + " is invalid"); return transformer.apply(children.getByIndex(childNo)); @@ -262,7 +264,7 @@ public class Tree implements ITree { @Override public ITree transformTree(Function transformer) { - if (hasChildren) { + if(hasChildren) { IList> transformedChildren = children .map((child) -> child.transformTree(transformer)); @@ -274,12 +276,11 @@ public class Tree implements ITree { @Override public void traverse(TreeLinearizationMethod linearizationMethod, Consumer action) { - if (hasChildren) { - switch (linearizationMethod) { + if(hasChildren) { + switch(linearizationMethod) { case INORDER: - if (childCount != 2) - throw new IllegalArgumentException( - "Can only do in-order traversal for binary trees."); + if(childCount != 2) throw new IllegalArgumentException( + "Can only do in-order traversal for binary trees."); children.getByIndex(0).traverse(linearizationMethod, action); @@ -320,40 +321,32 @@ public class Tree implements ITree { @Override public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; + if(this == obj) return true; + if(obj == null) return false; + if(getClass() != obj.getClass()) return false; Tree other = (Tree) obj; - if (data == null) { - if (other.data != null) - return false; - } else if (!data.equals(other.data)) - return false; + if(data == null) { + if(other.data != null) return false; + } else if(!data.equals(other.data)) return false; - if (childCount != other.childCount) - return false; + if(childCount != other.childCount) return false; - if (children == null) { - if (other.children != null) - return false; - } else if (!children.equals(other.children)) - return false; + if(children == null) { + if(other.children != null) return false; + } else if(!children.equals(other.children)) return false; return true; } @Override public int revFind(Predicate> childPred) { - if (childCount == 0) { + if(childCount == 0) { return -1; } else { - for (int i = childCount - 1; i >= 0; i--) { - if (childPred.test(getChild(i))) { + for(int i = childCount - 1; i >= 0; i--) { + if(childPred.test(getChild(i))) { return i; } } @@ -364,7 +357,7 @@ public class Tree implements ITree { @Override public void prependChild(ITree child) { - if (hasChildren == false) { + if(hasChildren == false) { hasChildren = true; children = new FunctionalList<>(); diff --git a/BJC-Utils2/src/main/java/bjc/utils/funcutils/ListUtils.java b/BJC-Utils2/src/main/java/bjc/utils/funcutils/ListUtils.java index f55eb36..791598f 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/funcutils/ListUtils.java +++ b/BJC-Utils2/src/main/java/bjc/utils/funcutils/ListUtils.java @@ -80,7 +80,7 @@ public class ListUtils { * The affixes to remove * @return The tokens that have been deaffixed * - * @deprecated Replaced by TokenSplitter. + * @deprecated Replaced by SimpleTokenSplitter. */ @Deprecated public static IList deAffixTokens(IList input, Deque> operators) { @@ -301,7 +301,7 @@ public class ListUtils { * those operators * @return A list of tokens split on all the operators * - * @deprecated Use TokenSplitter now + * @deprecated Use SimpleTokenSplitter now */ @Deprecated public static IList splitTokens(IList input, Deque> operators) { diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java deleted file mode 100644 index d2569d9..0000000 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java +++ /dev/null @@ -1,249 +0,0 @@ -package bjc.utils.parserutils; - -import java.util.HashSet; -import java.util.Set; -import java.util.regex.Pattern; - -/** - * Split a string and keep given delimiters. - * - * @author Ben Culkin - */ -public class TokenSplitter { - /* - * This string is a format template for the delimiter matching regex - * - * It does two things: - * - *
  1. Match to the left of the provided delimiter by positive - * lookahead
  2. Match to the right of the provided delimiter by - * positive lookbehind
- * - * Thus, it will only match in places where the delimiter is, but won't - * actually match the delimiter, leaving split to put it into the stream - */ - private static String WITH_DELIM = "(?:(?<=%1$s)|(?=%1$s))"; - - /* - * This string is a format template for the multi-delimiter matching - * regex. - * - * It does the same thing as the single delimiter regex, but has to have - * some negative lookahead/lookbehind assertions to avoid splitting a - * delimiter into pieces. - */ - private static String WITH_MULTI_DELIM = "(?:(?<=%1$s+)(?!%1$s)|(? delimSet; - private Set multidelimSet; - private Set exclusionSet; - - /** - * Create a new token splitter. - */ - public TokenSplitter() { - delimSet = new HashSet<>(); - multidelimSet = new HashSet<>(); - exclusionSet = new HashSet<>(); - } - - /** - * Split a provided string using configured delimiters, and keeping the - * delimiters. - * - *

- * The splitter must be compiled first. - *

- * - * @param inp - * The string to split. - * - * @return The split string, including delimiters. - * - * @throws IllegalStateException - * If the splitter isn't compiled. - */ - public String[] split(String inp) { - if(compPatt == null) throw new IllegalStateException("Token splitter has not been compiled yet"); - - /* - * Don't split something that we should exclude from being - * split. - */ - if(exclusionPatt.matcher(inp).matches()) return new String[] { inp }; - - return compPatt.split(inp); - } - - /** - * Adds one or more strings as matched delimiters to split on. - * - * Only works for fixed length delimiters. - * - * The provided strings are regex-escaped before being used. - * - * @param delims - * The delimiters to match on. - */ - public void addDelimiter(String... delims) { - for(String delim : delims) { - if(delim == null) throw new NullPointerException("Delim must not be null"); - - String quoteDelim = Pattern.quote(delim); - String delimPat = String.format(WITH_DELIM, quoteDelim); - - if(currPatt == null) { - currPatt = new StringBuilder(); - currExclusionPatt = new StringBuilder(); - - currPatt.append("(?:" + delimPat + ")"); - currExclusionPatt.append("(?:" + quoteDelim + ")"); - } else { - currPatt.append("|(?:" + delimPat + ")"); - currExclusionPatt.append("|(?:" + quoteDelim + ")"); - } - - delimSet.add(delim); - } - } - - /** - * Adds a character class as a matched delimiter to split on. - * - * The provided string should be a pattern to match one or more - * occurances of. - * - * @param delims - * The delimiter to split on. - */ - public void addMultiDelimiter(String... delims) { - for(String delim : delims) { - if(delim == null) throw new NullPointerException("Delim must not be null"); - - String delimPat = String.format(WITH_MULTI_DELIM, "(?:" + delim + ")"); - - if(currPatt == null) { - currPatt = new StringBuilder(); - currExclusionPatt = new StringBuilder(); - - currPatt.append("(?:" + delimPat + ")"); - currExclusionPatt.append("(?:(?:" + delim + ")+)"); - - } else { - currPatt.append("|(?:" + delimPat + ")"); - currExclusionPatt.append("|(?:(?:" + delim + ")+)"); - } - - multidelimSet.add(delim); - } - } - - /** - * Marks strings matching the pattern delim as non-splittable. - * - * @param delims - * The regex to not splitting matching strings. - */ - public void addNonMatcher(String... delims) { - for(String delim : delims) { - if(delim == null) throw new NullPointerException("Delim must not be null"); - - if(currPatt == null) { - currPatt = new StringBuilder(); - currExclusionPatt = new StringBuilder(); - - currExclusionPatt.append("(?:" + delim + ")"); - } else { - currExclusionPatt.append("|(?:" + delim + ")"); - } - - exclusionSet.add(delim); - } - } - - /** - * Compiles the current set of delimiters to a pattern. - * - * Makes this splitter ready to use. - */ - public void compile() { - if(currPatt == null) currPatt = new StringBuilder(); - if(currExclusionPatt == null) currExclusionPatt = new StringBuilder(); - - compPatt = Pattern.compile(currPatt.toString()); - exclusionPatt = Pattern.compile(currExclusionPatt.toString()); - } - - /* - * (non-Javadoc) - * - * @see java.lang.Object#toString() - */ - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - - builder.append("TokenSplitter ["); - - if(currPatt != null) { - builder.append("currPatt="); - builder.append(currPatt); - builder.append("\n\t, "); - } - - if(currExclusionPatt != null) { - builder.append("currExclusionPatt="); - builder.append(currExclusionPatt); - builder.append("\n\t, "); - } - - if(compPatt != null) { - builder.append("compPatt="); - builder.append(compPatt); - builder.append("\n\t, "); - } - - if(exclusionPatt != null) { - builder.append("exclusionPatt="); - builder.append(exclusionPatt); - builder.append("\n\t, "); - } - - if(delimSet != null) { - builder.append("delimSet="); - builder.append(delimSet); - builder.append("\n\t, "); - } - - if(multidelimSet != null) { - builder.append("multidelimSet="); - builder.append(multidelimSet); - builder.append("\n\t, "); - } - - if(exclusionSet != null) { - builder.append("exclusionSet="); - builder.append(exclusionSet); - } - - builder.append("]"); - return builder.toString(); - } -} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/DefaultLeftCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/DefaultLeftCommand.java deleted file mode 100644 index 24802a6..0000000 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/DefaultLeftCommand.java +++ /dev/null @@ -1,17 +0,0 @@ -package bjc.utils.parserutils.pratt; - -import bjc.utils.data.ITree; - -class DefaultLeftCommand extends LeftCommand { - - @Override - public ITree> leftDenote(ITree> operand, Token operator, ParserContext ctx) { - throw new UnsupportedOperationException("Default command has no left denotation"); - } - - @Override - public int leftBinding() { - return -1; - } - -} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/DefaultNullCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/DefaultNullCommand.java deleted file mode 100644 index 04216f9..0000000 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/DefaultNullCommand.java +++ /dev/null @@ -1,25 +0,0 @@ -package bjc.utils.parserutils.pratt; - -import bjc.utils.data.ITree; -import bjc.utils.parserutils.ParserException; - -/** - * Default implementation of null command. - * - * @author EVE - * - * @param - * The key type of the token. - * @param - * The value type of the token. - * - * @param - * The state type of the parser. - */ -public class DefaultNullCommand extends NullCommand { - @Override - public ITree> nullDenotation(Token operator, ParserContext ctx) - throws ParserException { - throw new ParserException("Unexpected token " + operator); - } -} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/InitialCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/InitialCommand.java new file mode 100644 index 0000000..716b99e --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/InitialCommand.java @@ -0,0 +1,38 @@ +package bjc.utils.parserutils.pratt; + +import bjc.utils.data.ITree; +import bjc.utils.parserutils.ParserException; + +/** + * Represents an initial command in parsing. + * + * @author EVE + * + * @param + * The key type for the tokens. + * + * @param + * The value type for the tokens. + * + * @param + * The state type of the parser. + * + * + */ +@FunctionalInterface +public interface InitialCommand { + /** + * Construct the null denotation of this command. + * + * @param operator + * The operator for this command. + * @param ctx + * The context for the command. + * + * @return The tree for this command. + * + * @throws ParserException + * If something goes wrong during parsing. + */ + ITree> denote(Token operator, ParserContext ctx) throws ParserException; +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/LeftCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/LeftCommand.java deleted file mode 100644 index 747c207..0000000 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/LeftCommand.java +++ /dev/null @@ -1,63 +0,0 @@ -package bjc.utils.parserutils.pratt; - -import bjc.utils.data.ITree; -import bjc.utils.parserutils.ParserException; - -/** - * Represents a non-initial command in parsing. - * - * @author EVE - * - * @param - * The key type for the tokens. - * - * @param - * The value type for the tokens. - * - * @param - * The state type of the parser. - * - */ -public abstract class LeftCommand { - /** - * Construct the left denotation of this command. - * - * @param operand - * The left-hand operand of this command. - * @param operator - * The operator for this command. - * - * @param ctx - * The state needed for commands. - * - * @return The tree this command forms. - * - * @throws ParserException - * If something went wrong during parsing. - */ - public abstract ITree> leftDenote(ITree> operand, Token operator, - ParserContext ctx) throws ParserException; - - /** - * Get the left-binding power of this command. - * - * This represents the general precedence of this command. - * - * @return The left-binding power of this command. - */ - public abstract int leftBinding(); - - /** - * Get the next-binding power of this command. - * - * This represents the highest precedence of command this command can be - * the left operand of. - * - * This is the same as the left-binding power by default. - * - * @return The next-binding power of this command. - */ - public int nextBinding() { - return leftBinding(); - } -} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/LeftCommands.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/LeftCommands.java deleted file mode 100644 index 82ec843..0000000 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/LeftCommands.java +++ /dev/null @@ -1,320 +0,0 @@ -package bjc.utils.parserutils.pratt; - -import bjc.utils.data.ITree; -import bjc.utils.data.Tree; -import bjc.utils.parserutils.ParserException; - -import java.util.Set; - -/** - * Contains factory methods for producing common implementations of - * {@link LeftCommand} - * - * @author EVE - * - */ -public class LeftCommands { - /* - * A command with constant binding power. - */ - private static abstract class BinaryPostCommand extends LeftCommand { - private final int leftPower; - - public BinaryPostCommand(int power) { - leftPower = power; - } - - @Override - public int leftBinding() { - return leftPower; - } - } - - private static abstract class BinaryCommand extends BinaryPostCommand { - public BinaryCommand(int leftPower) { - super(leftPower); - } - - protected abstract int rightBinding(); - - @Override - public ITree> leftDenote(ITree> operand, Token operator, - ParserContext ctx) throws ParserException { - ITree> opr = ctx.parse.parseExpression(rightBinding(), ctx.tokens, ctx.state, false); - - return new Tree<>(operator, operand, opr); - } - } - - private static class LeftBinaryCommand extends BinaryCommand { - public LeftBinaryCommand(int leftPower) { - super(leftPower); - } - - @Override - protected int rightBinding() { - return 1 + leftBinding(); - } - } - - private static class RightBinaryCommand extends BinaryCommand { - public RightBinaryCommand(int leftPower) { - super(leftPower); - } - - @Override - protected int rightBinding() { - return leftBinding(); - } - } - - private static class NonBinaryCommand extends BinaryCommand { - public NonBinaryCommand(int leftPower) { - super(leftPower); - } - - @Override - protected int rightBinding() { - return 1 + leftBinding(); - } - - @Override - public int nextBinding() { - return leftBinding() - 1; - } - } - - private static class PostCircumfixCommand extends BinaryPostCommand { - private int insidePrec; - private K term; - private Token mark; - - public PostCircumfixCommand(int leftPower, int insidePower, K terminator, Token marker) { - super(leftPower); - - insidePrec = insidePower; - term = terminator; - mark = marker; - } - - @SuppressWarnings("unchecked") - @Override - public ITree> leftDenote(ITree> operand, Token operator, - ParserContext ctx) throws ParserException { - ITree> inside = ctx.parse.parseExpression(insidePrec, ctx.tokens, ctx.state, false); - - ctx.tokens.expect(term); - - return new Tree<>(mark, operand, inside); - } - } - - private static class PostfixCommand extends BinaryPostCommand { - public PostfixCommand(int leftPower) { - super(leftPower); - } - - @Override - public ITree> leftDenote(ITree> operand, Token operator, - ParserContext ctx) throws ParserException { - return new Tree<>(operator, operand); - } - } - - private static class TernaryCommand extends BinaryPostCommand { - private K term; - - private int innerExp; - - private Token mark; - - private boolean nonassoc; - - public TernaryCommand(int leftPower, K terminator, Token marker, boolean isNonassoc) { - super(leftPower); - - term = terminator; - mark = marker; - nonassoc = isNonassoc; - } - - @SuppressWarnings("unchecked") - @Override - public ITree> leftDenote(ITree> operand, Token operator, - ParserContext ctx) throws ParserException { - ITree> inner = ctx.parse.parseExpression(innerExp, ctx.tokens, ctx.state, false); - - ctx.tokens.expect(term); - - ITree> outer = ctx.parse.parseExpression(1 + leftBinding(), ctx.tokens, ctx.state, false); - - return new Tree<>(mark, inner, operand, outer); - } - - @Override - public int nextBinding() { - if (nonassoc) { - return leftBinding() - 1; - } else { - return leftBinding(); - } - } - } - - private static class ChainCommand extends BinaryPostCommand { - private Set chainWith; - - private Token chain; - - public ChainCommand(int leftPower, Set chainSet, Token chainMarker) { - super(leftPower); - - chainWith = chainSet; - chain = chainMarker; - } - - @Override - public ITree> leftDenote(ITree> operand, Token operator, - ParserContext ctx) throws ParserException { - ITree> tree = ctx.parse.parseExpression(1 + leftBinding(), ctx.tokens, ctx.state, false); - - ITree> res = new Tree<>(operator, operand, tree); - - if (chainWith.contains(ctx.tokens.current().getKey())) { - Token tok = ctx.tokens.current(); - ctx.tokens.next(); - - ITree> other = leftDenote(tree, tok, - new ParserContext<>(ctx.tokens, ctx.parse, ctx.state)); - - return new Tree<>(chain, res, other); - } else { - return res; - } - } - - @Override - public int nextBinding() { - return leftBinding() - 1; - } - } - - /** - * Create a left-associative infix operator. - * - * @param precedence - * The precedence of the operator. - * - * @return A command implementing that operator. - */ - public static LeftCommand infixLeft(int precedence) { - return new LeftBinaryCommand<>(precedence); - } - - /** - * Create a right-associative infix operator. - * - * @param precedence - * The precedence of the operator. - * - * @return A command implementing that operator. - */ - public static LeftCommand infixRight(int precedence) { - return new RightBinaryCommand<>(precedence); - } - - /** - * Create a non-associative infix operator. - * - * @param precedence - * The precedence of the operator. - * - * @return A command implementing that operator. - */ - public static LeftCommand infixNon(int precedence) { - return new NonBinaryCommand<>(precedence); - } - - /** - * Create a chained operator. - * - * @param precedence - * The precedence of the operator. - * - * @param chainSet - * The operators it forms a chain with. - * - * @param marker - * The token to use as the AST node for the chained - * operators. - * - * @return A command implementing that operator. - */ - public static LeftCommand chain(int precedence, Set chainSet, Token marker) { - return new ChainCommand<>(precedence, chainSet, marker); - } - - /** - * Create a postfix operator. - * - * @param precedence - * The precedence of the operator. - * - * @return A command implementing that operator. - */ - public static LeftCommand postfix(int precedence) { - return new PostfixCommand<>(precedence); - } - - /** - * Create a post-circumfix operator. - * - * This is an operator in form similar to array indexing. - * - * @param precedence - * The precedence of this operator - * - * @param insidePrecedence - * The precedence of the expression inside the operator - * - * @param closer - * The token that closes the circumfix. - * - * @param marker - * The token to use as the AST node for the operator. - * - * @return A command implementing that operator. - */ - public static LeftCommand postCircumfix(int precedence, int insidePrecedence, K closer, - Token marker) { - return new PostCircumfixCommand<>(precedence, insidePrecedence, closer, marker); - } - - /** - * Create a ternary operator. - * - * This is like C's ?: operator. - * - * @param precedence - * The precedence of the operator. - * - * @param insidePrecedence - * The precedence of the inner section of the operator. - * - * @param closer - * The token that marks the end of the inner section. - * - * @param marker - * The token to use as the AST node for the operator. - * - * @param nonassoc - * True if the command is non-associative, false - * otherwise. - * - * @return A command implementing this operator. - */ - public static LeftCommand ternary(int precedence, int insidePrecedence, K closer, - Token marker, boolean nonassoc) { - return new TernaryCommand<>(insidePrecedence, closer, marker, nonassoc); - } -} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NonInitialCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NonInitialCommand.java new file mode 100644 index 0000000..b6797d3 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NonInitialCommand.java @@ -0,0 +1,63 @@ +package bjc.utils.parserutils.pratt; + +import bjc.utils.data.ITree; +import bjc.utils.parserutils.ParserException; + +/** + * Represents a non-initial command in parsing. + * + * @author EVE + * + * @param + * The key type for the tokens. + * + * @param + * The value type for the tokens. + * + * @param + * The state type of the parser. + * + */ +public abstract class NonInitialCommand { + /** + * Construct the left denotation of this command. + * + * @param operand + * The left-hand operand of this command. + * @param operator + * The operator for this command. + * + * @param ctx + * The state needed for commands. + * + * @return The tree this command forms. + * + * @throws ParserException + * If something went wrong during parsing. + */ + public abstract ITree> denote(ITree> operand, Token operator, + ParserContext ctx) throws ParserException; + + /** + * Get the left-binding power of this command. + * + * This represents the general precedence of this command. + * + * @return The left-binding power of this command. + */ + public abstract int leftBinding(); + + /** + * Get the next-binding power of this command. + * + * This represents the highest precedence of command this command can be + * the left operand of. + * + * This is the same as the left-binding power by default. + * + * @return The next-binding power of this command. + */ + public int nextBinding() { + return leftBinding(); + } +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NullCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NullCommand.java deleted file mode 100644 index c105361..0000000 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NullCommand.java +++ /dev/null @@ -1,38 +0,0 @@ -package bjc.utils.parserutils.pratt; - -import bjc.utils.data.ITree; -import bjc.utils.parserutils.ParserException; - -/** - * Represents an initial command in parsing. - * - * @author EVE - * - * @param - * The key type for the tokens. - * - * @param - * The value type for the tokens. - * - * @param - * The state type of the parser. - * - * - */ -public abstract class NullCommand { - /** - * Construct the null denotation of this command. - * - * @param operator - * The operator for this command. - * @param ctx - * The context for the command. - * - * @return The tree for this command. - * - * @throws ParserException - * If something goes wrong during parsing. - */ - public abstract ITree> nullDenotation(Token operator, ParserContext ctx) - throws ParserException; -} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NullCommands.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NullCommands.java deleted file mode 100644 index f0c2a80..0000000 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/NullCommands.java +++ /dev/null @@ -1,207 +0,0 @@ -package bjc.utils.parserutils.pratt; - -import bjc.utils.data.ITree; -import bjc.utils.data.Tree; -import bjc.utils.parserutils.ParserException; - -/** - * * Contains factory methods for producing common implementations of - * {@link NullCommand} - * - * @author EVE - * - */ -public class NullCommands { - private static abstract class AbstractNullCommand extends NullCommand { - @Override - public ITree> nullDenotation(Token operator, ParserContext ctx) - throws ParserException { - // tokens.next(); - - return intNullDenotation(operator, ctx); - } - - protected abstract ITree> intNullDenotation(Token operator, ParserContext ctx) - throws ParserException; - - } - - private static class UnaryCommand extends AbstractNullCommand { - private final int nullPwer; - - public UnaryCommand(int nullPower) { - nullPwer = nullPower; - } - - @Override - protected ITree> intNullDenotation(Token operator, ParserContext ctx) - throws ParserException { - ITree> opr = ctx.parse.parseExpression(nullPwer, ctx.tokens, ctx.state, false); - - return new Tree<>(operator, opr); - } - } - - private static class GroupingCommand extends AbstractNullCommand { - private K term; - private Token mark; - private int inner; - - public GroupingCommand(int innerPrec, K terminator, Token marker) { - inner = innerPrec; - term = terminator; - mark = marker; - } - - @SuppressWarnings("unchecked") - @Override - protected ITree> intNullDenotation(Token operator, ParserContext ctx) - throws ParserException { - ITree> opr = ctx.parse.parseExpression(inner, ctx.tokens, ctx.state, false); - - ctx.tokens.expect(term); - - return new Tree<>(mark, opr); - } - } - - private static class LeafCommand extends NullCommand { - @Override - public ITree> nullDenotation(Token operator, ParserContext ctx) - throws ParserException { - - return new Tree<>(operator); - } - } - - private static class ConstantCommand extends NullCommand { - private ITree> val; - - public ConstantCommand(ITree> con) { - val = con; - } - - @Override - public ITree> nullDenotation(Token operator, ParserContext ctx) - throws ParserException { - return val; - } - } - - private static class PreTernaryCommand extends AbstractNullCommand { - private int cond1; - private int block1; - private int block2; - - private K mark1; - private K mark2; - - private Token term; - - public PreTernaryCommand(int cond1, int block1, int block2, K mark1, K mark2, Token term) { - super(); - this.cond1 = cond1; - this.block1 = block1; - this.block2 = block2; - this.mark1 = mark1; - this.mark2 = mark2; - this.term = term; - } - - @SuppressWarnings("unchecked") - @Override - protected ITree> intNullDenotation(Token operator, ParserContext ctx) - throws ParserException { - ITree> cond = ctx.parse.parseExpression(cond1, ctx.tokens, ctx.state, false); - - ctx.tokens.expect(mark1); - - ITree> fstBlock = ctx.parse.parseExpression(block1, ctx.tokens, ctx.state, false); - - ctx.tokens.expect(mark2); - - ITree> sndBlock = ctx.parse.parseExpression(block2, ctx.tokens, ctx.state, false); - - return new Tree<>(term, cond, fstBlock, sndBlock); - } - } - - /** - * Create a new unary operator. - * - * @param precedence - * The precedence of the operator. - * - * @return A command implementing that operator. - */ - public static NullCommand unary(int precedence) { - return new UnaryCommand<>(precedence); - } - - /** - * Create a new grouping operator. - * - * @param precedence - * The precedence of the expression in the operator. - * - * @param term - * The type that closes the group. - * - * @param mark - * The token for the AST node of the group. - * - * @return A command implementing the operator. - */ - public static NullCommand grouping(int precedence, K term, Token mark) { - return new GroupingCommand<>(precedence, term, mark); - } - - /** - * Create a new leaf operator. - * - * @return A command implementing the operator. - */ - public static NullCommand leaf() { - return new LeafCommand<>(); - } - - /** - * Create a new pre-ternary operator, like an if-then-else statement. - * - * @param cond1 - * The priority of the first block. - * - * @param block1 - * The priority of the second block. - * - * @param block2 - * The priority of the third block. - * - * @param mark1 - * The marker that ends the first block. - * - * @param mark2 - * The marker that ends the second block. - * - * @param term - * The token for the AST node of the group. - * - * @return A command implementing the operator. - */ - public static NullCommand preTernary(int cond1, int block1, int block2, K mark1, K mark2, - Token term) { - return new PreTernaryCommand<>(cond1, block1, block2, mark1, mark2, term); - } - - /** - * Create a new named constant. - * - * @param val - * The value of the constant. - * - * @return A command implementing the constant. - */ - public static NullCommand constant(ITree> val) { - return new ConstantCommand<>(val); - } -} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java index a8c273d..e8543dc 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/PrattParser.java @@ -3,6 +3,8 @@ package bjc.utils.parserutils.pratt; import bjc.utils.data.ITree; import bjc.utils.funcutils.NumberUtils; import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.commands.DefaultNonInitialCommand; +import bjc.utils.parserutils.pratt.commands.DefaultInitialCommand; import java.util.HashMap; import java.util.Map; @@ -13,29 +15,29 @@ import java.util.Map; * @author EVE * * @param - * The key type for the tokens. + * The key type for the tokens. * * @param - * The value type for the tokens. + * The value type for the tokens. * * @param - * The state type of the parser. + * The state type of the parser. * * */ public class PrattParser { - private final LeftCommand DEFAULT_LEFT_COMMAND = new DefaultLeftCommand<>(); - private final NullCommand DEFAULT_NULL_COMMAND = new DefaultNullCommand<>(); + private final NonInitialCommand DEFAULT_LEFT_COMMAND = new DefaultNonInitialCommand<>(); + private final InitialCommand DEFAULT_NULL_COMMAND = new DefaultInitialCommand<>(); - private Map> leftCommands; - private Map> nullCommands; - private Map> statementCommands; + private Map> leftCommands; + private Map> nullCommands; + private Map> statementCommands; /** * Create a new Pratt parser. * * @param terminal - * The terminal symbol. + * The terminal symbol. */ public PrattParser() { leftCommands = new HashMap<>(); @@ -47,22 +49,25 @@ public class PrattParser { * Parse an expression. * * @param precedence - * The initial precedence for the expression. + * The initial precedence for the expression. * * @param tokens - * The tokens for the expression. + * The tokens for the expression. * * @param state - * The state of the parser. + * The state of the parser. + * + * @param isStatement + * Whether or not to parse statements. * * @return The expression as an AST. * * @throws ParserException - * If something goes wrong during parsing. + * If something goes wrong during parsing. */ - public ITree> parseExpression(int precedence, TokenStream tokens, C state, boolean isStatement) - throws ParserException { - if (precedence < 0) { + public ITree> parseExpression(int precedence, TokenStream tokens, C state, + boolean isStatement) throws ParserException { + if(precedence < 0) { throw new IllegalArgumentException("Precedence must be greater than zero"); } @@ -71,28 +76,28 @@ public class PrattParser { ITree> ast; - if (isStatement && statementCommands.containsKey(initToken.getKey())) { - ast = statementCommands.getOrDefault(initToken.getKey(), DEFAULT_NULL_COMMAND).nullDenotation(initToken, - new ParserContext<>(tokens, this, state)); + if(isStatement && statementCommands.containsKey(initToken.getKey())) { + ast = statementCommands.getOrDefault(initToken.getKey(), DEFAULT_NULL_COMMAND) + .denote(initToken, new ParserContext<>(tokens, this, state)); } else { - ast = nullCommands.getOrDefault(initToken.getKey(), DEFAULT_NULL_COMMAND).nullDenotation(initToken, - new ParserContext<>(tokens, this, state)); + ast = nullCommands.getOrDefault(initToken.getKey(), DEFAULT_NULL_COMMAND) + .denote(initToken, new ParserContext<>(tokens, this, state)); } int rightPrec = Integer.MAX_VALUE; - while (true) { + while(true) { Token tok = tokens.current(); K key = tok.getKey(); - LeftCommand command = leftCommands.getOrDefault(key, DEFAULT_LEFT_COMMAND); + NonInitialCommand command = leftCommands.getOrDefault(key, DEFAULT_LEFT_COMMAND); int leftBind = command.leftBinding(); - if (NumberUtils.between(precedence, rightPrec, leftBind)) { + if(NumberUtils.between(precedence, rightPrec, leftBind)) { tokens.next(); - ast = command.leftDenote(ast, tok, new ParserContext<>(tokens, this, state)); + ast = command.denote(ast, tok, new ParserContext<>(tokens, this, state)); rightPrec = command.nextBinding(); } else { break; @@ -106,12 +111,12 @@ public class PrattParser { * Add a non-initial command to this parser. * * @param marker - * The key that marks the command. + * The key that marks the command. * * @param comm - * The command. + * The command. */ - public void addNonInitialCommand(K marker, LeftCommand comm) { + public void addNonInitialCommand(K marker, NonInitialCommand comm) { leftCommands.put(marker, comm); } @@ -119,12 +124,28 @@ public class PrattParser { * Add a initial command to this parser. * * @param marker - * The key that marks the command. + * The key that marks the command. * * @param comm - * The command. + * The command. */ - public void addInitialCommand(K marker, NullCommand comm) { + public void addInitialCommand(K marker, InitialCommand comm) { nullCommands.put(marker, comm); } + + /** + * Add a statement command to this parser. + * + * The difference between statements and initial commands is that + * statements can only appear at the start of the expression. + * + * @param marker + * The key that marks the command. + * + * @param comm + * The command. + */ + public void addStatementCommand(K marker, InitialCommand comm) { + statementCommands.put(marker, comm); + } } diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/AbstractInitialCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/AbstractInitialCommand.java new file mode 100644 index 0000000..b263422 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/AbstractInitialCommand.java @@ -0,0 +1,19 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.InitialCommand; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +public abstract class AbstractInitialCommand implements InitialCommand { + @Override + public ITree> denote(Token operator, ParserContext ctx) + throws ParserException { + return intNullDenotation(operator, ctx); + } + + protected abstract ITree> intNullDenotation(Token operator, + ParserContext ctx) throws ParserException; + +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/BinaryCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/BinaryCommand.java new file mode 100644 index 0000000..bf6786b --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/BinaryCommand.java @@ -0,0 +1,23 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +public abstract class BinaryCommand extends BinaryPostCommand { + public BinaryCommand(int leftPower) { + super(leftPower); + } + + protected abstract int rightBinding(); + + @Override + public ITree> denote(ITree> operand, Token operator, + ParserContext ctx) throws ParserException { + ITree> opr = ctx.parse.parseExpression(rightBinding(), ctx.tokens, ctx.state, false); + + return new Tree<>(operator, operand, opr); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/BinaryPostCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/BinaryPostCommand.java new file mode 100644 index 0000000..7aaf735 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/BinaryPostCommand.java @@ -0,0 +1,19 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.parserutils.pratt.NonInitialCommand; + +/* + * A command with constant binding power. + */ +public abstract class BinaryPostCommand extends NonInitialCommand { + private final int leftPower; + + public BinaryPostCommand(int power) { + leftPower = power; + } + + @Override + public int leftBinding() { + return leftPower; + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/ChainCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/ChainCommand.java new file mode 100644 index 0000000..aa85f75 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/ChainCommand.java @@ -0,0 +1,47 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +import java.util.Set; + +public class ChainCommand extends BinaryPostCommand { + private Set chainWith; + + private Token chain; + + public ChainCommand(int leftPower, Set chainSet, Token chainMarker) { + super(leftPower); + + chainWith = chainSet; + chain = chainMarker; + } + + @Override + public ITree> denote(ITree> operand, Token operator, + ParserContext ctx) throws ParserException { + ITree> tree = ctx.parse.parseExpression(1 + leftBinding(), ctx.tokens, ctx.state, false); + + ITree> res = new Tree<>(operator, operand, tree); + + if (chainWith.contains(ctx.tokens.current().getKey())) { + Token tok = ctx.tokens.current(); + ctx.tokens.next(); + + ITree> other = denote(tree, tok, + new ParserContext<>(ctx.tokens, ctx.parse, ctx.state)); + + return new Tree<>(chain, res, other); + } else { + return res; + } + } + + @Override + public int nextBinding() { + return leftBinding() - 1; + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/ConstantCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/ConstantCommand.java new file mode 100644 index 0000000..2308cc7 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/ConstantCommand.java @@ -0,0 +1,21 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.InitialCommand; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +public class ConstantCommand implements InitialCommand { + private ITree> val; + + public ConstantCommand(ITree> con) { + val = con; + } + + @Override + public ITree> denote(Token operator, ParserContext ctx) + throws ParserException { + return val; + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DefaultInitialCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DefaultInitialCommand.java new file mode 100644 index 0000000..7409755 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DefaultInitialCommand.java @@ -0,0 +1,28 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.InitialCommand; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +/** + * Default implementation of an initial command. + * + * @author EVE + * + * @param + * The key type of the token. + * + * @param + * The value type of the token. + * + * @param + * The state type of the parser. + */ +public class DefaultInitialCommand implements InitialCommand { + @Override + public ITree> denote(Token operator, ParserContext ctx) throws ParserException { + throw new ParserException("Unexpected token " + operator); + } +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DefaultNonInitialCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DefaultNonInitialCommand.java new file mode 100644 index 0000000..8c500a9 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DefaultNonInitialCommand.java @@ -0,0 +1,34 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.parserutils.pratt.NonInitialCommand; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +/** + * Default implementation of a non-initial command. + * + * @author EVE + * + * @param + * The key type of the tokens. + * + * @param + * The value type of the tokens. + * + * @param + * The state type of the parser. + */ +public class DefaultNonInitialCommand extends NonInitialCommand { + + @Override + public ITree> denote(ITree> operand, Token operator, ParserContext ctx) { + throw new UnsupportedOperationException("Default command has no left denotation"); + } + + @Override + public int leftBinding() { + return -1; + } + +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DelimitedCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DelimitedCommand.java new file mode 100644 index 0000000..090b2f4 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/DelimitedCommand.java @@ -0,0 +1,65 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.funcdata.FunctionalList; +import bjc.utils.funcdata.IList; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +import java.util.function.UnaryOperator; + +public class DelimitedCommand extends AbstractInitialCommand { + private int inner; + + private K delim; + private K mark; + + private Token term; + + private UnaryOperator onEnter; + private UnaryOperator onDelim; + private UnaryOperator onExit; + + private boolean statement; + + public DelimitedCommand(int inner, K delim, K mark, Token term, UnaryOperator onEnter, + UnaryOperator onDelim, UnaryOperator onExit, boolean statement) { + this.inner = inner; + this.delim = delim; + this.mark = mark; + this.term = term; + this.onEnter = onEnter; + this.onDelim = onDelim; + this.onExit = onExit; + this.statement = statement; + } + + @SuppressWarnings("unchecked") + @Override + protected ITree> intNullDenotation(Token operator, ParserContext ctx) + throws ParserException { + C newState = onEnter.apply(ctx.state); + + IList>> kids = new FunctionalList<>(); + + while(true) { + ITree> kid = ctx.parse.parseExpression(inner, ctx.tokens, newState, + statement); + kids.add(kid); + + Token tok = ctx.tokens.current(); + + ctx.tokens.expect(delim, mark); + + if(tok.getKey().equals(mark)) break; + + newState = onDelim.apply(newState); + } + + ctx.state = onExit.apply(newState); + + return new Tree<>(term, kids); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/GroupingCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/GroupingCommand.java new file mode 100644 index 0000000..407a39e --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/GroupingCommand.java @@ -0,0 +1,30 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +public class GroupingCommand extends AbstractInitialCommand { + private K term; + private Token mark; + private int inner; + + public GroupingCommand(int innerPrec, K terminator, Token marker) { + inner = innerPrec; + term = terminator; + mark = marker; + } + + @SuppressWarnings("unchecked") + @Override + protected ITree> intNullDenotation(Token operator, ParserContext ctx) + throws ParserException { + ITree> opr = ctx.parse.parseExpression(inner, ctx.tokens, ctx.state, false); + + ctx.tokens.expect(term); + + return new Tree<>(mark, opr); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java new file mode 100644 index 0000000..d9e7f90 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialCommands.java @@ -0,0 +1,134 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.parserutils.pratt.InitialCommand; +import bjc.utils.parserutils.pratt.Token; + +import java.util.function.UnaryOperator; + +/** + * * Contains factory methods for producing common implementations of + * {@link InitialCommand} + * + * @author EVE + * + */ +public class InitialCommands { + /** + * Create a new unary operator. + * + * @param precedence + * The precedence of the operator. + * + * @return A command implementing that operator. + */ + public static InitialCommand unary(int precedence) { + return new UnaryCommand<>(precedence); + } + + /** + * Create a new grouping operator. + * + * @param precedence + * The precedence of the expression in the operator. + * + * @param term + * The type that closes the group. + * + * @param mark + * The token for the AST node of the group. + * + * @return A command implementing the operator. + */ + public static InitialCommand grouping(int precedence, K term, Token mark) { + return new GroupingCommand<>(precedence, term, mark); + } + + /** + * Create a new leaf operator. + * + * @return A command implementing the operator. + */ + public static InitialCommand leaf() { + return new LeafCommand<>(); + } + + /** + * Create a new pre-ternary operator, like an if-then-else statement. + * + * @param cond1 + * The priority of the first block. + * + * @param block1 + * The priority of the second block. + * + * @param block2 + * The priority of the third block. + * + * @param mark1 + * The marker that ends the first block. + * + * @param mark2 + * The marker that ends the second block. + * + * @param term + * The token for the AST node of the group. + * + * @return A command implementing the operator. + */ + public static InitialCommand preTernary(int cond1, int block1, int block2, K mark1, K mark2, + Token term) { + return new PreTernaryCommand<>(cond1, block1, block2, mark1, mark2, term); + } + + /** + * Create a new named constant. + * + * @param val + * The value of the constant. + * + * @return A command implementing the constant. + */ + public static InitialCommand constant(ITree> val) { + return new ConstantCommand<>(val); + } + + /** + * Create a new delimited command. This is for block-like constructs. + * + * @param inner + * The precedence of the inner blocks. + * + * @param delim + * The marker between sub-blocks. + * + * @param mark + * The block terminator. + * + * @param term + * The token for the AST node of the group. + * + * @param onEnter + * The function to apply to the state on entering the + * block. + * + * @param onDelim + * The function to apply to the state on finishing a + * sub-block. + * + * @param onExit + * The function to apply to the state on exiting the + * block. + * + * @param statement + * Whether or not the sub-blocks are statements or + * expressions. + * + * @return A command implementing the operator. + */ + public static InitialCommand delimited(int inner, K delim, K mark, Token term, + UnaryOperator onEnter, UnaryOperator onDelim, UnaryOperator onExit, + boolean statement) { + return new DelimitedCommand<>(inner, delim, mark, term, onEnter, onDelim, onExit, statement); + } +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/LeafCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/LeafCommand.java new file mode 100644 index 0000000..87fe7c1 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/LeafCommand.java @@ -0,0 +1,17 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.InitialCommand; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +public class LeafCommand implements InitialCommand { + @Override + public ITree> denote(Token operator, ParserContext ctx) + throws ParserException { + + return new Tree<>(operator); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/LeftBinaryCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/LeftBinaryCommand.java new file mode 100644 index 0000000..1306735 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/LeftBinaryCommand.java @@ -0,0 +1,12 @@ +package bjc.utils.parserutils.pratt.commands; + +public class LeftBinaryCommand extends BinaryCommand { + public LeftBinaryCommand(int leftPower) { + super(leftPower); + } + + @Override + protected int rightBinding() { + return 1 + leftBinding(); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/NonBinaryCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/NonBinaryCommand.java new file mode 100644 index 0000000..36ea0e1 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/NonBinaryCommand.java @@ -0,0 +1,17 @@ +package bjc.utils.parserutils.pratt.commands; + +public class NonBinaryCommand extends BinaryCommand { + public NonBinaryCommand(int leftPower) { + super(leftPower); + } + + @Override + protected int rightBinding() { + return 1 + leftBinding(); + } + + @Override + public int nextBinding() { + return leftBinding() - 1; + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/NonInitialCommands.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/NonInitialCommands.java new file mode 100644 index 0000000..086ecf8 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/NonInitialCommands.java @@ -0,0 +1,134 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.parserutils.pratt.NonInitialCommand; +import bjc.utils.parserutils.pratt.Token; + +import java.util.Set; + +/** + * Contains factory methods for producing common implementations of + * {@link NonInitialCommand} + * + * @author EVE + * + */ +public class NonInitialCommands { + /** + * Create a left-associative infix operator. + * + * @param precedence + * The precedence of the operator. + * + * @return A command implementing that operator. + */ + public static NonInitialCommand infixLeft(int precedence) { + return new LeftBinaryCommand<>(precedence); + } + + /** + * Create a right-associative infix operator. + * + * @param precedence + * The precedence of the operator. + * + * @return A command implementing that operator. + */ + public static NonInitialCommand infixRight(int precedence) { + return new RightBinaryCommand<>(precedence); + } + + /** + * Create a non-associative infix operator. + * + * @param precedence + * The precedence of the operator. + * + * @return A command implementing that operator. + */ + public static NonInitialCommand infixNon(int precedence) { + return new NonBinaryCommand<>(precedence); + } + + /** + * Create a chained operator. + * + * @param precedence + * The precedence of the operator. + * + * @param chainSet + * The operators it forms a chain with. + * + * @param marker + * The token to use as the AST node for the chained + * operators. + * + * @return A command implementing that operator. + */ + public static NonInitialCommand chain(int precedence, Set chainSet, Token marker) { + return new ChainCommand<>(precedence, chainSet, marker); + } + + /** + * Create a postfix operator. + * + * @param precedence + * The precedence of the operator. + * + * @return A command implementing that operator. + */ + public static NonInitialCommand postfix(int precedence) { + return new PostfixCommand<>(precedence); + } + + /** + * Create a post-circumfix operator. + * + * This is an operator in form similar to array indexing. + * + * @param precedence + * The precedence of this operator + * + * @param insidePrecedence + * The precedence of the expression inside the operator + * + * @param closer + * The token that closes the circumfix. + * + * @param marker + * The token to use as the AST node for the operator. + * + * @return A command implementing that operator. + */ + public static NonInitialCommand postCircumfix(int precedence, int insidePrecedence, K closer, + Token marker) { + return new PostCircumfixCommand<>(precedence, insidePrecedence, closer, marker); + } + + /** + * Create a ternary operator. + * + * This is like C's ?: operator. + * + * @param precedence + * The precedence of the operator. + * + * @param insidePrecedence + * The precedence of the inner section of the operator. + * + * @param closer + * The token that marks the end of the inner section. + * + * @param marker + * The token to use as the AST node for the operator. + * + * @param nonassoc + * True if the command is non-associative, false + * otherwise. + * + * @return A command implementing this operator. + */ + public static NonInitialCommand ternary(int precedence, int insidePrecedence, K closer, + Token marker, boolean nonassoc) { + return new TernaryCommand<>(insidePrecedence, closer, marker, nonassoc); + } +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PostCircumfixCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PostCircumfixCommand.java new file mode 100644 index 0000000..ba5099e --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PostCircumfixCommand.java @@ -0,0 +1,32 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +public class PostCircumfixCommand extends BinaryPostCommand { + private int insidePrec; + private K term; + private Token mark; + + public PostCircumfixCommand(int leftPower, int insidePower, K terminator, Token marker) { + super(leftPower); + + insidePrec = insidePower; + term = terminator; + mark = marker; + } + + @SuppressWarnings("unchecked") + @Override + public ITree> denote(ITree> operand, Token operator, + ParserContext ctx) throws ParserException { + ITree> inside = ctx.parse.parseExpression(insidePrec, ctx.tokens, ctx.state, false); + + ctx.tokens.expect(term); + + return new Tree<>(mark, operand, inside); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PostfixCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PostfixCommand.java new file mode 100644 index 0000000..19c540a --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PostfixCommand.java @@ -0,0 +1,19 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +public class PostfixCommand extends BinaryPostCommand { + public PostfixCommand(int leftPower) { + super(leftPower); + } + + @Override + public ITree> denote(ITree> operand, Token operator, + ParserContext ctx) throws ParserException { + return new Tree<>(operator, operand); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PreTernaryCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PreTernaryCommand.java new file mode 100644 index 0000000..04604be --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/PreTernaryCommand.java @@ -0,0 +1,45 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +public class PreTernaryCommand extends AbstractInitialCommand { + private int cond1; + private int block1; + private int block2; + + private K mark1; + private K mark2; + + private Token term; + + public PreTernaryCommand(int cond1, int block1, int block2, K mark1, K mark2, Token term) { + super(); + this.cond1 = cond1; + this.block1 = block1; + this.block2 = block2; + this.mark1 = mark1; + this.mark2 = mark2; + this.term = term; + } + + @SuppressWarnings("unchecked") + @Override + protected ITree> intNullDenotation(Token operator, ParserContext ctx) + throws ParserException { + ITree> cond = ctx.parse.parseExpression(cond1, ctx.tokens, ctx.state, false); + + ctx.tokens.expect(mark1); + + ITree> fstBlock = ctx.parse.parseExpression(block1, ctx.tokens, ctx.state, false); + + ctx.tokens.expect(mark2); + + ITree> sndBlock = ctx.parse.parseExpression(block2, ctx.tokens, ctx.state, false); + + return new Tree<>(term, cond, fstBlock, sndBlock); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/RightBinaryCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/RightBinaryCommand.java new file mode 100644 index 0000000..729c33d --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/RightBinaryCommand.java @@ -0,0 +1,12 @@ +package bjc.utils.parserutils.pratt.commands; + +public class RightBinaryCommand extends BinaryCommand { + public RightBinaryCommand(int leftPower) { + super(leftPower); + } + + @Override + protected int rightBinding() { + return leftBinding(); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/TernaryCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/TernaryCommand.java new file mode 100644 index 0000000..c3204b9 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/TernaryCommand.java @@ -0,0 +1,47 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +public class TernaryCommand extends BinaryPostCommand { + private K term; + + private int innerExp; + + private Token mark; + + private boolean nonassoc; + + public TernaryCommand(int leftPower, K terminator, Token marker, boolean isNonassoc) { + super(leftPower); + + term = terminator; + mark = marker; + nonassoc = isNonassoc; + } + + @SuppressWarnings("unchecked") + @Override + public ITree> denote(ITree> operand, Token operator, + ParserContext ctx) throws ParserException { + ITree> inner = ctx.parse.parseExpression(innerExp, ctx.tokens, ctx.state, false); + + ctx.tokens.expect(term); + + ITree> outer = ctx.parse.parseExpression(1 + leftBinding(), ctx.tokens, ctx.state, false); + + return new Tree<>(mark, inner, operand, outer); + } + + @Override + public int nextBinding() { + if (nonassoc) { + return leftBinding() - 1; + } else { + return leftBinding(); + } + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/UnaryCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/UnaryCommand.java new file mode 100644 index 0000000..9a79d21 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/UnaryCommand.java @@ -0,0 +1,23 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +public class UnaryCommand extends AbstractInitialCommand { + private final int nullPwer; + + public UnaryCommand(int nullPower) { + nullPwer = nullPower; + } + + @Override + protected ITree> intNullDenotation(Token operator, ParserContext ctx) + throws ParserException { + ITree> opr = ctx.parse.parseExpression(nullPwer, ctx.tokens, ctx.state, false); + + return new Tree<>(operator, opr); + } +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java new file mode 100644 index 0000000..8b078a9 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java @@ -0,0 +1,235 @@ +package bjc.utils.parserutils.splitter; + +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * Simple implementation of {@link TokenSplitter} + * + * @author EVE + * + */ +public class SimpleTokenSplitter implements TokenSplitter { + /* + * This string is a format template for the delimiter matching regex + * + * It does two things: + * + *
  1. Match to the left of the provided delimiter by positive + * lookahead
  2. Match to the right of the provided delimiter by + * positive lookbehind
+ * + * Thus, it will only match in places where the delimiter is, but won't + * actually match the delimiter, leaving split to put it into the stream + */ + private static String WITH_DELIM = "(?:(?<=%1$s)|(?=%1$s))"; + + /* + * This string is a format template for the multi-delimiter matching + * regex. + * + * It does the same thing as the single delimiter regex, but has to have + * some negative lookahead/lookbehind assertions to avoid splitting a + * delimiter into pieces. + */ + private static String WITH_MULTI_DELIM = "(?:(?<=%1$s+)(?!%1$s)|(? delimSet; + private Set multidelimSet; + private Set exclusionSet; + + /** + * Create a new token splitter. + */ + public SimpleTokenSplitter() { + delimSet = new HashSet<>(); + multidelimSet = new HashSet<>(); + exclusionSet = new HashSet<>(); + } + + @Override + public String[] split(String inp) { + if(compPatt == null) throw new IllegalStateException("Token splitter has not been compiled yet"); + + /* + * Don't split something that we should exclude from being + * split. + */ + if(exclusionPatt.matcher(inp).matches()) return new String[] { inp }; + + return compPatt.split(inp); + } + + /** + * Adds one or more strings as matched delimiters to split on. + * + * Only works for fixed length delimiters. + * + * The provided strings are regex-escaped before being used. + * + * @param delims + * The delimiters to match on. + */ + public void addDelimiter(String... delims) { + for(String delim : delims) { + if(delim == null) throw new NullPointerException("Delim must not be null"); + + String quoteDelim = Pattern.quote(delim); + String delimPat = String.format(WITH_DELIM, quoteDelim); + + if(currPatt == null) { + currPatt = new StringBuilder(); + currExclusionPatt = new StringBuilder(); + + currPatt.append("(?:" + delimPat + ")"); + currExclusionPatt.append("(?:" + quoteDelim + ")"); + } else { + currPatt.append("|(?:" + delimPat + ")"); + currExclusionPatt.append("|(?:" + quoteDelim + ")"); + } + + delimSet.add(delim); + } + } + + /** + * Adds a character class as a matched delimiter to split on. + * + * The provided string should be a pattern to match one or more + * occurances of. + * + * @param delims + * The delimiter to split on. + */ + public void addMultiDelimiter(String... delims) { + for(String delim : delims) { + if(delim == null) throw new NullPointerException("Delim must not be null"); + + String delimPat = String.format(WITH_MULTI_DELIM, "(?:" + delim + ")"); + + if(currPatt == null) { + currPatt = new StringBuilder(); + currExclusionPatt = new StringBuilder(); + + currPatt.append("(?:" + delimPat + ")"); + currExclusionPatt.append("(?:(?:" + delim + ")+)"); + + } else { + currPatt.append("|(?:" + delimPat + ")"); + currExclusionPatt.append("|(?:(?:" + delim + ")+)"); + } + + multidelimSet.add(delim); + } + } + + /** + * Marks strings matching the pattern delim as non-splittable. + * + * @param delims + * The regex to not splitting matching strings. + */ + public void addNonMatcher(String... delims) { + for(String delim : delims) { + if(delim == null) throw new NullPointerException("Delim must not be null"); + + if(currPatt == null) { + currPatt = new StringBuilder(); + currExclusionPatt = new StringBuilder(); + + currExclusionPatt.append("(?:" + delim + ")"); + } else { + currExclusionPatt.append("|(?:" + delim + ")"); + } + + exclusionSet.add(delim); + } + } + + /** + * Compiles the current set of delimiters to a pattern. + * + * Makes this splitter ready to use. + */ + public void compile() { + if(currPatt == null) currPatt = new StringBuilder(); + if(currExclusionPatt == null) currExclusionPatt = new StringBuilder(); + + compPatt = Pattern.compile(currPatt.toString()); + exclusionPatt = Pattern.compile(currExclusionPatt.toString()); + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + + builder.append("SimpleTokenSplitter ["); + + if(currPatt != null) { + builder.append("currPatt="); + builder.append(currPatt); + builder.append("\n\t, "); + } + + if(currExclusionPatt != null) { + builder.append("currExclusionPatt="); + builder.append(currExclusionPatt); + builder.append("\n\t, "); + } + + if(compPatt != null) { + builder.append("compPatt="); + builder.append(compPatt); + builder.append("\n\t, "); + } + + if(exclusionPatt != null) { + builder.append("exclusionPatt="); + builder.append(exclusionPatt); + builder.append("\n\t, "); + } + + if(delimSet != null) { + builder.append("delimSet="); + builder.append(delimSet); + builder.append("\n\t, "); + } + + if(multidelimSet != null) { + builder.append("multidelimSet="); + builder.append(multidelimSet); + builder.append("\n\t, "); + } + + if(exclusionSet != null) { + builder.append("exclusionSet="); + builder.append(exclusionSet); + } + + builder.append("]"); + return builder.toString(); + } +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TokenSplitter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TokenSplitter.java new file mode 100644 index 0000000..e59d88e --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TokenSplitter.java @@ -0,0 +1,26 @@ +package bjc.utils.parserutils.splitter; + +/** + * Split a string and keep given delimiters. + * + * @author Ben Culkin + */ +public interface TokenSplitter { + /** + * Split a provided string using configured delimiters, and keeping the + * delimiters. + * + *

+ * The splitter must be compiled first. + *

+ * + * @param inp + * The string to split. + * + * @return The split string, including delimiters. + * + * @throws IllegalStateException + * If the splitter isn't compiled. + */ + String[] split(String inp); +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TwoLevelSplitter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TwoLevelSplitter.java new file mode 100644 index 0000000..38f303d --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TwoLevelSplitter.java @@ -0,0 +1,110 @@ +package bjc.utils.parserutils.splitter; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +/** + * Implementation of a splitter that runs in two passes. + * + * This is useful because {@link SimpleTokenSplitter} doesn't like handling both + * <= and = without mangling them. + * + * The first pass splits on compound operators, which are built up from simple + * operators. + * + * The second pass removes simple operators. + * + * @author EVE + * + */ +public class TwoLevelSplitter implements TokenSplitter { + private SimpleTokenSplitter high; + private SimpleTokenSplitter low; + + /** + * Create a new two level splitter. + */ + public TwoLevelSplitter() { + high = new SimpleTokenSplitter(); + low = new SimpleTokenSplitter(); + } + + @Override + public String[] split(String inp) { + List ret = new ArrayList<>(); + + String[] partials = high.split(inp); + + for(String partial : partials) { + String[] finals = low.split(partial); + + for(String fin : finals) { + ret.add(fin); + } + } + + return ret.toArray(new String[ret.size()]); + } + + /** + * Adds compound operators to split on. + * + * @param delims + * The compound operators to split on. + */ + public void addCompoundDelim(String... delims) { + for(String delim : delims) { + high.addDelimiter(delim); + + low.addNonMatcher(Pattern.quote(delim)); + } + } + + /** + * Adds simple operators to split on. + * + * @param delims + * The simple operators to split on. + */ + public void addSimpleDelim(String... delims) { + for(String delim : delims) { + low.addDelimiter(delim); + } + } + + /** + * Adds repeated compound operators to split on. + * + * @param delims + * The repeated compound operators to split on. + */ + public void addCompoundMulti(String... delims) { + for(String delim : delims) { + high.addMultiDelimiter(delim); + + low.addNonMatcher("(?:" + delim + ")+"); + } + } + + /** + * Adds simple compound operators to split on. + * + * @param delims + * The repeated simple operators to split on. + */ + public void addSimpleMulti(String... delims) { + for(String delim : delims) { + low.addMultiDelimiter(delim); + } + } + + /** + * Ready the splitter for use. + */ + public void compile() { + high.compile(); + + low.compile(); + } +} -- cgit v1.2.3