diff options
Diffstat (limited to 'BJC-Utils2')
8 files changed, 214 insertions, 14 deletions
diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/AssignCommand.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/AssignCommand.java new file mode 100644 index 0000000..9cd30a8 --- /dev/null +++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/AssignCommand.java @@ -0,0 +1,35 @@ +package bjc.utils.examples.parsing; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.StringToken; +import bjc.utils.parserutils.pratt.Token; +import bjc.utils.parserutils.pratt.commands.NonBinaryCommand; + +class AssignCommand extends NonBinaryCommand<String, String, TestContext> { + public AssignCommand() { + super(10); + } + + @Override + public ITree<Token<String, String>> denote(ITree<Token<String, String>> operand, Token<String, String> operator, + ParserContext<String, String, TestContext> ctx) throws ParserException { + Token<String, String> name = operand.getHead(); + + switch(name.getKey()) { + case "(literal)": + case "(vref)": + break; + default: + throw new ParserException("Variable name must be simple"); + } + + ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + ctx.state.scopes.top().putKey(name.getValue(), body); + + return new Tree<>(new StringToken("assign", "assign"), operand, body); + } +} diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java index 3462d74..110d331 100644 --- a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java +++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java @@ -25,7 +25,7 @@ import static bjc.utils.parserutils.pratt.commands.NonInitialCommands.*; import static bjc.utils.parserutils.pratt.commands.InitialCommands.*; /** - * Simple test for pratt parser. + * Simple test for Pratt parser. * * @author EVE * @@ -45,7 +45,7 @@ public class PrattParserTest { private static final class BlockEnter implements UnaryOperator<TestContext> { @Override public TestContext apply(TestContext state) { - Directory<String, String> enclosing = state.scopes.top(); + Directory<String, ITree<Token<String, String>>> enclosing = state.scopes.top(); int currBlockNumber = state.blockCount.pop(); state.scopes.push(enclosing.newSubdirectory("block" + currBlockNumber)); @@ -73,8 +73,7 @@ public class PrattParserTest { ops.addAll(Arrays.asList("||", "&&")); ops.addAll(Arrays.asList("<=", ">=")); - ops.add("."); - ops.add(";"); + ops.addAll(Arrays.asList(".", ",", ";", ":")); ops.addAll(Arrays.asList("=", "<", ">")); ops.addAll(Arrays.asList("+", "-", "*", "/")); ops.addAll(Arrays.asList("^", "!")); @@ -89,6 +88,8 @@ public class PrattParserTest { reserved.addAll(Arrays.asList("if", "then", "else")); reserved.addAll(Arrays.asList("and", "or")); reserved.addAll(Arrays.asList("begin", "end")); + reserved.addAll(Arrays.asList("switch", "case")); + reserved.add("var"); TwoLevelSplitter split = new TwoLevelSplitter(); @@ -96,15 +97,17 @@ public class PrattParserTest { split.addCompoundDelim("||", "&&"); split.addCompoundDelim("<=", ">="); - split.addSimpleDelim("."); - split.addSimpleDelim(";"); + split.addSimpleDelim(".", ",", ";", ":"); split.addSimpleDelim("=", "<", ">"); split.addSimpleDelim("+", "-", "*", "/"); split.addSimpleDelim("^", "!"); + split.addSimpleMulti("\\(", "\\)"); split.addSimpleMulti("\\[", "\\]"); split.addSimpleMulti("\\{", "\\}"); + split.exclude(reserved.toArray(new String[0])); + split.compile(); PrattParser<String, String, TestContext> parser = createParser(); @@ -130,7 +133,7 @@ public class PrattParserTest { ITree<Token<String, String>> tree = parser.parseExpression(0, tokenStream, ctx, true); if(!tokenStream.current().getKey().equals("(end)")) { - System.out.println("Multipe expressions on line"); + System.out.println("Multiple expressions on line"); } System.out.println("Parsed expression:\n" + tree); @@ -143,7 +146,7 @@ public class PrattParserTest { } System.out.println(); - System.out.println("Context is:\n" + ctx); + System.out.println("Context is: " + ctx); scn.close(); } @@ -155,9 +158,22 @@ public class PrattParserTest { List<String> splitTokens = new LinkedList<>(); for(String raw : rawTokens) { - String[] strangs = split.split(raw); + boolean doSplit = false; - splitTokens.addAll(Arrays.asList(strangs)); + for(String op : ops) { + if(raw.contains(op)) { + doSplit = true; + break; + } + } + + if(doSplit) { + String[] strangs = split.split(raw); + + splitTokens.addAll(Arrays.asList(strangs)); + } else { + splitTokens.add(raw); + } } System.out.println("Split string: " + splitTokens); @@ -167,6 +183,8 @@ public class PrattParserTest { Iterator<Token<String, String>> tokens = new TransformIterator<>(source, (String strang) -> { if(ops.contains(strang) || reserved.contains(strang)) { return new StringToken(strang, strang); + } else if(ctx.scopes.top().containsKey(strang)) { + return new StringToken("(vref)", strang); } else { return new StringToken("(literal)", strang); } @@ -188,9 +206,11 @@ public class PrattParserTest { PrattParser<String, String, TestContext> parser = new PrattParser<>(); + parser.addNonInitialCommand(":", infixNon(3)); + parser.addNonInitialCommand("if", ternary(5, 0, "else", new StringToken("cond", "cond"), false)); - parser.addNonInitialCommand(":=", infixNon(10)); + parser.addNonInitialCommand(":=", new AssignCommand()); parser.addNonInitialCommand("and", infixLeft(13)); parser.addNonInitialCommand("or", infixLeft(13)); @@ -223,12 +243,25 @@ public class PrattParserTest { parser.addInitialCommand("(", grouping(0, ")", new StringToken("parens", "parens"))); - parser.addInitialCommand("{", delimited(0, ";", "}", new StringToken("block", "block"), + parser.addInitialCommand("begin", delimited(0, ";", "end", new StringToken("block", "block"), new BlockEnter(), (state) -> state, new BlockExit(), true)); + parser.addInitialCommand("[", delimited(0, ",", "]", new StringToken("array", "array"), + (state) -> state, (state) -> state, (state) -> state, false)); + + parser.addInitialCommand("{", delimited(0, ",", "}", new StringToken("json", "json"), (state) -> state, + (state) -> state, (state) -> state, false)); + + parser.addInitialCommand("case", unary(5)); + parser.addInitialCommand("-", unary(30)); parser.addInitialCommand("(literal)", leaf()); + parser.addInitialCommand("(vref)", leaf()); + + parser.addInitialCommand("var", new VarCommand()); + + parser.addInitialCommand("switch", new SwitchCommand()); return parser; } diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/SwitchCommand.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/SwitchCommand.java new file mode 100644 index 0000000..7464709 --- /dev/null +++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/SwitchCommand.java @@ -0,0 +1,21 @@ +package bjc.utils.examples.parsing; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.InitialCommand; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.StringToken; +import bjc.utils.parserutils.pratt.Token; + +class SwitchCommand implements InitialCommand<String, String, TestContext> { + @Override + public ITree<Token<String, String>> denote(Token<String, String> operator, + ParserContext<String, String, TestContext> ctx) throws ParserException { + ITree<Token<String, String>> object = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + return new Tree<>(new StringToken("switch", "switch"), object, body); + } +} diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java index cc1be17..7f9986e 100644 --- a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java +++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/TestContext.java @@ -1,9 +1,11 @@ package bjc.utils.examples.parsing; +import bjc.utils.data.ITree; import bjc.utils.esodata.Directory; import bjc.utils.esodata.SimpleDirectory; import bjc.utils.esodata.SimpleStack; import bjc.utils.esodata.Stack; +import bjc.utils.parserutils.pratt.Token; /** * Simple context for the parser. @@ -15,7 +17,7 @@ public class TestContext { /** * The variable scoping information. */ - public Stack<Directory<String, String>> scopes; + public Stack<Directory<String, ITree<Token<String, String>>>> scopes; /** * The current number of scopes inside this scope. @@ -35,6 +37,6 @@ public class TestContext { @Override public String toString() { - return String.format("TestContext [scopes=%s, blockCount=%s]", scopes, blockCount); + return String.format("TestContext [scopes=%s\n, blockCount=%s]", scopes, blockCount); } } diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/VarCommand.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/VarCommand.java new file mode 100644 index 0000000..b702f0e --- /dev/null +++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/parsing/VarCommand.java @@ -0,0 +1,36 @@ +package bjc.utils.examples.parsing; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.StringToken; +import bjc.utils.parserutils.pratt.Token; +import bjc.utils.parserutils.pratt.commands.AbstractInitialCommand; + +class VarCommand extends AbstractInitialCommand<String, String, TestContext> { + + @Override + protected ITree<Token<String, String>> intNullDenotation(Token<String, String> operator, + ParserContext<String, String, TestContext> ctx) throws ParserException { + Token<String, String> name = ctx.tokens.current(); + + switch(name.getKey()) { + case "(literal)": + case "(vref)": + ctx.tokens.next(); + break; + default: + throw new ParserException("Variable name must be simple"); + } + + ctx.tokens.expect("="); + + ITree<Token<String, String>> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + ctx.state.scopes.top().putKey(name.getValue(), body); + + return new Tree<>(new StringToken("var-bind", "var-bind"), new Tree<>(name), body); + } + +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialInterleaveCommand.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialInterleaveCommand.java new file mode 100644 index 0000000..554cf30 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InitialInterleaveCommand.java @@ -0,0 +1,29 @@ +package bjc.utils.parserutils.pratt.commands; + +import bjc.utils.data.ITree; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.pratt.ParserContext; +import bjc.utils.parserutils.pratt.Token; + +/** + * Represents a configurable initial command. + * + * @author EVE + * + * @param <K> + * The token key type. + * + * @param <V> + * The token value type. + * + * @param <C> + * The parser state type. + */ +public class InitialInterleaveCommand<K, V, C> extends AbstractInitialCommand<K, V, C> { + @Override + protected ITree<Token<K, V>> intNullDenotation(Token<K, V> operator, ParserContext<K, V, C> ctx) + throws ParserException { + return null; + } + +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InterleaveSpecifier.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InterleaveSpecifier.java new file mode 100644 index 0000000..fa2c592 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/pratt/commands/InterleaveSpecifier.java @@ -0,0 +1,30 @@ +package bjc.utils.parserutils.pratt.commands; + +/** + * The specifier for what to do for each element of a command. + * + * @author EVE + * + */ +public class InterleaveSpecifier { + /** + * The type of this specifier. + * + * @author EVE + * + */ + public static enum Type { + /** + * Parse an expression with the given priority. + */ + EXPRESSION, + /** + * Parse a statement with the given priority. + */ + STATEMENT, + /** + * Expect a token of a certain type to be present. + */ + EXPECT, LITERAL; + } +}
\ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TwoLevelSplitter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TwoLevelSplitter.java index 38f303d..d65b06a 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TwoLevelSplitter.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/TwoLevelSplitter.java @@ -100,6 +100,20 @@ public class TwoLevelSplitter implements TokenSplitter { } /** + * Exclude strings matching a regex from both splits. + * + * @param exclusions + * The regexes to exclude matches for. + */ + public void exclude(String... exclusions) { + for(String exclusion : exclusions) { + high.addNonMatcher(exclusion); + + low.addNonMatcher(exclusion); + } + } + + /** * Ready the splitter for use. */ public void compile() { |