From c82452e59b1547392c3e89d08d9173cc6dc79e23 Mon Sep 17 00:00:00 2001 From: bjculkin Date: Wed, 5 Apr 2017 15:35:13 -0400 Subject: Reorganize --- .../java/bjc/pratt/examples/AssignCommand.java | 35 +++ .../java/bjc/pratt/examples/BlockEnter.java | 22 ++ .../java/bjc/pratt/examples/BlockExit.java | 14 ++ .../java/bjc/pratt/examples/PrattParserTest.java | 253 ++++++++++++++++++++ .../java/bjc/pratt/examples/SwitchCommand.java | 21 ++ .../java/bjc/pratt/examples/TestContext.java | 42 ++++ .../java/bjc/pratt/examples/Tokenizer.java | 32 +++ .../java/bjc/pratt/examples/VarCommand.java | 36 +++ .../bjc/utils/examples/parsing/AssignCommand.java | 35 --- .../bjc/utils/examples/parsing/BlockEnter.java | 22 -- .../java/bjc/utils/examples/parsing/BlockExit.java | 14 -- .../utils/examples/parsing/PrattParserTest.java | 255 --------------------- .../bjc/utils/examples/parsing/SwitchCommand.java | 21 -- .../bjc/utils/examples/parsing/TestContext.java | 42 ---- .../java/bjc/utils/examples/parsing/Tokenizer.java | 32 --- .../bjc/utils/examples/parsing/VarCommand.java | 36 --- 16 files changed, 455 insertions(+), 457 deletions(-) create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/AssignCommand.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/BlockEnter.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/BlockExit.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/SwitchCommand.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/TestContext.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/VarCommand.java delete mode 100644 JPratt/src/examples/java/bjc/utils/examples/parsing/AssignCommand.java delete mode 100644 JPratt/src/examples/java/bjc/utils/examples/parsing/BlockEnter.java delete mode 100644 JPratt/src/examples/java/bjc/utils/examples/parsing/BlockExit.java delete mode 100644 JPratt/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java delete mode 100644 JPratt/src/examples/java/bjc/utils/examples/parsing/SwitchCommand.java delete mode 100644 JPratt/src/examples/java/bjc/utils/examples/parsing/TestContext.java delete mode 100644 JPratt/src/examples/java/bjc/utils/examples/parsing/Tokenizer.java delete mode 100644 JPratt/src/examples/java/bjc/utils/examples/parsing/VarCommand.java (limited to 'JPratt/src/examples/java/bjc') diff --git a/JPratt/src/examples/java/bjc/pratt/examples/AssignCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/AssignCommand.java new file mode 100644 index 0000000..eb99350 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/AssignCommand.java @@ -0,0 +1,35 @@ +package bjc.pratt.examples; + +import bjc.pratt.ParserContext; +import bjc.pratt.Token; +import bjc.pratt.commands.NonBinaryCommand; +import bjc.pratt.tokens.StringToken; +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; + +class AssignCommand extends NonBinaryCommand { + public AssignCommand(int prec) { + super(prec); + } + + @Override + public ITree> denote(ITree> operand, Token operator, + ParserContext ctx) throws ParserException { + Token name = operand.getHead(); + + switch(name.getKey()) { + case "(literal)": + case "(vref)": + break; + default: + throw new ParserException("Variable name must be simple"); + } + + ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + ctx.state.scopes.top().putKey(name.getValue(), body); + + return new Tree<>(new StringToken("assign", "assign"), operand, body); + } +} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/BlockEnter.java b/JPratt/src/examples/java/bjc/pratt/examples/BlockEnter.java new file mode 100644 index 0000000..71d4e72 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/BlockEnter.java @@ -0,0 +1,22 @@ +package bjc.pratt.examples; + +import bjc.pratt.Token; +import bjc.utils.data.ITree; +import bjc.utils.esodata.Directory; + +import java.util.function.UnaryOperator; + +final class BlockEnter implements UnaryOperator { + @Override + public TestContext apply(TestContext state) { + Directory>> enclosing = state.scopes.top(); + int currBlockNumber = state.blockCount.pop(); + + state.scopes.push(enclosing.newSubdirectory("block" + currBlockNumber)); + + state.blockCount.push(currBlockNumber + 1); + state.blockCount.push(0); + + return state; + } +} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/pratt/examples/BlockExit.java b/JPratt/src/examples/java/bjc/pratt/examples/BlockExit.java new file mode 100644 index 0000000..1cca6d5 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/BlockExit.java @@ -0,0 +1,14 @@ +package bjc.pratt.examples; + +import java.util.function.UnaryOperator; + +final class BlockExit implements UnaryOperator { + @Override + public TestContext apply(TestContext state) { + state.scopes.pop(); + + state.blockCount.pop(); + + return state; + } +} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java b/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java new file mode 100644 index 0000000..4a9ae56 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java @@ -0,0 +1,253 @@ +package bjc.pratt.examples; + +import bjc.pratt.InitialCommand; +import bjc.pratt.NonInitialCommand; +import bjc.pratt.PrattParser; +import bjc.pratt.Token; +import bjc.pratt.tokens.StringToken; +import bjc.pratt.tokens.StringTokenStream; +import bjc.utils.data.ITree; +import bjc.utils.data.TransformIterator; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.splitter.TokenSplitter; +import bjc.utils.parserutils.splitter.TwoLevelSplitter; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Scanner; +import java.util.Set; +import java.util.function.UnaryOperator; + +import static bjc.pratt.commands.InitialCommands.*; +import static bjc.pratt.commands.NonInitialCommands.*; +import static bjc.pratt.tokens.StringToken.litToken; +import static bjc.utils.functypes.ID.id; + +/** + * Simple test for Pratt parser. + * + * @author EVE + * + */ +public class PrattParserTest { + /** + * Main method. + * + * @param args + * Unused CLI arguments. + */ + public static void main(String[] args) { + /* + * Use a linked hash set to preserve insertion order. + */ + Set ops = new LinkedHashSet<>(); + + ops.addAll(Arrays.asList("->")); + ops.add(":="); + ops.addAll(Arrays.asList("||", "&&")); + ops.addAll(Arrays.asList("<=", ">=")); + + ops.addAll(Arrays.asList(".", ",", ";", ":")); + ops.addAll(Arrays.asList("=", "<", ">")); + ops.addAll(Arrays.asList("+", "-", "*", "/")); + ops.addAll(Arrays.asList("^", "!")); + ops.addAll(Arrays.asList("(", ")")); + ops.addAll(Arrays.asList("[", "]")); + ops.addAll(Arrays.asList("{", "}")); + + /* + * Reserved words that represent themselves, not literals. + */ + Set reserved = new LinkedHashSet<>(); + reserved.addAll(Arrays.asList("if", "then", "else")); + reserved.addAll(Arrays.asList("and", "or")); + reserved.addAll(Arrays.asList("begin", "end")); + reserved.addAll(Arrays.asList("switch", "case")); + reserved.addAll(Arrays.asList("sqrt", "cbrt", "root")); + reserved.add("var"); + + TwoLevelSplitter split = new TwoLevelSplitter(); + + split.addCompoundDelim("->"); + split.addCompoundDelim(":="); + split.addCompoundDelim("||", "&&"); + split.addCompoundDelim("<=", ">="); + + split.addSimpleDelim(".", ",", ";", ":"); + split.addSimpleDelim("=", "<", ">"); + split.addSimpleDelim("+", "-", "*", "/"); + split.addSimpleDelim("^", "!"); + + split.addSimpleMulti("\\(", "\\)"); + split.addSimpleMulti("\\[", "\\]"); + split.addSimpleMulti("\\{", "\\}"); + + split.exclude(reserved.toArray(new String[0])); + + split.compile(); + + PrattParser parser = createParser(); + + TestContext ctx = new TestContext(); + + Scanner scn = new Scanner(System.in); + + System.out.print("Enter a command (blank line to exit): "); + String ln = scn.nextLine(); + + while(!ln.trim().equals("")) { + Iterator> tokens = preprocessInput(ops, split, ln, reserved, ctx); + + try { + StringTokenStream tokenStream = new StringTokenStream(tokens); + + /* + * Prime stream. + */ + tokenStream.next(); + + ITree> tree = parser.parseExpression(0, tokenStream, ctx, true); + + if(!tokenStream.headIs("(end)")) { + System.out.println("\nMultiple expressions on line"); + } + + System.out.println("\nParsed expression:\n" + tree); + } catch(ParserException pex) { + pex.printStackTrace(); + } + + System.out.print("\nEnter a command (blank line to exit): "); + ln = scn.nextLine(); + } + + System.out.println(); + System.out.println("\nContext is: " + ctx); + + scn.close(); + } + + private static Iterator> preprocessInput(Set ops, TokenSplitter split, String ln, + Set reserved, TestContext ctx) { + String[] rawTokens = ln.split("\\s+"); + + List splitTokens = new LinkedList<>(); + + for(String raw : rawTokens) { + boolean doSplit = false; + + for(String op : ops) { + if(raw.contains(op)) { + doSplit = true; + break; + } + } + + if(doSplit) { + String[] strangs = split.split(raw); + + splitTokens.addAll(Arrays.asList(strangs)); + } else { + splitTokens.add(raw); + } + } + + System.out.println("\nSplit string: " + splitTokens); + + Iterator source = splitTokens.iterator(); + + Iterator> tokens = new TransformIterator<>(source, + new Tokenizer(ops, reserved, ctx)); + return tokens; + } + + private static PrattParser createParser() { + /* + * Set of which relational operators chain with each other. + */ + HashSet relChain = new HashSet<>(); + relChain.addAll(Arrays.asList("=", "<", ">", "<=", ">=")); + + /* + * Token for marking chains. + */ + StringToken chainToken = litToken("and"); + + /* + * ID function. + */ + UnaryOperator idfun = id(); + + PrattParser parser = new PrattParser<>(); + + parser.addNonInitialCommand(":", infixNon(3)); + + parser.addNonInitialCommand("if", ternary(5, 0, "else", litToken("cond"), false)); + + parser.addNonInitialCommand(":=", new AssignCommand(10)); + + parser.addNonInitialCommand("->", infixRight(11)); + + NonInitialCommand nonSSRelJoin = infixLeft(13); + parser.addNonInitialCommand("and", nonSSRelJoin); + parser.addNonInitialCommand("or", nonSSRelJoin); + + NonInitialCommand chainRelOp = chain(15, relChain, chainToken); + parser.addNonInitialCommand("=", chainRelOp); + parser.addNonInitialCommand("<", chainRelOp); + parser.addNonInitialCommand(">", chainRelOp); + parser.addNonInitialCommand("<=", chainRelOp); + parser.addNonInitialCommand(">=", chainRelOp); + + NonInitialCommand ssRelJoin = infixRight(17); + parser.addNonInitialCommand("&&", ssRelJoin); + parser.addNonInitialCommand("||", ssRelJoin); + + NonInitialCommand addSub = infixLeft(20); + parser.addNonInitialCommand("+", addSub); + parser.addNonInitialCommand("-", addSub); + + NonInitialCommand mulDiv = infixLeft(30); + parser.addNonInitialCommand("*", mulDiv); + parser.addNonInitialCommand("/", mulDiv); + + parser.addNonInitialCommand("!", postfix(40)); + + NonInitialCommand expon = infixRight(50); + parser.addNonInitialCommand("^", expon); + parser.addNonInitialCommand("root", expon); + + parser.addNonInitialCommand(".", infixLeft(60)); + + parser.addNonInitialCommand("[", postCircumfix(60, 0, "]", litToken("idx"))); + + parser.addInitialCommand("if", preTernary(0, 0, 0, "then", "else", litToken("ifelse"))); + + parser.addInitialCommand("(", grouping(0, ")", litToken("parens"))); + + parser.addInitialCommand("begin", delimited(0, ";", "end", litToken("block"), new BlockEnter(), idfun, + new BlockExit(), true)); + + parser.addInitialCommand("[", delimited(0, ",", "]", litToken("array"), idfun, idfun, idfun, false)); + + parser.addInitialCommand("{", delimited(0, ",", "}", litToken("json"), idfun, idfun, idfun, false)); + + parser.addInitialCommand("case", unary(5)); + + parser.addInitialCommand("-", unary(30)); + + InitialCommand leaf = leaf(); + parser.addInitialCommand("(literal)", leaf); + parser.addInitialCommand("(vref)", leaf); + + parser.addInitialCommand("var", new VarCommand()); + + parser.addInitialCommand("switch", new SwitchCommand()); + + return parser; + } +} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/pratt/examples/SwitchCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/SwitchCommand.java new file mode 100644 index 0000000..4a5bd22 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/SwitchCommand.java @@ -0,0 +1,21 @@ +package bjc.pratt.examples; + +import bjc.pratt.InitialCommand; +import bjc.pratt.ParserContext; +import bjc.pratt.Token; +import bjc.pratt.tokens.StringToken; +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; + +class SwitchCommand implements InitialCommand { + @Override + public ITree> denote(Token operator, + ParserContext ctx) throws ParserException { + ITree> object = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + return new Tree<>(new StringToken("switch", "switch"), object, body); + } +} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/TestContext.java b/JPratt/src/examples/java/bjc/pratt/examples/TestContext.java new file mode 100644 index 0000000..b5e8448 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/TestContext.java @@ -0,0 +1,42 @@ +package bjc.pratt.examples; + +import bjc.pratt.Token; +import bjc.utils.data.ITree; +import bjc.utils.esodata.Directory; +import bjc.utils.esodata.SimpleDirectory; +import bjc.utils.esodata.SimpleStack; +import bjc.utils.esodata.Stack; + +/** + * Simple context for the parser. + * + * @author EVE + * + */ +public class TestContext { + /** + * The variable scoping information. + */ + public Stack>>> scopes; + + /** + * The current number of scopes inside this scope. + */ + public Stack blockCount; + + /** + * Create a new test context. + */ + public TestContext() { + scopes = new SimpleStack<>(); + blockCount = new SimpleStack<>(); + + scopes.push(new SimpleDirectory<>()); + blockCount.push(0); + } + + @Override + public String toString() { + return String.format("TestContext [scopes=%s\n, blockCount=%s]", scopes, blockCount); + } +} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java b/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java new file mode 100644 index 0000000..563b578 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java @@ -0,0 +1,32 @@ +package bjc.pratt.examples; + +import bjc.pratt.Token; +import bjc.pratt.tokens.StringToken; + +import java.util.Set; +import java.util.function.Function; + +import static bjc.pratt.tokens.StringToken.litToken; + +final class Tokenizer implements Function> { + private Set ops; + private Set reserved; + private TestContext ctx; + + public Tokenizer(Set operators, Set reservedWords, TestContext context) { + ops = operators; + reserved = reservedWords; + ctx = context; + } + + @Override + public Token apply(String strang) { + if (ops.contains(strang) || reserved.contains(strang)) { + return litToken(strang); + } else if (ctx.scopes.top().containsKey(strang)) { + return new StringToken("(vref)", strang); + } else { + return new StringToken("(literal)", strang); + } + } +} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/pratt/examples/VarCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/VarCommand.java new file mode 100644 index 0000000..dd04a59 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/VarCommand.java @@ -0,0 +1,36 @@ +package bjc.pratt.examples; + +import bjc.pratt.ParserContext; +import bjc.pratt.Token; +import bjc.pratt.commands.AbstractInitialCommand; +import bjc.pratt.tokens.StringToken; +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; + +class VarCommand extends AbstractInitialCommand { + + @Override + protected ITree> intNullDenotation(Token operator, + ParserContext ctx) throws ParserException { + Token name = ctx.tokens.current(); + + switch(name.getKey()) { + case "(literal)": + case "(vref)": + ctx.tokens.next(); + break; + default: + throw new ParserException("Variable name must be simple"); + } + + ctx.tokens.expect("="); + + ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + ctx.state.scopes.top().putKey(name.getValue(), body); + + return new Tree<>(new StringToken("var-bind", "var-bind"), new Tree<>(name), body); + } + +} diff --git a/JPratt/src/examples/java/bjc/utils/examples/parsing/AssignCommand.java b/JPratt/src/examples/java/bjc/utils/examples/parsing/AssignCommand.java deleted file mode 100644 index 4bdd60f..0000000 --- a/JPratt/src/examples/java/bjc/utils/examples/parsing/AssignCommand.java +++ /dev/null @@ -1,35 +0,0 @@ -package bjc.utils.examples.parsing; - -import bjc.utils.data.ITree; -import bjc.utils.data.Tree; -import bjc.utils.parserutils.ParserException; -import bjc.utils.parserutils.pratt.ParserContext; -import bjc.utils.parserutils.pratt.Token; -import bjc.utils.parserutils.pratt.commands.NonBinaryCommand; -import bjc.utils.parserutils.pratt.tokens.StringToken; - -class AssignCommand extends NonBinaryCommand { - public AssignCommand(int prec) { - super(prec); - } - - @Override - public ITree> denote(ITree> operand, Token operator, - ParserContext ctx) throws ParserException { - Token name = operand.getHead(); - - switch(name.getKey()) { - case "(literal)": - case "(vref)": - break; - default: - throw new ParserException("Variable name must be simple"); - } - - ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); - - ctx.state.scopes.top().putKey(name.getValue(), body); - - return new Tree<>(new StringToken("assign", "assign"), operand, body); - } -} diff --git a/JPratt/src/examples/java/bjc/utils/examples/parsing/BlockEnter.java b/JPratt/src/examples/java/bjc/utils/examples/parsing/BlockEnter.java deleted file mode 100644 index ac04720..0000000 --- a/JPratt/src/examples/java/bjc/utils/examples/parsing/BlockEnter.java +++ /dev/null @@ -1,22 +0,0 @@ -package bjc.utils.examples.parsing; - -import bjc.utils.data.ITree; -import bjc.utils.esodata.Directory; -import bjc.utils.parserutils.pratt.Token; - -import java.util.function.UnaryOperator; - -final class BlockEnter implements UnaryOperator { - @Override - public TestContext apply(TestContext state) { - Directory>> enclosing = state.scopes.top(); - int currBlockNumber = state.blockCount.pop(); - - state.scopes.push(enclosing.newSubdirectory("block" + currBlockNumber)); - - state.blockCount.push(currBlockNumber + 1); - state.blockCount.push(0); - - return state; - } -} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/utils/examples/parsing/BlockExit.java b/JPratt/src/examples/java/bjc/utils/examples/parsing/BlockExit.java deleted file mode 100644 index b067464..0000000 --- a/JPratt/src/examples/java/bjc/utils/examples/parsing/BlockExit.java +++ /dev/null @@ -1,14 +0,0 @@ -package bjc.utils.examples.parsing; - -import java.util.function.UnaryOperator; - -final class BlockExit implements UnaryOperator { - @Override - public TestContext apply(TestContext state) { - state.scopes.pop(); - - state.blockCount.pop(); - - return state; - } -} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java b/JPratt/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java deleted file mode 100644 index 2cc53b0..0000000 --- a/JPratt/src/examples/java/bjc/utils/examples/parsing/PrattParserTest.java +++ /dev/null @@ -1,255 +0,0 @@ -package bjc.utils.examples.parsing; - -import bjc.utils.data.ITree; -import bjc.utils.data.TransformIterator; -import bjc.utils.parserutils.ParserException; -import bjc.utils.parserutils.pratt.InitialCommand; -import bjc.utils.parserutils.pratt.NonInitialCommand; -import bjc.utils.parserutils.pratt.PrattParser; -import bjc.utils.parserutils.pratt.Token; -import bjc.utils.parserutils.pratt.tokens.StringToken; -import bjc.utils.parserutils.pratt.tokens.StringTokenStream; -import bjc.utils.parserutils.splitter.TokenSplitter; -import bjc.utils.parserutils.splitter.TwoLevelSplitter; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Scanner; -import java.util.Set; -import java.util.function.UnaryOperator; - -import static bjc.utils.parserutils.pratt.commands.NonInitialCommands.*; -import static bjc.utils.parserutils.pratt.commands.InitialCommands.*; - -import static bjc.utils.parserutils.pratt.tokens.StringToken.litToken; - -import static bjc.utils.functypes.ID.id; - -/** - * Simple test for Pratt parser. - * - * @author EVE - * - */ -public class PrattParserTest { - /** - * Main method. - * - * @param args - * Unused CLI arguments. - */ - public static void main(String[] args) { - /* - * Use a linked hash set to preserve insertion order. - */ - Set ops = new LinkedHashSet<>(); - - ops.addAll(Arrays.asList("->")); - ops.add(":="); - ops.addAll(Arrays.asList("||", "&&")); - ops.addAll(Arrays.asList("<=", ">=")); - - ops.addAll(Arrays.asList(".", ",", ";", ":")); - ops.addAll(Arrays.asList("=", "<", ">")); - ops.addAll(Arrays.asList("+", "-", "*", "/")); - ops.addAll(Arrays.asList("^", "!")); - ops.addAll(Arrays.asList("(", ")")); - ops.addAll(Arrays.asList("[", "]")); - ops.addAll(Arrays.asList("{", "}")); - - /* - * Reserved words that represent themselves, not literals. - */ - Set reserved = new LinkedHashSet<>(); - reserved.addAll(Arrays.asList("if", "then", "else")); - reserved.addAll(Arrays.asList("and", "or")); - reserved.addAll(Arrays.asList("begin", "end")); - reserved.addAll(Arrays.asList("switch", "case")); - reserved.addAll(Arrays.asList("sqrt", "cbrt", "root")); - reserved.add("var"); - - TwoLevelSplitter split = new TwoLevelSplitter(); - - split.addCompoundDelim("->"); - split.addCompoundDelim(":="); - split.addCompoundDelim("||", "&&"); - split.addCompoundDelim("<=", ">="); - - split.addSimpleDelim(".", ",", ";", ":"); - split.addSimpleDelim("=", "<", ">"); - split.addSimpleDelim("+", "-", "*", "/"); - split.addSimpleDelim("^", "!"); - - split.addSimpleMulti("\\(", "\\)"); - split.addSimpleMulti("\\[", "\\]"); - split.addSimpleMulti("\\{", "\\}"); - - split.exclude(reserved.toArray(new String[0])); - - split.compile(); - - PrattParser parser = createParser(); - - TestContext ctx = new TestContext(); - - Scanner scn = new Scanner(System.in); - - System.out.print("Enter a command (blank line to exit): "); - String ln = scn.nextLine(); - - while(!ln.trim().equals("")) { - Iterator> tokens = preprocessInput(ops, split, ln, reserved, ctx); - - try { - StringTokenStream tokenStream = new StringTokenStream(tokens); - - /* - * Prime stream. - */ - tokenStream.next(); - - ITree> tree = parser.parseExpression(0, tokenStream, ctx, true); - - if(!tokenStream.headIs("(end)")) { - System.out.println("\nMultiple expressions on line"); - } - - System.out.println("\nParsed expression:\n" + tree); - } catch(ParserException pex) { - pex.printStackTrace(); - } - - System.out.print("\nEnter a command (blank line to exit): "); - ln = scn.nextLine(); - } - - System.out.println(); - System.out.println("\nContext is: " + ctx); - - scn.close(); - } - - private static Iterator> preprocessInput(Set ops, TokenSplitter split, String ln, - Set reserved, TestContext ctx) { - String[] rawTokens = ln.split("\\s+"); - - List splitTokens = new LinkedList<>(); - - for(String raw : rawTokens) { - boolean doSplit = false; - - for(String op : ops) { - if(raw.contains(op)) { - doSplit = true; - break; - } - } - - if(doSplit) { - String[] strangs = split.split(raw); - - splitTokens.addAll(Arrays.asList(strangs)); - } else { - splitTokens.add(raw); - } - } - - System.out.println("\nSplit string: " + splitTokens); - - Iterator source = splitTokens.iterator(); - - Iterator> tokens = new TransformIterator<>(source, - new Tokenizer(ops, reserved, ctx)); - return tokens; - } - - private static PrattParser createParser() { - /* - * Set of which relational operators chain with each other. - */ - HashSet relChain = new HashSet<>(); - relChain.addAll(Arrays.asList("=", "<", ">", "<=", ">=")); - - /* - * Token for marking chains. - */ - StringToken chainToken = litToken("and"); - - /* - * ID function. - */ - UnaryOperator idfun = id(); - - PrattParser parser = new PrattParser<>(); - - parser.addNonInitialCommand(":", infixNon(3)); - - parser.addNonInitialCommand("if", ternary(5, 0, "else", litToken("cond"), false)); - - parser.addNonInitialCommand(":=", new AssignCommand(10)); - - parser.addNonInitialCommand("->", infixRight(11)); - - NonInitialCommand nonSSRelJoin = infixLeft(13); - parser.addNonInitialCommand("and", nonSSRelJoin); - parser.addNonInitialCommand("or", nonSSRelJoin); - - NonInitialCommand chainRelOp = chain(15, relChain, chainToken); - parser.addNonInitialCommand("=", chainRelOp); - parser.addNonInitialCommand("<", chainRelOp); - parser.addNonInitialCommand(">", chainRelOp); - parser.addNonInitialCommand("<=", chainRelOp); - parser.addNonInitialCommand(">=", chainRelOp); - - NonInitialCommand ssRelJoin = infixRight(17); - parser.addNonInitialCommand("&&", ssRelJoin); - parser.addNonInitialCommand("||", ssRelJoin); - - NonInitialCommand addSub = infixLeft(20); - parser.addNonInitialCommand("+", addSub); - parser.addNonInitialCommand("-", addSub); - - NonInitialCommand mulDiv = infixLeft(30); - parser.addNonInitialCommand("*", mulDiv); - parser.addNonInitialCommand("/", mulDiv); - - parser.addNonInitialCommand("!", postfix(40)); - - NonInitialCommand expon = infixRight(50); - parser.addNonInitialCommand("^", expon); - parser.addNonInitialCommand("root", expon); - - parser.addNonInitialCommand(".", infixLeft(60)); - - parser.addNonInitialCommand("[", postCircumfix(60, 0, "]", litToken("idx"))); - - parser.addInitialCommand("if", preTernary(0, 0, 0, "then", "else", litToken("ifelse"))); - - parser.addInitialCommand("(", grouping(0, ")", litToken("parens"))); - - parser.addInitialCommand("begin", delimited(0, ";", "end", litToken("block"), new BlockEnter(), idfun, - new BlockExit(), true)); - - parser.addInitialCommand("[", delimited(0, ",", "]", litToken("array"), idfun, idfun, idfun, false)); - - parser.addInitialCommand("{", delimited(0, ",", "}", litToken("json"), idfun, idfun, idfun, false)); - - parser.addInitialCommand("case", unary(5)); - - parser.addInitialCommand("-", unary(30)); - - InitialCommand leaf = leaf(); - parser.addInitialCommand("(literal)", leaf); - parser.addInitialCommand("(vref)", leaf); - - parser.addInitialCommand("var", new VarCommand()); - - parser.addInitialCommand("switch", new SwitchCommand()); - - return parser; - } -} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/utils/examples/parsing/SwitchCommand.java b/JPratt/src/examples/java/bjc/utils/examples/parsing/SwitchCommand.java deleted file mode 100644 index 78d8093..0000000 --- a/JPratt/src/examples/java/bjc/utils/examples/parsing/SwitchCommand.java +++ /dev/null @@ -1,21 +0,0 @@ -package bjc.utils.examples.parsing; - -import bjc.utils.data.ITree; -import bjc.utils.data.Tree; -import bjc.utils.parserutils.ParserException; -import bjc.utils.parserutils.pratt.InitialCommand; -import bjc.utils.parserutils.pratt.ParserContext; -import bjc.utils.parserutils.pratt.Token; -import bjc.utils.parserutils.pratt.tokens.StringToken; - -class SwitchCommand implements InitialCommand { - @Override - public ITree> denote(Token operator, - ParserContext ctx) throws ParserException { - ITree> object = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); - - ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); - - return new Tree<>(new StringToken("switch", "switch"), object, body); - } -} diff --git a/JPratt/src/examples/java/bjc/utils/examples/parsing/TestContext.java b/JPratt/src/examples/java/bjc/utils/examples/parsing/TestContext.java deleted file mode 100644 index 7f9986e..0000000 --- a/JPratt/src/examples/java/bjc/utils/examples/parsing/TestContext.java +++ /dev/null @@ -1,42 +0,0 @@ -package bjc.utils.examples.parsing; - -import bjc.utils.data.ITree; -import bjc.utils.esodata.Directory; -import bjc.utils.esodata.SimpleDirectory; -import bjc.utils.esodata.SimpleStack; -import bjc.utils.esodata.Stack; -import bjc.utils.parserutils.pratt.Token; - -/** - * Simple context for the parser. - * - * @author EVE - * - */ -public class TestContext { - /** - * The variable scoping information. - */ - public Stack>>> scopes; - - /** - * The current number of scopes inside this scope. - */ - public Stack blockCount; - - /** - * Create a new test context. - */ - public TestContext() { - scopes = new SimpleStack<>(); - blockCount = new SimpleStack<>(); - - scopes.push(new SimpleDirectory<>()); - blockCount.push(0); - } - - @Override - public String toString() { - return String.format("TestContext [scopes=%s\n, blockCount=%s]", scopes, blockCount); - } -} diff --git a/JPratt/src/examples/java/bjc/utils/examples/parsing/Tokenizer.java b/JPratt/src/examples/java/bjc/utils/examples/parsing/Tokenizer.java deleted file mode 100644 index a224de6..0000000 --- a/JPratt/src/examples/java/bjc/utils/examples/parsing/Tokenizer.java +++ /dev/null @@ -1,32 +0,0 @@ -package bjc.utils.examples.parsing; - -import bjc.utils.parserutils.pratt.Token; -import bjc.utils.parserutils.pratt.tokens.StringToken; - -import java.util.Set; -import java.util.function.Function; - -import static bjc.utils.parserutils.pratt.tokens.StringToken.litToken; - -final class Tokenizer implements Function> { - private Set ops; - private Set reserved; - private TestContext ctx; - - public Tokenizer(Set operators, Set reservedWords, TestContext context) { - ops = operators; - reserved = reservedWords; - ctx = context; - } - - @Override - public Token apply(String strang) { - if (ops.contains(strang) || reserved.contains(strang)) { - return litToken(strang); - } else if (ctx.scopes.top().containsKey(strang)) { - return new StringToken("(vref)", strang); - } else { - return new StringToken("(literal)", strang); - } - } -} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/utils/examples/parsing/VarCommand.java b/JPratt/src/examples/java/bjc/utils/examples/parsing/VarCommand.java deleted file mode 100644 index 5cf4b25..0000000 --- a/JPratt/src/examples/java/bjc/utils/examples/parsing/VarCommand.java +++ /dev/null @@ -1,36 +0,0 @@ -package bjc.utils.examples.parsing; - -import bjc.utils.data.ITree; -import bjc.utils.data.Tree; -import bjc.utils.parserutils.ParserException; -import bjc.utils.parserutils.pratt.ParserContext; -import bjc.utils.parserutils.pratt.Token; -import bjc.utils.parserutils.pratt.commands.AbstractInitialCommand; -import bjc.utils.parserutils.pratt.tokens.StringToken; - -class VarCommand extends AbstractInitialCommand { - - @Override - protected ITree> intNullDenotation(Token operator, - ParserContext ctx) throws ParserException { - Token name = ctx.tokens.current(); - - switch(name.getKey()) { - case "(literal)": - case "(vref)": - ctx.tokens.next(); - break; - default: - throw new ParserException("Variable name must be simple"); - } - - ctx.tokens.expect("="); - - ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); - - ctx.state.scopes.top().putKey(name.getValue(), body); - - return new Tree<>(new StringToken("var-bind", "var-bind"), new Tree<>(name), body); - } - -} -- cgit v1.2.3