From 3f74e1e25fd572adab34e53eb90edcf49404fbe5 Mon Sep 17 00:00:00 2001 From: bculkin2442 Date: Tue, 11 Apr 2017 21:57:37 -0400 Subject: Cleanup --- .../java/bjc/pratt/examples/AssignCommand.java | 36 --- .../java/bjc/pratt/examples/BlockEnter.java | 25 -- .../java/bjc/pratt/examples/BlockExit.java | 14 - .../java/bjc/pratt/examples/PrattParserTest.java | 303 --------------------- .../java/bjc/pratt/examples/SwitchCommand.java | 21 -- .../java/bjc/pratt/examples/TestContext.java | 42 --- .../java/bjc/pratt/examples/Tokenizer.java | 38 --- .../java/bjc/pratt/examples/VarCommand.java | 36 --- .../bjc/pratt/examples/lang/AssignCommand.java | 36 +++ .../java/bjc/pratt/examples/lang/BlockEnter.java | 25 ++ .../java/bjc/pratt/examples/lang/BlockExit.java | 14 + .../bjc/pratt/examples/lang/PrattParserTest.java | 303 +++++++++++++++++++++ .../bjc/pratt/examples/lang/SwitchCommand.java | 21 ++ .../java/bjc/pratt/examples/lang/TestContext.java | 42 +++ .../java/bjc/pratt/examples/lang/Tokenizer.java | 46 ++++ .../java/bjc/pratt/examples/lang/VarCommand.java | 36 +++ 16 files changed, 523 insertions(+), 515 deletions(-) delete mode 100644 JPratt/src/examples/java/bjc/pratt/examples/AssignCommand.java delete mode 100644 JPratt/src/examples/java/bjc/pratt/examples/BlockEnter.java delete mode 100644 JPratt/src/examples/java/bjc/pratt/examples/BlockExit.java delete mode 100644 JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java delete mode 100644 JPratt/src/examples/java/bjc/pratt/examples/SwitchCommand.java delete mode 100644 JPratt/src/examples/java/bjc/pratt/examples/TestContext.java delete mode 100644 JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java delete mode 100644 JPratt/src/examples/java/bjc/pratt/examples/VarCommand.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/lang/AssignCommand.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/lang/BlockEnter.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/lang/BlockExit.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/lang/PrattParserTest.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/lang/SwitchCommand.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/lang/TestContext.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/lang/Tokenizer.java create mode 100644 JPratt/src/examples/java/bjc/pratt/examples/lang/VarCommand.java (limited to 'JPratt/src/examples/java') diff --git a/JPratt/src/examples/java/bjc/pratt/examples/AssignCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/AssignCommand.java deleted file mode 100644 index f3cc003..0000000 --- a/JPratt/src/examples/java/bjc/pratt/examples/AssignCommand.java +++ /dev/null @@ -1,36 +0,0 @@ -package bjc.pratt.examples; - -import bjc.pratt.ParserContext; -import bjc.pratt.Token; -import bjc.pratt.commands.NonBinaryCommand; -import bjc.pratt.tokens.StringToken; -import bjc.utils.data.ITree; -import bjc.utils.data.Tree; -import bjc.utils.parserutils.ParserException; - -class AssignCommand extends NonBinaryCommand { - public AssignCommand(final int prec) { - super(prec); - } - - @Override - public ITree> denote(final ITree> operand, - final Token operator, final ParserContext ctx) - throws ParserException { - final Token name = operand.getHead(); - - switch (name.getKey()) { - case "(literal)": - case "(vref)": - break; - default: - throw new ParserException("Variable name must be simple"); - } - - final ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); - - ctx.state.scopes.top().putKey(name.getValue(), body); - - return new Tree<>(new StringToken("assign", "assign"), operand, body); - } -} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/BlockEnter.java b/JPratt/src/examples/java/bjc/pratt/examples/BlockEnter.java deleted file mode 100644 index 092c6b2..0000000 --- a/JPratt/src/examples/java/bjc/pratt/examples/BlockEnter.java +++ /dev/null @@ -1,25 +0,0 @@ -package bjc.pratt.examples; - -import java.util.function.UnaryOperator; - -import bjc.pratt.Token; -import bjc.utils.data.ITree; -import bjc.utils.esodata.Directory; -import bjc.utils.esodata.Stack; - -final class BlockEnter implements UnaryOperator { - @Override - public TestContext apply(final TestContext state) { - final Directory>> enclosing = state.scopes.top(); - final Stack blockCount = state.blockCount; - - final int currBlockNumber = blockCount.pop(); - - state.scopes.push(enclosing.newSubdirectory("block" + currBlockNumber)); - - blockCount.push(currBlockNumber + 1); - blockCount.push(0); - - return state; - } -} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/pratt/examples/BlockExit.java b/JPratt/src/examples/java/bjc/pratt/examples/BlockExit.java deleted file mode 100644 index ea9526d..0000000 --- a/JPratt/src/examples/java/bjc/pratt/examples/BlockExit.java +++ /dev/null @@ -1,14 +0,0 @@ -package bjc.pratt.examples; - -import java.util.function.UnaryOperator; - -final class BlockExit implements UnaryOperator { - @Override - public TestContext apply(final TestContext state) { - state.scopes.pop(); - - state.blockCount.pop(); - - return state; - } -} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java b/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java deleted file mode 100644 index 7b9feea..0000000 --- a/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java +++ /dev/null @@ -1,303 +0,0 @@ -package bjc.pratt.examples; - -import static bjc.pratt.commands.InitialCommands.delimited; -import static bjc.pratt.commands.InitialCommands.grouping; -import static bjc.pratt.commands.InitialCommands.leaf; -import static bjc.pratt.commands.InitialCommands.preTernary; -import static bjc.pratt.commands.InitialCommands.unary; -import static bjc.pratt.commands.NonInitialCommands.chain; -import static bjc.pratt.commands.NonInitialCommands.infixLeft; -import static bjc.pratt.commands.NonInitialCommands.infixNon; -import static bjc.pratt.commands.NonInitialCommands.infixRight; -import static bjc.pratt.commands.NonInitialCommands.postCircumfix; -import static bjc.pratt.commands.NonInitialCommands.postfix; -import static bjc.pratt.commands.NonInitialCommands.ternary; -import static bjc.pratt.tokens.StringToken.litToken; -import static bjc.utils.functypes.ID.id; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Scanner; -import java.util.Set; -import java.util.function.UnaryOperator; - -import bjc.pratt.InitialCommand; -import bjc.pratt.NonInitialCommand; -import bjc.pratt.PrattParser; -import bjc.pratt.Token; -import bjc.pratt.tokens.StringToken; -import bjc.pratt.tokens.StringTokenStream; -import bjc.utils.data.ITree; -import bjc.utils.data.TransformIterator; -import bjc.utils.parserutils.ParserException; -import bjc.utils.parserutils.splitterv2.ChainTokenSplitter; -import bjc.utils.parserutils.splitterv2.ConfigurableTokenSplitter; -import bjc.utils.parserutils.splitterv2.ExcludingTokenSplitter; -import bjc.utils.parserutils.splitterv2.TokenSplitter; - -/** - * Simple test for Pratt parser. - * - * @author EVE - * - */ -public class PrattParserTest { - /** - * Main method. - * - * @param args - * Unused CLI arguments. - */ - public static void main(final String[] args) { - /* - * Use a linked hash set to preserve insertion order. - */ - final Set ops = new LinkedHashSet<>(); - - ops.add("!!!"); - - ops.addAll(Arrays.asList("->", "=>")); - ops.add(":="); - ops.addAll(Arrays.asList("||", "&&")); - ops.addAll(Arrays.asList("<=", ">=")); - - ops.addAll(Arrays.asList("\u00B1")); // Unicode plus/minus - ops.addAll(Arrays.asList(".", ",", ";", ":")); - ops.addAll(Arrays.asList("=", "<", ">")); - ops.addAll(Arrays.asList("+", "-", "*", "/")); - ops.addAll(Arrays.asList("^", "!")); - ops.addAll(Arrays.asList("(", ")")); - ops.addAll(Arrays.asList("[", "]")); - ops.addAll(Arrays.asList("{", "}")); - - /* - * Reserved words that represent themselves, not literals. - */ - final Set reserved = new LinkedHashSet<>(); - reserved.addAll(Arrays.asList("if", "then", "else")); - reserved.addAll(Arrays.asList("and", "or")); - reserved.addAll(Arrays.asList("begin", "end")); - reserved.addAll(Arrays.asList("switch", "case")); - reserved.addAll(Arrays.asList("sqrt", "cbrt", "root")); - reserved.add("var"); - - final ChainTokenSplitter nsplit = new ChainTokenSplitter(); - - final ConfigurableTokenSplitter hi = new ConfigurableTokenSplitter(true); - final ConfigurableTokenSplitter lo = new ConfigurableTokenSplitter(true); - - hi.addSimpleDelimiters("->"); - hi.addSimpleDelimiters(":="); - hi.addSimpleDelimiters("||", "&&"); - hi.addSimpleDelimiters("<=", ">="); - - lo.addSimpleDelimiters("\u00B1"); // Unicode plus/minus - lo.addSimpleDelimiters(".", ",", ";", ":"); - lo.addSimpleDelimiters("=", "<", ">"); - lo.addSimpleDelimiters("+", "-", "*", "/"); - lo.addSimpleDelimiters("^"); - - lo.addMultiDelimiters("!"); - lo.addMultiDelimiters("(", ")"); - lo.addMultiDelimiters("[", "]"); - lo.addMultiDelimiters("{", "}"); - - hi.compile(); - lo.compile(); - - nsplit.appendSplitters(hi, lo); - - final ExcludingTokenSplitter excluder = new ExcludingTokenSplitter(nsplit); - - excluder.addLiteralExclusions(reserved.toArray(new String[0])); - - final PrattParser parser = createParser(); - - final TestContext ctx = new TestContext(); - - final Scanner scn = new Scanner(System.in); - - System.out.print("Enter a command (blank line to exit): "); - String ln = scn.nextLine(); - - while (!ln.trim().equals("")) { - final Iterator> tokens = preprocessInput(ops, excluder, ln, reserved, - ctx); - - try { - final StringTokenStream tokenStream = new StringTokenStream(tokens); - - /* - * Prime stream. - */ - tokenStream.next(); - - final ITree> tree = parser.parseExpression(0, tokenStream, ctx, - true); - - if (!tokenStream.headIs("(end)")) { - System.out.println("\nMultiple expressions on line"); - } - - System.out.println("\nParsed expression:\n" + tree); - } catch (final ParserException pex) { - pex.printStackTrace(); - } - - System.out.print("\nEnter a command (blank line to exit): "); - ln = scn.nextLine(); - } - - System.out.println(); - System.out.println("\nContext is: " + ctx); - - scn.close(); - } - - private static Iterator> preprocessInput(final Set ops, final TokenSplitter split, - final String ln, final Set reserved, final TestContext ctx) { - final String[] rawTokens = ln.split("\\s+"); - - final List splitTokens = new LinkedList<>(); - - for (final String raw : rawTokens) { - boolean doSplit = false; - - for (final String op : ops) { - if (raw.contains(op)) { - doSplit = true; - break; - } - } - - if (doSplit) { - final String[] strangs = split.split(raw).toArray(new String[0]); - - splitTokens.addAll(Arrays.asList(strangs)); - } else { - splitTokens.add(raw); - } - } - - System.out.println("\nSplit string: " + splitTokens); - - final Iterator source = splitTokens.iterator(); - - final Tokenizer tokenzer = new Tokenizer(ops, reserved, ctx); - - final Iterator> tokens = new TransformIterator<>(source, tokenzer); - - return tokens; - } - - private static PrattParser createParser() { - /* - * Set of which relational operators chain with each other. - */ - final HashSet relChain = new HashSet<>(); - relChain.addAll(Arrays.asList("=", "<", ">", "<=", ">=")); - - /* - * Token for marking chains. - */ - final StringToken chainToken = litToken("and"); - - /* - * ID function. - */ - final UnaryOperator idfun = id(); - - final PrattParser parser = new PrattParser<>(); - - parser.addNonInitialCommand("!!!", postfix(0)); - - parser.addNonInitialCommand(":", infixNon(3)); - - final NonInitialCommand ifElse = ternary(5, 0, "else", litToken("cond"), - false); - parser.addNonInitialCommand("if", ifElse); - - parser.addNonInitialCommand(":=", new AssignCommand(10)); - - parser.addNonInitialCommand("->", infixRight(11)); - - final NonInitialCommand nonSSRelJoin = infixLeft(13); - parser.addNonInitialCommand("and", nonSSRelJoin); - parser.addNonInitialCommand("or", nonSSRelJoin); - - final NonInitialCommand chainRelOp = chain(15, relChain, chainToken); - parser.addNonInitialCommand("=", chainRelOp); - parser.addNonInitialCommand("<", chainRelOp); - parser.addNonInitialCommand(">", chainRelOp); - parser.addNonInitialCommand("<=", chainRelOp); - parser.addNonInitialCommand(">=", chainRelOp); - - final NonInitialCommand ssRelJoin = infixRight(17); - parser.addNonInitialCommand("&&", ssRelJoin); - parser.addNonInitialCommand("||", ssRelJoin); - - final NonInitialCommand addSub = infixLeft(20); - parser.addNonInitialCommand("+", addSub); - parser.addNonInitialCommand("-", addSub); - parser.addNonInitialCommand("\u00B1", addSub); // Unicode plus/minus - - final NonInitialCommand mulDiv = infixLeft(30); - parser.addNonInitialCommand("*", mulDiv); - parser.addNonInitialCommand("/", mulDiv); - - parser.addNonInitialCommand("!", postfix(40)); - - final NonInitialCommand expon = infixRight(50); - parser.addNonInitialCommand("^", expon); - parser.addNonInitialCommand("root", expon); - - final NonInitialCommand superexpon = postfix(50); - parser.addNonInitialCommand("(superexp)", superexpon); - - parser.addNonInitialCommand(".", infixLeft(60)); - - final NonInitialCommand arrayIdx = postCircumfix(60, 0, "]", - litToken("idx")); - parser.addNonInitialCommand("[", arrayIdx); - - final InitialCommand ifThenElse = preTernary(0, 0, 0, "then", "else", - litToken("ifelse")); - parser.addInitialCommand("if", ifThenElse); - - final InitialCommand parens = grouping(0, ")", litToken("parens")); - parser.addInitialCommand("(", parens); - - final InitialCommand scoper = delimited(0, ";", "end", litToken("block"), - new BlockEnter(), idfun, new BlockExit(), true); - parser.addInitialCommand("begin", scoper); - - final InitialCommand arrayLiteral = delimited(0, ",", "]", - litToken("array"), idfun, idfun, idfun, false); - parser.addInitialCommand("[", arrayLiteral); - - final InitialCommand jsonLiteral = delimited(0, ",", "}", litToken("json"), - idfun, idfun, idfun, false); - parser.addInitialCommand("{", jsonLiteral); - - parser.addInitialCommand("case", unary(5)); - - parser.addInitialCommand("-", unary(30)); - - final InitialCommand root = unary(50); - parser.addInitialCommand("sqrt", root); - parser.addInitialCommand("cbrt", root); - - final InitialCommand leaf = leaf(); - parser.addInitialCommand("(literal)", leaf); - - parser.addInitialCommand("var", new VarCommand()); - - parser.addInitialCommand("switch", new SwitchCommand()); - - return parser; - } -} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/pratt/examples/SwitchCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/SwitchCommand.java deleted file mode 100644 index 6c4e2b6..0000000 --- a/JPratt/src/examples/java/bjc/pratt/examples/SwitchCommand.java +++ /dev/null @@ -1,21 +0,0 @@ -package bjc.pratt.examples; - -import bjc.pratt.InitialCommand; -import bjc.pratt.ParserContext; -import bjc.pratt.Token; -import bjc.pratt.tokens.StringToken; -import bjc.utils.data.ITree; -import bjc.utils.data.Tree; -import bjc.utils.parserutils.ParserException; - -class SwitchCommand implements InitialCommand { - @Override - public ITree> denote(final Token operator, - final ParserContext ctx) throws ParserException { - final ITree> object = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); - - final ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); - - return new Tree<>(new StringToken("switch", "switch"), object, body); - } -} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/TestContext.java b/JPratt/src/examples/java/bjc/pratt/examples/TestContext.java deleted file mode 100644 index e5583b4..0000000 --- a/JPratt/src/examples/java/bjc/pratt/examples/TestContext.java +++ /dev/null @@ -1,42 +0,0 @@ -package bjc.pratt.examples; - -import bjc.pratt.Token; -import bjc.utils.data.ITree; -import bjc.utils.esodata.Directory; -import bjc.utils.esodata.SimpleDirectory; -import bjc.utils.esodata.SimpleStack; -import bjc.utils.esodata.Stack; - -/** - * Simple context for the parser. - * - * @author EVE - * - */ -public class TestContext { - /** - * The variable scoping information. - */ - public Stack>>> scopes; - - /** - * The current number of scopes inside this scope. - */ - public Stack blockCount; - - /** - * Create a new test context. - */ - public TestContext() { - scopes = new SimpleStack<>(); - blockCount = new SimpleStack<>(); - - scopes.push(new SimpleDirectory<>()); - blockCount.push(0); - } - - @Override - public String toString() { - return String.format("TestContext [scopes=%s\n, blockCount=%s]", scopes, blockCount); - } -} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java b/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java deleted file mode 100644 index 4f8da12..0000000 --- a/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java +++ /dev/null @@ -1,38 +0,0 @@ -package bjc.pratt.examples; - -import static bjc.pratt.tokens.StringToken.litToken; - -import java.util.Set; -import java.util.function.Function; - -import bjc.pratt.Token; -import bjc.pratt.tokens.StringToken; - -final class Tokenizer implements Function> { - private final Set ops; - private final Set reserved; - - @SuppressWarnings("unused") - private final TestContext ctx; - - public Tokenizer(final Set operators, final Set reservedWords, final TestContext context) { - ops = operators; - reserved = reservedWords; - ctx = context; - } - - @Override - public Token apply(final String strang) { - if (ops.contains(strang) || reserved.contains(strang)) { - return litToken(strang); - } else if (strang.matches("(?:[\\u00B2\\u00B3\\u00B9\\u2070]|[\\u2074-\\u2079])+")) { - /* - * This regular expression matches series of unicode - * super - scripts 1 - 9. - */ - return new StringToken("(superexp)", strang); - } else { - return new StringToken("(literal)", strang); - } - } -} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/VarCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/VarCommand.java deleted file mode 100644 index a194b72..0000000 --- a/JPratt/src/examples/java/bjc/pratt/examples/VarCommand.java +++ /dev/null @@ -1,36 +0,0 @@ -package bjc.pratt.examples; - -import bjc.pratt.ParserContext; -import bjc.pratt.Token; -import bjc.pratt.commands.AbstractInitialCommand; -import bjc.pratt.tokens.StringToken; -import bjc.utils.data.ITree; -import bjc.utils.data.Tree; -import bjc.utils.parserutils.ParserException; - -class VarCommand extends AbstractInitialCommand { - - @Override - protected ITree> intNullDenotation(final Token operator, - final ParserContext ctx) throws ParserException { - final Token name = ctx.tokens.current(); - - switch (name.getKey()) { - case "(literal)": - case "(vref)": - ctx.tokens.next(); - break; - default: - throw new ParserException("Variable name must be simple"); - } - - ctx.tokens.expect("="); - - final ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); - - ctx.state.scopes.top().putKey(name.getValue(), body); - - return new Tree<>(new StringToken("var-bind", "var-bind"), new Tree<>(name), body); - } - -} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/AssignCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/AssignCommand.java new file mode 100644 index 0000000..5053f67 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/AssignCommand.java @@ -0,0 +1,36 @@ +package bjc.pratt.examples.lang; + +import bjc.pratt.ParserContext; +import bjc.pratt.Token; +import bjc.pratt.commands.NonBinaryCommand; +import bjc.pratt.tokens.StringToken; +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; + +class AssignCommand extends NonBinaryCommand { + public AssignCommand(final int prec) { + super(prec); + } + + @Override + public ITree> denote(final ITree> operand, + final Token operator, final ParserContext ctx) + throws ParserException { + final Token name = operand.getHead(); + + switch (name.getKey()) { + case "(literal)": + case "(vref)": + break; + default: + throw new ParserException("Variable name must be simple"); + } + + final ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + ctx.state.scopes.top().putKey(name.getValue(), body); + + return new Tree<>(new StringToken("assign", "assign"), operand, body); + } +} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/BlockEnter.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/BlockEnter.java new file mode 100644 index 0000000..41e0303 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/BlockEnter.java @@ -0,0 +1,25 @@ +package bjc.pratt.examples.lang; + +import java.util.function.UnaryOperator; + +import bjc.pratt.Token; +import bjc.utils.data.ITree; +import bjc.utils.esodata.Directory; +import bjc.utils.esodata.Stack; + +final class BlockEnter implements UnaryOperator { + @Override + public TestContext apply(final TestContext state) { + final Directory>> enclosing = state.scopes.top(); + final Stack blockCount = state.blockCount; + + final int currBlockNumber = blockCount.pop(); + + state.scopes.push(enclosing.newSubdirectory("block" + currBlockNumber)); + + blockCount.push(currBlockNumber + 1); + blockCount.push(0); + + return state; + } +} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/BlockExit.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/BlockExit.java new file mode 100644 index 0000000..15b2a5e --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/BlockExit.java @@ -0,0 +1,14 @@ +package bjc.pratt.examples.lang; + +import java.util.function.UnaryOperator; + +final class BlockExit implements UnaryOperator { + @Override + public TestContext apply(final TestContext state) { + state.scopes.pop(); + + state.blockCount.pop(); + + return state; + } +} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/PrattParserTest.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/PrattParserTest.java new file mode 100644 index 0000000..20450a1 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/PrattParserTest.java @@ -0,0 +1,303 @@ +package bjc.pratt.examples.lang; + +import static bjc.pratt.commands.InitialCommands.delimited; +import static bjc.pratt.commands.InitialCommands.grouping; +import static bjc.pratt.commands.InitialCommands.leaf; +import static bjc.pratt.commands.InitialCommands.preTernary; +import static bjc.pratt.commands.InitialCommands.unary; +import static bjc.pratt.commands.NonInitialCommands.chain; +import static bjc.pratt.commands.NonInitialCommands.infixLeft; +import static bjc.pratt.commands.NonInitialCommands.infixNon; +import static bjc.pratt.commands.NonInitialCommands.infixRight; +import static bjc.pratt.commands.NonInitialCommands.postCircumfix; +import static bjc.pratt.commands.NonInitialCommands.postfix; +import static bjc.pratt.commands.NonInitialCommands.ternary; +import static bjc.pratt.tokens.StringToken.litToken; +import static bjc.utils.functypes.ID.id; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Scanner; +import java.util.Set; +import java.util.function.UnaryOperator; + +import bjc.pratt.InitialCommand; +import bjc.pratt.NonInitialCommand; +import bjc.pratt.PrattParser; +import bjc.pratt.Token; +import bjc.pratt.tokens.StringToken; +import bjc.pratt.tokens.StringTokenStream; +import bjc.utils.data.ITree; +import bjc.utils.data.TransformIterator; +import bjc.utils.parserutils.ParserException; +import bjc.utils.parserutils.splitter.ChainTokenSplitter; +import bjc.utils.parserutils.splitter.ConfigurableTokenSplitter; +import bjc.utils.parserutils.splitter.ExcludingTokenSplitter; +import bjc.utils.parserutils.splitter.TokenSplitter; + +/** + * Simple test for Pratt parser. + * + * @author EVE + * + */ +public class PrattParserTest { + /** + * Main method. + * + * @param args + * Unused CLI arguments. + */ + public static void main(final String[] args) { + /* + * Use a linked hash set to preserve insertion order. + */ + final Set ops = new LinkedHashSet<>(); + + ops.add("!!!"); + + ops.addAll(Arrays.asList("->", "=>")); + ops.add(":="); + ops.addAll(Arrays.asList("||", "&&")); + ops.addAll(Arrays.asList("<=", ">=")); + + ops.addAll(Arrays.asList("\u00B1")); // Unicode plus/minus + ops.addAll(Arrays.asList(".", ",", ";", ":")); + ops.addAll(Arrays.asList("=", "<", ">")); + ops.addAll(Arrays.asList("+", "-", "*", "/")); + ops.addAll(Arrays.asList("^", "!")); + ops.addAll(Arrays.asList("(", ")")); + ops.addAll(Arrays.asList("[", "]")); + ops.addAll(Arrays.asList("{", "}")); + + /* + * Reserved words that represent themselves, not literals. + */ + final Set reserved = new LinkedHashSet<>(); + reserved.addAll(Arrays.asList("if", "then", "else")); + reserved.addAll(Arrays.asList("and", "or")); + reserved.addAll(Arrays.asList("begin", "end")); + reserved.addAll(Arrays.asList("switch", "case")); + reserved.addAll(Arrays.asList("sqrt", "cbrt", "root")); + reserved.add("var"); + + final ChainTokenSplitter nsplit = new ChainTokenSplitter(); + + final ConfigurableTokenSplitter hi = new ConfigurableTokenSplitter(true); + final ConfigurableTokenSplitter lo = new ConfigurableTokenSplitter(true); + + hi.addSimpleDelimiters("->"); + hi.addSimpleDelimiters(":="); + hi.addSimpleDelimiters("||", "&&"); + hi.addSimpleDelimiters("<=", ">="); + + lo.addSimpleDelimiters("\u00B1"); // Unicode plus/minus + lo.addSimpleDelimiters(".", ",", ";", ":"); + lo.addSimpleDelimiters("=", "<", ">"); + lo.addSimpleDelimiters("+", "-", "*", "/"); + lo.addSimpleDelimiters("^"); + + lo.addMultiDelimiters("!"); + lo.addMultiDelimiters("(", ")"); + lo.addMultiDelimiters("[", "]"); + lo.addMultiDelimiters("{", "}"); + + hi.compile(); + lo.compile(); + + nsplit.appendSplitters(hi, lo); + + final ExcludingTokenSplitter excluder = new ExcludingTokenSplitter(nsplit); + + excluder.addLiteralExclusions(reserved.toArray(new String[0])); + + final PrattParser parser = createParser(); + + final TestContext ctx = new TestContext(); + + final Scanner scn = new Scanner(System.in); + + System.out.print("Enter a command (blank line to exit): "); + String ln = scn.nextLine(); + + while (!ln.trim().equals("")) { + final Iterator> tokens = preprocessInput(ops, excluder, ln, reserved, + ctx); + + try { + final StringTokenStream tokenStream = new StringTokenStream(tokens); + + /* + * Prime stream. + */ + tokenStream.next(); + + final ITree> tree = parser.parseExpression(0, tokenStream, ctx, + true); + + if (!tokenStream.headIs("(end)")) { + System.out.println("\nMultiple expressions on line"); + } + + System.out.println("\nParsed expression:\n" + tree); + } catch (final ParserException pex) { + pex.printStackTrace(); + } + + System.out.print("\nEnter a command (blank line to exit): "); + ln = scn.nextLine(); + } + + System.out.println(); + System.out.println("\nContext is: " + ctx); + + scn.close(); + } + + private static Iterator> preprocessInput(final Set ops, final TokenSplitter split, + final String ln, final Set reserved, final TestContext ctx) { + final String[] rawTokens = ln.split("\\s+"); + + final List splitTokens = new LinkedList<>(); + + for (final String raw : rawTokens) { + boolean doSplit = false; + + for (final String op : ops) { + if (raw.contains(op)) { + doSplit = true; + break; + } + } + + if (doSplit) { + final String[] strangs = split.split(raw).toArray(new String[0]); + + splitTokens.addAll(Arrays.asList(strangs)); + } else { + splitTokens.add(raw); + } + } + + System.out.println("\nSplit string: " + splitTokens); + + final Iterator source = splitTokens.iterator(); + + final Tokenizer tokenzer = new Tokenizer(ops, reserved, ctx); + + final Iterator> tokens = new TransformIterator<>(source, tokenzer); + + return tokens; + } + + private static PrattParser createParser() { + /* + * Set of which relational operators chain with each other. + */ + final HashSet relChain = new HashSet<>(); + relChain.addAll(Arrays.asList("=", "<", ">", "<=", ">=")); + + /* + * Token for marking chains. + */ + final StringToken chainToken = litToken("and"); + + /* + * ID function. + */ + final UnaryOperator idfun = id(); + + final PrattParser parser = new PrattParser<>(); + + parser.addNonInitialCommand("!!!", postfix(0)); + + parser.addNonInitialCommand(":", infixNon(3)); + + final NonInitialCommand ifElse = ternary(5, 0, "else", litToken("cond"), + false); + parser.addNonInitialCommand("if", ifElse); + + parser.addNonInitialCommand(":=", new AssignCommand(10)); + + parser.addNonInitialCommand("->", infixRight(11)); + + final NonInitialCommand nonSSRelJoin = infixLeft(13); + parser.addNonInitialCommand("and", nonSSRelJoin); + parser.addNonInitialCommand("or", nonSSRelJoin); + + final NonInitialCommand chainRelOp = chain(15, relChain, chainToken); + parser.addNonInitialCommand("=", chainRelOp); + parser.addNonInitialCommand("<", chainRelOp); + parser.addNonInitialCommand(">", chainRelOp); + parser.addNonInitialCommand("<=", chainRelOp); + parser.addNonInitialCommand(">=", chainRelOp); + + final NonInitialCommand ssRelJoin = infixRight(17); + parser.addNonInitialCommand("&&", ssRelJoin); + parser.addNonInitialCommand("||", ssRelJoin); + + final NonInitialCommand addSub = infixLeft(20); + parser.addNonInitialCommand("+", addSub); + parser.addNonInitialCommand("-", addSub); + parser.addNonInitialCommand("\u00B1", addSub); // Unicode plus/minus + + final NonInitialCommand mulDiv = infixLeft(30); + parser.addNonInitialCommand("*", mulDiv); + parser.addNonInitialCommand("/", mulDiv); + + parser.addNonInitialCommand("!", postfix(40)); + + final NonInitialCommand expon = infixRight(50); + parser.addNonInitialCommand("^", expon); + parser.addNonInitialCommand("root", expon); + + final NonInitialCommand superexpon = postfix(50); + parser.addNonInitialCommand("(superexp)", superexpon); + + parser.addNonInitialCommand(".", infixLeft(60)); + + final NonInitialCommand arrayIdx = postCircumfix(60, 0, "]", + litToken("idx")); + parser.addNonInitialCommand("[", arrayIdx); + + final InitialCommand ifThenElse = preTernary(0, 0, 0, "then", "else", + litToken("ifelse")); + parser.addInitialCommand("if", ifThenElse); + + final InitialCommand parens = grouping(0, ")", litToken("parens")); + parser.addInitialCommand("(", parens); + + final InitialCommand scoper = delimited(0, ";", "end", litToken("block"), + new BlockEnter(), idfun, new BlockExit(), true); + parser.addInitialCommand("begin", scoper); + + final InitialCommand arrayLiteral = delimited(0, ",", "]", + litToken("array"), idfun, idfun, idfun, false); + parser.addInitialCommand("[", arrayLiteral); + + final InitialCommand jsonLiteral = delimited(0, ",", "}", litToken("json"), + idfun, idfun, idfun, false); + parser.addInitialCommand("{", jsonLiteral); + + parser.addInitialCommand("case", unary(5)); + + parser.addInitialCommand("-", unary(30)); + + final InitialCommand root = unary(50); + parser.addInitialCommand("sqrt", root); + parser.addInitialCommand("cbrt", root); + + final InitialCommand leaf = leaf(); + parser.addInitialCommand("(literal)", leaf); + + parser.addInitialCommand("var", new VarCommand()); + + parser.addInitialCommand("switch", new SwitchCommand()); + + return parser; + } +} \ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/SwitchCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/SwitchCommand.java new file mode 100644 index 0000000..f5fc9b1 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/SwitchCommand.java @@ -0,0 +1,21 @@ +package bjc.pratt.examples.lang; + +import bjc.pratt.InitialCommand; +import bjc.pratt.ParserContext; +import bjc.pratt.Token; +import bjc.pratt.tokens.StringToken; +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; + +class SwitchCommand implements InitialCommand { + @Override + public ITree> denote(final Token operator, + final ParserContext ctx) throws ParserException { + final ITree> object = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + final ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + return new Tree<>(new StringToken("switch", "switch"), object, body); + } +} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/TestContext.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/TestContext.java new file mode 100644 index 0000000..659ba36 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/TestContext.java @@ -0,0 +1,42 @@ +package bjc.pratt.examples.lang; + +import bjc.pratt.Token; +import bjc.utils.data.ITree; +import bjc.utils.esodata.Directory; +import bjc.utils.esodata.SimpleDirectory; +import bjc.utils.esodata.SimpleStack; +import bjc.utils.esodata.Stack; + +/** + * Simple context for the parser. + * + * @author EVE + * + */ +public class TestContext { + /** + * The variable scoping information. + */ + public Stack>>> scopes; + + /** + * The current number of scopes inside this scope. + */ + public Stack blockCount; + + /** + * Create a new test context. + */ + public TestContext() { + scopes = new SimpleStack<>(); + blockCount = new SimpleStack<>(); + + scopes.push(new SimpleDirectory<>()); + blockCount.push(0); + } + + @Override + public String toString() { + return String.format("TestContext [scopes=%s\n, blockCount=%s]", scopes, blockCount); + } +} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/Tokenizer.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/Tokenizer.java new file mode 100644 index 0000000..ea162be --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/Tokenizer.java @@ -0,0 +1,46 @@ +package bjc.pratt.examples.lang; + +import static bjc.pratt.tokens.StringToken.litToken; + +import java.util.Set; +import java.util.function.Function; + +import bjc.pratt.Token; +import bjc.pratt.tokens.StringToken; + +final class Tokenizer implements Function> { + private final Set ops; + private final Set reserved; + + @SuppressWarnings("unused") + private final TestContext ctx; + + public Tokenizer(final Set operators, final Set reservedWords, final TestContext context) { + ops = operators; + reserved = reservedWords; + ctx = context; + } + + @Override + public Token apply(final String strang) { + if (ops.contains(strang) || reserved.contains(strang)) + return litToken(strang); + else if (strang.matches("(?:[\\u00B2\\u00B3\\u00B9\\u2070]|[\\u2074-\\u2079])+")) /* + * This + * regular + * expression + * matches + * series + * of + * unicode + * super + * - + * scripts + * 1 + * - + * 9. + */ + return new StringToken("(superexp)", strang); + else return new StringToken("(literal)", strang); + } +} diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/VarCommand.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/VarCommand.java new file mode 100644 index 0000000..a2f428b --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/VarCommand.java @@ -0,0 +1,36 @@ +package bjc.pratt.examples.lang; + +import bjc.pratt.ParserContext; +import bjc.pratt.Token; +import bjc.pratt.commands.AbstractInitialCommand; +import bjc.pratt.tokens.StringToken; +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; + +class VarCommand extends AbstractInitialCommand { + + @Override + protected ITree> intNullDenotation(final Token operator, + final ParserContext ctx) throws ParserException { + final Token name = ctx.tokens.current(); + + switch (name.getKey()) { + case "(literal)": + case "(vref)": + ctx.tokens.next(); + break; + default: + throw new ParserException("Variable name must be simple"); + } + + ctx.tokens.expect("="); + + final ITree> body = ctx.parse.parseExpression(0, ctx.tokens, ctx.state, false); + + ctx.state.scopes.top().putKey(name.getValue(), body); + + return new Tree<>(new StringToken("var-bind", "var-bind"), new Tree<>(name), body); + } + +} -- cgit v1.2.3