diff options
Diffstat (limited to 'JPratt/src/examples')
4 files changed, 219 insertions, 125 deletions
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/InputState.java b/JPratt/src/examples/java/bjc/pratt/examples/InputState.java new file mode 100644 index 0000000..7fe74fb --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/InputState.java @@ -0,0 +1,150 @@ +package bjc.pratt.examples; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; + +import bjc.pratt.Token; +import bjc.utils.data.TransformIterator; +import bjc.utils.parserutils.splitterv2.ChainTokenSplitter; +import bjc.utils.parserutils.splitterv2.ConfigurableTokenSplitter; +import bjc.utils.parserutils.splitterv2.ExcludingTokenSplitter; +import bjc.utils.parserutils.splitterv2.TokenSplitter; + +/** + * Packaged input state for the parser. + * + * @author student + * + */ +public class InputState { + public Set<String> ops; + public Set<String> reserved; + + public TokenSplitter splitter; + + public TestContext ctx; + + public String ln; + + private InputState(Set<String> opps, TokenSplitter splt, String lin, Set<String> rserved, TestContext ctxt) { + ops = opps; + reserved = rserved; + + splitter = splt; + + ctx = ctxt; + + ln = lin; + } + + public static InputState createState() { + /* + * Use a linked hash set to preserve insertion order. + */ + Set<String> ops = new LinkedHashSet<>(); + + ops.add("!!!"); + + ops.addAll(Arrays.asList("->", "=>")); + ops.add(":="); + ops.addAll(Arrays.asList("||", "&&")); + ops.addAll(Arrays.asList("<=", ">=")); + + ops.addAll(Arrays.asList("±")); + ops.addAll(Arrays.asList(".", ",", ";", ":")); + ops.addAll(Arrays.asList("=", "<", ">")); + ops.addAll(Arrays.asList("+", "-", "*", "/")); + ops.addAll(Arrays.asList("^", "!")); + ops.addAll(Arrays.asList("(", ")")); + ops.addAll(Arrays.asList("[", "]")); + ops.addAll(Arrays.asList("{", "}")); + + /* + * Reserved words that represent themselves, not literals. + */ + Set<String> reserved = new LinkedHashSet<>(); + + reserved.addAll(Arrays.asList("if", "then", "else")); + reserved.addAll(Arrays.asList("and", "or")); + reserved.addAll(Arrays.asList("begin", "end")); + reserved.addAll(Arrays.asList("switch", "case")); + reserved.addAll(Arrays.asList("sqrt", "cbrt", "root")); + + reserved.add("var"); + + ChainTokenSplitter nsplit = new ChainTokenSplitter(); + + ConfigurableTokenSplitter hi = new ConfigurableTokenSplitter(true); + ConfigurableTokenSplitter lo = new ConfigurableTokenSplitter(true); + + hi.addSimpleDelimiters("->"); + hi.addSimpleDelimiters(":="); + hi.addSimpleDelimiters("||", "&&"); + hi.addSimpleDelimiters("<=", ">="); + + lo.addSimpleDelimiters("±"); + lo.addSimpleDelimiters(".", ",", ";", ":"); + lo.addSimpleDelimiters("=", "<", ">"); + lo.addSimpleDelimiters("+", "-", "*", "/"); + lo.addSimpleDelimiters("^"); + + lo.addMultiDelimiters("!"); + lo.addMultiDelimiters("(", ")"); + lo.addMultiDelimiters("[", "]"); + lo.addMultiDelimiters("{", "}"); + + hi.compile(); + lo.compile(); + + nsplit.appendSplitters(hi, lo); + + ExcludingTokenSplitter excluder = new ExcludingTokenSplitter(nsplit); + + excluder.addLiteralExclusions(reserved.toArray(new String[0])); + + TestContext ctx = new TestContext(); + + InputState state = new InputState(ops, excluder, null, reserved, ctx); + + return state; + } + + public Iterator<Token<String, String>> preprocessInput() { + String[] rawTokens = ln.split("\\s+"); + + List<String> splitTokens = new LinkedList<>(); + + for (String raw : rawTokens) { + boolean doSplit = false; + + for (String op : ops) { + if (raw.contains(op)) { + doSplit = true; + break; + } + } + + if (doSplit) { + String[] strangs = splitter.split(raw).toArray(new String[0]); + + splitTokens.addAll(Arrays.asList(strangs)); + } else { + splitTokens.add(raw); + } + } + + System.out.println("\nSplit string: " + splitTokens); + + Iterator<String> source = splitTokens.iterator(); + + Tokenizer tokenzer = new Tokenizer(ops, reserved, ctx); + + Iterator<Token<String, String>> tokens = new TransformIterator<>(source, tokenzer); + + return tokens; + } +}
\ No newline at end of file diff --git a/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java b/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java index b7d31eb..7b9795e 100644 --- a/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java +++ b/JPratt/src/examples/java/bjc/pratt/examples/PrattParserTest.java @@ -7,21 +7,12 @@ import bjc.pratt.Token; import bjc.pratt.tokens.StringToken; import bjc.pratt.tokens.StringTokenStream; import bjc.utils.data.ITree; -import bjc.utils.data.TransformIterator; import bjc.utils.parserutils.ParserException; -import bjc.utils.parserutils.splitterv2.ChainTokenSplitter; -import bjc.utils.parserutils.splitterv2.ConfigurableTokenSplitter; -import bjc.utils.parserutils.splitterv2.ExcludingTokenSplitter; -import bjc.utils.parserutils.splitterv2.TokenSplitter; import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; import java.util.Scanner; -import java.util.Set; import java.util.function.UnaryOperator; import static bjc.pratt.commands.InitialCommands.*; @@ -40,82 +31,22 @@ public class PrattParserTest { * Main method. * * @param args - * Unused CLI arguments. + * Unused CLI arguments. */ public static void main(String[] args) { - /* - * Use a linked hash set to preserve insertion order. - */ - Set<String> ops = new LinkedHashSet<>(); - - ops.add("!!!"); - - ops.addAll(Arrays.asList("->", "=>")); - ops.add(":="); - ops.addAll(Arrays.asList("||", "&&")); - ops.addAll(Arrays.asList("<=", ">=")); - - ops.addAll(Arrays.asList("±")); - ops.addAll(Arrays.asList(".", ",", ";", ":")); - ops.addAll(Arrays.asList("=", "<", ">")); - ops.addAll(Arrays.asList("+", "-", "*", "/")); - ops.addAll(Arrays.asList("^", "!")); - ops.addAll(Arrays.asList("(", ")")); - ops.addAll(Arrays.asList("[", "]")); - ops.addAll(Arrays.asList("{", "}")); - - /* - * Reserved words that represent themselves, not literals. - */ - Set<String> reserved = new LinkedHashSet<>(); - reserved.addAll(Arrays.asList("if", "then", "else")); - reserved.addAll(Arrays.asList("and", "or")); - reserved.addAll(Arrays.asList("begin", "end")); - reserved.addAll(Arrays.asList("switch", "case")); - reserved.addAll(Arrays.asList("sqrt", "cbrt", "root")); - reserved.add("var"); - - ChainTokenSplitter nsplit = new ChainTokenSplitter(); - - ConfigurableTokenSplitter hi = new ConfigurableTokenSplitter(true); - ConfigurableTokenSplitter lo = new ConfigurableTokenSplitter(true); - - hi.addSimpleDelimiters("->"); - hi.addSimpleDelimiters(":="); - hi.addSimpleDelimiters("||", "&&"); - hi.addSimpleDelimiters("<=", ">="); - - lo.addSimpleDelimiters("±"); - lo.addSimpleDelimiters(".", ",", ";", ":"); - lo.addSimpleDelimiters("=", "<", ">"); - lo.addSimpleDelimiters("+", "-", "*", "/"); - lo.addSimpleDelimiters("^"); - - lo.addMultiDelimiters("!"); - lo.addMultiDelimiters("(", ")"); - lo.addMultiDelimiters("[", "]"); - lo.addMultiDelimiters("{", "}"); - - hi.compile(); - lo.compile(); - - nsplit.appendSplitters(hi, lo); - - ExcludingTokenSplitter excluder = new ExcludingTokenSplitter(nsplit); - - excluder.addLiteralExclusions(reserved.toArray(new String[0])); + InputState state = InputState.createState(); PrattParser<String, String, TestContext> parser = createParser(); - TestContext ctx = new TestContext(); - Scanner scn = new Scanner(System.in); System.out.print("Enter a command (blank line to exit): "); String ln = scn.nextLine(); while (!ln.trim().equals("")) { - Iterator<Token<String, String>> tokens = preprocessInput(ops, excluder, ln, reserved, ctx); + state.ln = ln; + + Iterator<Token<String, String>> tokens = state.preprocessInput(); try { StringTokenStream tokenStream = new StringTokenStream(tokens); @@ -125,7 +56,7 @@ public class PrattParserTest { */ tokenStream.next(); - ITree<Token<String, String>> tree = parser.parseExpression(0, tokenStream, ctx, true); + ITree<Token<String, String>> tree = parser.parseExpression(0, tokenStream, state.ctx, true); if (!tokenStream.headIs("(end)")) { System.out.println("\nMultiple expressions on line"); @@ -141,47 +72,11 @@ public class PrattParserTest { } System.out.println(); - System.out.println("\nContext is: " + ctx); + System.out.println("\nContext is: " + state.ctx); scn.close(); } - private static Iterator<Token<String, String>> preprocessInput(Set<String> ops, TokenSplitter split, String ln, - Set<String> reserved, TestContext ctx) { - String[] rawTokens = ln.split("\\s+"); - - List<String> splitTokens = new LinkedList<>(); - - for (String raw : rawTokens) { - boolean doSplit = false; - - for (String op : ops) { - if (raw.contains(op)) { - doSplit = true; - break; - } - } - - if (doSplit) { - String[] strangs = split.split(raw).toArray(new String[0]); - - splitTokens.addAll(Arrays.asList(strangs)); - } else { - splitTokens.add(raw); - } - } - - System.out.println("\nSplit string: " + splitTokens); - - Iterator<String> source = splitTokens.iterator(); - - Tokenizer tokenzer = new Tokenizer(ops, reserved, ctx); - - Iterator<Token<String, String>> tokens = new TransformIterator<>(source, tokenzer); - - return tokens; - } - private static PrattParser<String, String, TestContext> createParser() { /* * Set of which relational operators chain with each other. @@ -230,7 +125,7 @@ public class PrattParserTest { NonInitialCommand<String, String, TestContext> addSub = infixLeft(20); parser.addNonInitialCommand("+", addSub); parser.addNonInitialCommand("-", addSub); - parser.addNonInitialCommand("±", addSub); + parser.addNonInitialCommand("±", addSub); NonInitialCommand<String, String, TestContext> mulDiv = infixLeft(30); parser.addNonInitialCommand("*", mulDiv); @@ -261,12 +156,12 @@ public class PrattParserTest { new BlockEnter(), idfun, new BlockExit(), true); parser.addInitialCommand("begin", scoper); - InitialCommand<String, String, TestContext> arrayLiteral = delimited(0, ",", "]", litToken("array"), - idfun, idfun, idfun, false); + InitialCommand<String, String, TestContext> arrayLiteral = delimited(0, ",", "]", litToken("array"), idfun, + idfun, idfun, false); parser.addInitialCommand("[", arrayLiteral); - InitialCommand<String, String, TestContext> jsonLiteral = delimited(0, ",", "}", litToken("json"), - idfun, idfun, idfun, false); + InitialCommand<String, String, TestContext> jsonLiteral = delimited(0, ",", "}", litToken("json"), idfun, idfun, + idfun, false); parser.addInitialCommand("{", jsonLiteral); parser.addInitialCommand("case", unary(5)); @@ -279,7 +174,6 @@ public class PrattParserTest { InitialCommand<String, String, TestContext> leaf = leaf(); parser.addInitialCommand("(literal)", leaf); - parser.addInitialCommand("(vref)", leaf); parser.addInitialCommand("var", new VarCommand()); diff --git a/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java b/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java index 0fccc60..cfaf3e2 100644 --- a/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java +++ b/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java @@ -9,9 +9,11 @@ import java.util.function.Function; import static bjc.pratt.tokens.StringToken.litToken; final class Tokenizer implements Function<String, Token<String, String>> { - private Set<String> ops; - private Set<String> reserved; - private TestContext ctx; + private Set<String> ops; + private Set<String> reserved; + + @SuppressWarnings("unused") + private TestContext ctx; public Tokenizer(Set<String> operators, Set<String> reservedWords, TestContext context) { ops = operators; @@ -23,11 +25,10 @@ final class Tokenizer implements Function<String, Token<String, String>> { public Token<String, String> apply(String strang) { if (ops.contains(strang) || reserved.contains(strang)) { return litToken(strang); - } else if (ctx.scopes.top().containsKey(strang)) { - return new StringToken("(vref)", strang); - } else if(strang.matches("(?:[\\u00B2\\u00B3\\u00B9\\u2070]|[\\u2074-\\u2079])+")) { + } else if (strang.matches("(?:[\\u00B2\\u00B3\\u00B9\\u2070]|[\\u2074-\\u2079])+")) { /* - * This regular expression matches series of unicode super-scripts 1-9. + * This regular expression matches series of unicode super-scripts + * 1-9. */ return new StringToken("(superexp)", strang); } else { diff --git a/JPratt/src/examples/java/bjc/pratt/examples/TreeEvaluator.java b/JPratt/src/examples/java/bjc/pratt/examples/TreeEvaluator.java new file mode 100644 index 0000000..e4325b2 --- /dev/null +++ b/JPratt/src/examples/java/bjc/pratt/examples/TreeEvaluator.java @@ -0,0 +1,49 @@ +package bjc.pratt.examples; + +import bjc.utils.data.ITree; +import bjc.utils.data.TopDownTransformResult; + +/** + * Evaluate a tree to a result. + * + * @author student + * + */ +public class TreeEvaluator { + /** + * The result of evaluating a tree. + * + * @author student + * + */ + public static final class EvaluationResult { + public static EvaluationResult fromToken(EvaluationToken tok) { + return null; + } + } + + /** + * The token type for trees to evaluate. + * + * @author student + * + */ + public static final class EvaluationToken { + + } + + public static EvaluationResult evaluateTree(ITree<EvaluationToken> tree, TestContext ctx) { + ITree<EvaluationToken> result = tree.topDownTransform(TreeEvaluator::pickNodeEvaluation, + TreeEvaluator::evaluateNode); + + return EvaluationResult.fromToken(result.getHead()); + } + + private static TopDownTransformResult pickNodeEvaluation(EvaluationToken node) { + return TopDownTransformResult.PUSHDOWN; + } + + private static ITree<EvaluationToken> evaluateNode(ITree<EvaluationToken> tree) { + return tree; + } +} |