diff options
Diffstat (limited to 'JPratt/src')
4 files changed, 106 insertions, 96 deletions
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/PrattParserTest.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/PrattParserTest.java index 20450a1..3686a84 100644 --- a/JPratt/src/examples/java/bjc/pratt/examples/lang/PrattParserTest.java +++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/PrattParserTest.java @@ -33,10 +33,12 @@ import bjc.pratt.tokens.StringToken; import bjc.pratt.tokens.StringTokenStream; import bjc.utils.data.ITree; import bjc.utils.data.TransformIterator; +import bjc.utils.funcdata.IList; import bjc.utils.parserutils.ParserException; import bjc.utils.parserutils.splitter.ChainTokenSplitter; import bjc.utils.parserutils.splitter.ConfigurableTokenSplitter; import bjc.utils.parserutils.splitter.ExcludingTokenSplitter; +import bjc.utils.parserutils.splitter.FilteredTokenSplitter; import bjc.utils.parserutils.splitter.TokenSplitter; /** @@ -83,6 +85,7 @@ public class PrattParserTest { reserved.addAll(Arrays.asList("begin", "end")); reserved.addAll(Arrays.asList("switch", "case")); reserved.addAll(Arrays.asList("sqrt", "cbrt", "root")); + reserved.addAll(Arrays.asList("try", "catch", "finally")); reserved.add("var"); final ChainTokenSplitter nsplit = new ChainTokenSplitter(); @@ -115,6 +118,8 @@ public class PrattParserTest { excluder.addLiteralExclusions(reserved.toArray(new String[0])); + final FilteredTokenSplitter filtered = new FilteredTokenSplitter(excluder, (tok) -> !tok.equals("")); + final PrattParser<String, String, TestContext> parser = createParser(); final TestContext ctx = new TestContext(); @@ -125,7 +130,7 @@ public class PrattParserTest { String ln = scn.nextLine(); while (!ln.trim().equals("")) { - final Iterator<Token<String, String>> tokens = preprocessInput(ops, excluder, ln, reserved, + final Iterator<Token<String, String>> tokens = preprocessInput(ops, filtered, ln, reserved, ctx); try { @@ -165,6 +170,8 @@ public class PrattParserTest { final List<String> splitTokens = new LinkedList<>(); for (final String raw : rawTokens) { + if (raw.equals("")) continue; + boolean doSplit = false; for (final String op : ops) { @@ -175,9 +182,10 @@ public class PrattParserTest { } if (doSplit) { - final String[] strangs = split.split(raw).toArray(new String[0]); + IList<String> splitStrangs = split.split(raw); + splitStrangs.removeMatching(""); - splitTokens.addAll(Arrays.asList(strangs)); + splitStrangs.forEach(splitTokens::add); } else { splitTokens.add(raw); } @@ -217,7 +225,11 @@ public class PrattParserTest { parser.addNonInitialCommand(":", infixNon(3)); - final NonInitialCommand<String, String, TestContext> ifElse = ternary(5, 0, "else", litToken("cond"), + parser.addNonInitialCommand("finally", infixLeft(4)); + + parser.addNonInitialCommand("catch", infixLeft(5)); + + final NonInitialCommand<String, String, TestContext> ifElse = ternary(6, 0, "else", litToken("cond"), false); parser.addNonInitialCommand("if", ifElse); @@ -283,6 +295,8 @@ public class PrattParserTest { idfun, idfun, idfun, false); parser.addInitialCommand("{", jsonLiteral); + parser.addInitialCommand("try", unary(3)); + parser.addInitialCommand("case", unary(5)); parser.addInitialCommand("-", unary(30)); diff --git a/JPratt/src/examples/java/bjc/pratt/examples/lang/Tokenizer.java b/JPratt/src/examples/java/bjc/pratt/examples/lang/Tokenizer.java index ea162be..800a4ec 100644 --- a/JPratt/src/examples/java/bjc/pratt/examples/lang/Tokenizer.java +++ b/JPratt/src/examples/java/bjc/pratt/examples/lang/Tokenizer.java @@ -25,22 +25,14 @@ final class Tokenizer implements Function<String, Token<String, String>> { public Token<String, String> apply(final String strang) { if (ops.contains(strang) || reserved.contains(strang)) return litToken(strang); - else if (strang.matches("(?:[\\u00B2\\u00B3\\u00B9\\u2070]|[\\u2074-\\u2079])+")) /* - * This - * regular - * expression - * matches - * series - * of - * unicode - * super - * - - * scripts - * 1 - * - - * 9. - */ - return new StringToken("(superexp)", strang); - else return new StringToken("(literal)", strang); + else if (strang.matches("(?:[\\u00B2\\u00B3\\u00B9\\u2070]|[\\u2074-\\u2079])+")) { + /* + * This regular expression matches series of unicode + * super - scripts 1 - 9. + */ + String subscript = strang; + + return new StringToken("(superexp)", subscript); + } else return new StringToken("(literal)", strang); } } diff --git a/JPratt/src/main/java/bjc/pratt/blocks/ChainParseBlock.java b/JPratt/src/main/java/bjc/pratt/blocks/ChainParseBlock.java new file mode 100644 index 0000000..1758c17 --- /dev/null +++ b/JPratt/src/main/java/bjc/pratt/blocks/ChainParseBlock.java @@ -0,0 +1,79 @@ +package bjc.pratt.blocks; + +import java.util.Set; + +import bjc.pratt.ParseBlock; +import bjc.pratt.ParserContext; +import bjc.pratt.Token; +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.parserutils.ParserException; + +/** + * A {@link ParseBlock} for a series of parse blocks, linked by a set of tokens. + * + * Roughly analogous to Perl 6s list associative operators. + * + * @author bjculkin + * + * @param <K> + * The token key type. + * + * @param <V> + * The token value type. + * + * @param <C> + * The parser state type. + * + */ +public class ChainParseBlock<K, V, C> implements ParseBlock<K, V, C> { + private ParseBlock<K, V, C> iner; + + private Set<K> indicators; + + private Token<K, V> trm; + + /** + * Create a new chain parser block. + * + * @param inner + * The block for the chains interior. + * + * @param chainIndicators + * The set of markers that indicate continuing the chain + * + * @param term + * The node in the AST for the expression. + */ + public ChainParseBlock(ParseBlock<K, V, C> inner, Set<K> chainIndicators, Token<K, V> term) { + iner = inner; + indicators = chainIndicators; + trm = term; + } + + @Override + public ITree<Token<K, V>> parse(ParserContext<K, V, C> ctx) throws ParserException { + ITree<Token<K, V>> expression = iner.parse(ctx); + + Token<K, V> currentToken = ctx.tokens.current(); + if (indicators.contains(currentToken.getKey())) { + ITree<Token<K, V>> res = new Tree<>(trm); + res.addChild(expression); + + while (indicators.contains(currentToken.getKey())) { + res.addChild(new Tree<>(currentToken)); + ctx.tokens.next(); + + ITree<Token<K, V>> innerExpression = iner.parse(ctx); + res.addChild(innerExpression); + + currentToken = ctx.tokens.current(); + } + + return res; + } + + return expression; + } + +} diff --git a/JPratt/src/main/java/bjc/pratt/commands/BlockNonInitialCommand.java b/JPratt/src/main/java/bjc/pratt/commands/BlockNonInitialCommand.java deleted file mode 100644 index c361c73..0000000 --- a/JPratt/src/main/java/bjc/pratt/commands/BlockNonInitialCommand.java +++ /dev/null @@ -1,75 +0,0 @@ -package bjc.pratt.commands; - -import bjc.pratt.NonInitialCommand; -import bjc.pratt.ParseBlock; -import bjc.pratt.ParserContext; -import bjc.pratt.Token; -import bjc.utils.data.ITree; -import bjc.utils.data.Tree; -import bjc.utils.parserutils.ParserException; - -/** - * A non-initial command that delegates all of the work to a {@link ParseBlock} - * - * @author bjculkin - * - * @param <K> - * The token key type. - * - * @param <V> - * The token value type. - * - * @param <C> - * The parser state type. - */ -public class BlockNonInitialCommand<K, V, C> extends NonInitialCommand<K, V, C> { - private final ParseBlock<K, V, C> innr; - - private final int lftBind; - private final int nxtBind; - - private final Token<K, V> trm; - - /** - * Create a new non-initial command that delegates to a parse block. - * - * @param inner - * The parse block to delegate to. - * - * @param leftBind - * The left binding power (precedence). - * - * @param rightBind - * The right binding power (associativity control). - * - * @param term - * The token to use as the node in the AST. - */ - public BlockNonInitialCommand(final ParseBlock<K, V, C> inner, final int leftBind, final int rightBind, - final Token<K, V> term) { - innr = inner; - - lftBind = leftBind; - nxtBind = rightBind; - - trm = term; - } - - @Override - public ITree<Token<K, V>> denote(final ITree<Token<K, V>> operand, final Token<K, V> operator, - final ParserContext<K, V, C> ctx) throws ParserException { - final ITree<Token<K, V>> expression = innr.parse(ctx); - - return new Tree<>(trm, expression); - } - - @Override - public int leftBinding() { - return lftBind; - } - - @Override - public int nextBinding() { - return nxtBind; - } -}
\ No newline at end of file |