From d766896972c9e9be4a9e0021ec5f4f0665901865 Mon Sep 17 00:00:00 2001 From: "Benjamin J. Culkin" Date: Sat, 9 Sep 2017 21:46:16 -0300 Subject: Update Most of it is documentation changes. The rest is more work on BlockReaders, as well as a simple command language for configuring them. --- .../java/bjc/utils/parserutils/DoubleMatcher.java | 8 ++- .../java/bjc/utils/parserutils/ShuntingYard.java | 84 ++++++++++++++++------ .../bjc/utils/parserutils/TokenTransformer.java | 59 ++++++++++----- .../java/bjc/utils/parserutils/TokenUtils.java | 17 ++++- .../bjc/utils/parserutils/TreeConstructor.java | 22 ++++-- .../utils/parserutils/delims/DelimiterGroup.java | 47 ++++++++---- .../bjc/utils/parserutils/delims/RegexOpener.java | 4 +- .../parserutils/delims/SequenceDelimiter.java | 56 +++++++++------ .../parserutils/splitter/SimpleTokenSplitter.java | 2 +- 9 files changed, 210 insertions(+), 89 deletions(-) (limited to 'BJC-Utils2/src/main/java/bjc/utils/parserutils') diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java index 888ea7a..a885808 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java @@ -36,9 +36,11 @@ class DoubleMatcher { * Floating point components. */ private static final String rFPLeader = getRegex("fpLeader"); - private static final String rFPNum = applyFormat("fpNumber", rSimpleIntDec, rSimpleDec, - rHexString); + private static final String rFPNum = applyFormat("fpNumber", rSimpleIntDec, rSimpleDec, rHexString); + /* + * Full double. + */ private static final String rDouble = applyFormat("fpDouble", rFPLeader, rFPNum); public static final Pattern doubleLiteral = Pattern.compile("\\A" + rDouble + "\\Z"); -} \ No newline at end of file +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java index 44744f5..a1b5feb 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java @@ -57,6 +57,9 @@ public class ShuntingYard { } } + /* + * Function that shunts tokens. + */ private final class TokenShunter implements Consumer { private final IList output; private final Deque stack; @@ -71,34 +74,47 @@ public class ShuntingYard { @Override public void accept(final String token) { - // Handle operators + /* + * Handle operators + */ if (operators.containsKey(token)) { - // Pop operators while there isn't a higher - // precedence one + /* + * Pop operators while there isn't a higher precedence one + */ while (!stack.isEmpty() && isHigherPrec(token, stack.peek())) { output.add(transformer.apply(stack.pop())); } - // Put this operator onto the stack + /* + * Put this operator onto the stack + */ stack.push(token); } else if (StringUtils.containsOnly(token, "\\(")) { - // Handle groups of parenthesis for multiple - // nesting levels + /* + * Handle groups of parenthesis for multiple nesting levels + */ stack.push(token); } else if (StringUtils.containsOnly(token, "\\)")) { - // Handle groups of parenthesis for multiple - // nesting levels + /* + * Handle groups of parenthesis for multiple nesting levels + */ final String swappedToken = token.replace(')', '('); - // Remove tokens up to a matching parenthesis + /* + * Remove tokens up to a matching parenthesis + */ while (!stack.peek().equals(swappedToken)) { output.add(transformer.apply(stack.pop())); } - // Remove the parenthesis + /* + * Remove the parenthesis + */ stack.pop(); } else { - // Just add the transformed token + /* + * Just add the transformed token + */ output.add(transformer.apply(token)); } } @@ -119,7 +135,9 @@ public class ShuntingYard { public ShuntingYard(final boolean configureBasics) { operators = new FunctionalMap<>(); - // Add basic operators if we're configured to do so + /* + * Add basic operators if we're configured to do so + */ if (configureBasics) { operators.put("+", Operator.ADD); operators.put("-", Operator.SUBTRACT); @@ -170,17 +188,25 @@ public class ShuntingYard { } private boolean isHigherPrec(final String left, final String right) { - // Check if the right operator exists + /* + * Check if the right operator exists + */ final boolean exists = operators.containsKey(right); - // If it doesn't, the left is higher precedence. + /* + * If it doesn't, the left is higher precedence. + */ if (!exists) return false; - // Get the precedence of operators + /* + * Get the precedence of operators + */ final int rightPrecedence = operators.get(right).getPrecedence(); final int leftPrecedence = operators.get(left).getPrecedence(); - // Evaluate what we were asked + /* + * Evaluate what we were asked + */ return rightPrecedence >= leftPrecedence; } @@ -196,21 +222,31 @@ public class ShuntingYard { * @return A list of tokens in postfix notation. */ public IList postfix(final IList input, final Function transformer) { - // Check our input + /* + * Check our input + */ if (input == null) throw new NullPointerException("Input must not be null"); else if (transformer == null) throw new NullPointerException("Transformer must not be null"); - // Here's what we're handing back + /* + * Here's what we're handing back + */ final IList output = new FunctionalList<>(); - // The stack to put operators on + /* + * The stack to put operators on + */ final Deque stack = new LinkedList<>(); - // Shunt the tokens + /* + * Shunt the tokens + */ input.forEach(new TokenShunter(output, stack, transformer)); - // Transform any resulting tokens + /* + * Transform any resulting tokens + */ stack.forEach(token -> { output.add(transformer.apply(token)); }); @@ -226,11 +262,13 @@ public class ShuntingYard { * all operators. */ public void removeOp(final String operator) { - // Check if we want to remove all operators + /* + * Check if we want to remove all operators + */ if (operator == null) { operators = new FunctionalMap<>(); } else { operators.remove(operator); } } -} \ No newline at end of file +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java index 89dc35f..30ccc5a 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java @@ -13,8 +13,13 @@ import bjc.utils.data.Tree; import bjc.utils.parserutils.TreeConstructor.ConstructorState; import bjc.utils.parserutils.TreeConstructor.QueueFlattener; +/* + * Handle creating ASTs from tokens. + */ final class TokenTransformer implements Consumer { - // Handle operators + /* + * Handle operators + */ private final class OperatorHandler implements UnaryOperator> { private final TokenType element; @@ -24,23 +29,29 @@ final class TokenTransformer implements Consumer { @Override public ConstructorState apply(final ConstructorState pair) { - // Replace the current AST with the result of handling - // an operator + /* + * Replace the current AST with the result of handling an operator + */ return new ConstructorState<>(pair.bindLeft(queuedASTs -> { return handleOperator(queuedASTs); })); } private ConstructorState handleOperator(final Deque> queuedASTs) { - // The AST we're going to hand back + /* + * The AST we're going to hand back + */ ITree newAST; - // Handle special operators + /* + * Handle special operators + */ if (isSpecialOperator.test(element)) { newAST = handleSpecialOperator.apply(element).apply(queuedASTs); } else { - // Error if we don't have enough for a binary - // operator + /* + * Error if we don't have enough for a binary operator + */ if (queuedASTs.size() < 2) { final String msg = String.format( "Attempted to parse binary operator without enough operands\n\tProblem operator is: %s\n\tPossible operand is: %s", @@ -49,18 +60,26 @@ final class TokenTransformer implements Consumer { throw new IllegalStateException(msg); } - // Grab the two operands + /* + * Grab the two operands + */ final ITree right = queuedASTs.pop(); final ITree left = queuedASTs.pop(); - // Create a new AST + /* + * Create a new AST + */ newAST = new Tree<>(element, left, right); } - // Stick it onto the stack + /* + * Stick it onto the stack + */ queuedASTs.push(newAST); - // Hand back the state + /* + * Hand back the state + */ return new ConstructorState<>(queuedASTs, newAST); } } @@ -72,7 +91,9 @@ final class TokenTransformer implements Consumer { private final Predicate isSpecialOperator; private final Function> handleSpecialOperator; - // Create a new transformer + /* + * Create a new transformer + */ public TokenTransformer(final IHolder> initialState, final Predicate operatorPredicate, final Predicate isSpecialOperator, final Function> handleSpecialOperator) { @@ -84,17 +105,21 @@ final class TokenTransformer implements Consumer { @Override public void accept(final TokenType element) { - // Handle operators + /* + * Handle operators + */ if (operatorPredicate.test(element)) { initialState.transform(new OperatorHandler(element)); } else { final ITree newAST = new Tree<>(element); - // Insert the new tree into the AST + /* + * Insert the new tree into the AST + */ initialState.transform(pair -> { - // Transform the pair, ignoring the current AST - // in favor of the - // one consisting of the current element + /* + * Transform the pair, ignoring the current AST in favor of the one consisting of the current element + */ return new ConstructorState<>(pair.bindLeft(queue -> { queue.push(newAST); diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java index a8f3db2..33dd1a2 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java @@ -155,13 +155,19 @@ public class TokenUtils { public static String descapeString(final String inp) { if (inp == null) throw new NullPointerException("inp must not be null"); + /* + * Prepare the buffer and escape finder. + */ final StringBuffer work = new StringBuffer(); - final Matcher possibleEscapeFinder = possibleEscapePatt.matcher(inp); final Matcher escapeFinder = escapePatt.matcher(inp); while (possibleEscapeFinder.find()) { if (!escapeFinder.find()) { + /* + * Found a possible escape that isn't actually an + * escape. + */ final String msg = String.format("Illegal escape sequence '%s' at position %d", possibleEscapeFinder.group(), possibleEscapeFinder.start()); @@ -170,6 +176,9 @@ public class TokenUtils { final String escapeSeq = escapeFinder.group(); + /* + * Convert the escape to a string. + */ String escapeRep = ""; switch (escapeSeq) { case "\\b": @@ -216,6 +225,9 @@ public class TokenUtils { return work.toString(); } + /* + * Handle a unicode codepoint. + */ private static String handleUnicodeEscape(final String seq) { try { final int codepoint = Integer.parseInt(seq, 16); @@ -232,6 +244,9 @@ public class TokenUtils { } } + /* + * Handle a octal codepoint. + */ private static String handleOctalEscape(final String seq) { try { final int codepoint = Integer.parseInt(seq, 8); diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java index d7ed5b0..90141ef 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java @@ -58,7 +58,9 @@ public class TreeConstructor { */ public static ITree constructTree(final IList tokens, final Predicate isOperator) { - // Construct a tree with no special operators + /* + * Construct a tree with no special operators + */ return constructTree(tokens, isOperator, op -> false, null); } @@ -91,7 +93,9 @@ public class TreeConstructor { public static ITree constructTree(final IList tokens, final Predicate isOperator, final Predicate isSpecialOperator, final Function> handleSpecialOperator) { - // Make sure our parameters are valid + /* + * Make sure our parameters are valid + */ if (tokens == null) throw new NullPointerException("Tokens must not be null"); else if (isOperator == null) @@ -99,17 +103,23 @@ public class TreeConstructor { else if (isSpecialOperator == null) throw new NullPointerException("Special operator determiner must not be null"); - // Here is the state for the tree construction + /* + * Here is the state for the tree construction + */ final IHolder> initialState = new Identity<>( new ConstructorState<>(new LinkedList<>(), null)); - // Transform each of the tokens + /* + * Transform each of the tokens + */ tokens.forEach(new TokenTransformer<>(initialState, isOperator, isSpecialOperator, handleSpecialOperator)); - // Grab the tree from the state + /* + * Grab the tree from the state + */ return initialState.unwrap(pair -> { return pair.getRight(); }); } -} \ No newline at end of file +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/DelimiterGroup.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/DelimiterGroup.java index 85d4038..b1d8597 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/DelimiterGroup.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/DelimiterGroup.java @@ -34,12 +34,20 @@ public class DelimiterGroup { * */ public class OpenGroup { + /* + * The contents of this group. + */ private final Deque> contents; + /* + * The contents of the current subgroup. + */ private IList> currentGroup; + /* + * The token that opened the group, and any opening parameters. + */ private final T opener; - private final T[] params; /** @@ -80,14 +88,23 @@ public class DelimiterGroup { * The characteristics for building the tree. */ public void markSubgroup(final T marker, final SequenceCharacteristics chars) { + /* + * Add all of the contents to the subgroup. + */ final ITree subgroupContents = new Tree<>(chars.contents); for (final ITree itm : currentGroup) { subgroupContents.addChild(itm); } + /* + * Handle subordinate sub-groups. + */ while (!contents.isEmpty()) { final ITree possibleSubordinate = contents.peek(); + /* + * Subordinate lower priority subgroups. + */ if (possibleSubordinate.getHead().equals(chars.subgroup)) { final T otherMarker = possibleSubordinate.getChild(1).getHead(); @@ -103,7 +120,6 @@ public class DelimiterGroup { final Tree subgroup = new Tree<>(chars.subgroup, subgroupContents, new Tree<>(marker)); - //System.out.println("\tTRACE: generated subgroup\n" + subgroup + "\n\n"); contents.push(subgroup); currentGroup = new FunctionalList<>(); @@ -121,12 +137,19 @@ public class DelimiterGroup { * @return This group as a tree. */ public ITree toTree(final T closer, final SequenceCharacteristics chars) { + /* + * Mark any implied subgroups. + */ if (impliedSubgroups.containsKey(closer)) { markSubgroup(impliedSubgroups.get(closer), chars); } final ITree res = new Tree<>(chars.contents); + /* + * Add either the contents of the current group, + * or subgroups if they're their. + */ if (contents.isEmpty()) { currentGroup.forEach(res::addChild); } else { @@ -440,8 +463,7 @@ public class DelimiterGroup { * The group opened by the marker. */ public void addOpener(final T opener, final T group) { - if (opener == null) - throw new NullPointerException("Opener must not be null"); + if (opener == null) throw new NullPointerException("Opener must not be null"); else if (group == null) throw new NullPointerException("Group to open must not be null"); openDelimiters.put(opener, group); @@ -457,8 +479,7 @@ public class DelimiterGroup { * The group opened by the marker. */ public void addNestedOpener(final T opener, final T group) { - if (opener == null) - throw new NullPointerException("Opener must not be null"); + if (opener == null) throw new NullPointerException("Opener must not be null"); else if (group == null) throw new NullPointerException("Group to open must not be null"); nestedOpenDelimiters.put(opener, group); @@ -474,14 +495,10 @@ public class DelimiterGroup { * The subgroup to imply. */ public void implySubgroup(final T closer, final T subgroup) { - if (closer == null) - throw new NullPointerException("Closer must not be null"); - else if (subgroup == null) - throw new NullPointerException("Subgroup must not be null"); - else if (!closingDelimiters.contains(closer)) - throw new IllegalArgumentException(String.format("No closing delimiter '%s' defined", closer)); - else if (!subgroups.containsKey(subgroup)) - throw new IllegalArgumentException(String.format("No subgroup '%s' defined", subgroup)); + if (closer == null) throw new NullPointerException("Closer must not be null"); + else if (subgroup == null) throw new NullPointerException("Subgroup must not be null"); + else if (!closingDelimiters.contains(closer)) throw new IllegalArgumentException(String.format("No closing delimiter '%s' defined", closer)); + else if (!subgroups.containsKey(subgroup)) throw new IllegalArgumentException(String.format("No subgroup '%s' defined", subgroup)); impliedSubgroups.put(closer, subgroup); } @@ -573,4 +590,4 @@ public class DelimiterGroup { public void setForgetful(final boolean forgetful) { this.forgetful = forgetful; } -} \ No newline at end of file +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/RegexOpener.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/RegexOpener.java index 98c1dc1..ee93b73 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/RegexOpener.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/RegexOpener.java @@ -8,7 +8,7 @@ import bjc.utils.data.IPair; import bjc.utils.data.Pair; /** - * A predicated opener for use with {@link RegexOpener} + * A predicated opener for use with {@link RegexCloser} * * @author bjculkin * @@ -51,4 +51,4 @@ public class RegexOpener implements Function> { return new Pair<>(null, null); } -} \ No newline at end of file +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/SequenceDelimiter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/SequenceDelimiter.java index 48d85c1..ccfaffb 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/SequenceDelimiter.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/SequenceDelimiter.java @@ -32,6 +32,9 @@ public class SequenceDelimiter { */ private final Map> groups; + /* + * The initial group to start with. + */ private DelimiterGroup initialGroup; /** @@ -49,14 +52,14 @@ public class SequenceDelimiter { * following grammar while obeying the defined grouping rules. * *
-	 *              -> ( |  | )*
-	 *          ->  
-	 *             ->   
+	 *              → ( |  | )*
+	 *          
+	 *           
 	 *
-	 *              -> STRING
-	 *              -> STRING
-	 *             -> STRING
-	 *            -> STRING
+	 *              → STRING
+	 *              → STRING
+	 *             → STRING
+	 *            → STRING
 	 * 
* * @param chars @@ -92,9 +95,8 @@ public class SequenceDelimiter { */ public ITree delimitSequence(final SequenceCharacteristics chars, @SuppressWarnings("unchecked") final T... seq) throws DelimiterException { - if (initialGroup == null) - throw new NullPointerException("Initial group must be specified."); - else if (chars == null) throw new NullPointerException("Sequence characteristics must not be null"); + if (initialGroup == null) throw new NullPointerException("Initial group must be specified."); + else if (chars == null) throw new NullPointerException("Sequence characteristics must not be null"); /* * The stack of opened and not yet closed groups. @@ -123,9 +125,15 @@ public class SequenceDelimiter { */ final IMap whoForbid = new PushdownMap<>(); + /* + * Process each member of the sequence. + */ for (int i = 0; i < seq.length; i++) { final T tok = seq[i]; + /* + * Check if this token could open a group. + */ final IPair possibleOpenPar = groupStack.top().doesOpen(tok); T possibleOpen = possibleOpenPar.getLeft(); @@ -156,8 +164,6 @@ public class SequenceDelimiter { * exclusions from all enclosing groups. */ if (isForbidden(groupStack, forbiddenDelimiters, possibleOpen)) { - final StringBuilder msgBuilder = new StringBuilder(); - T forbiddenBy; if (whoForbid.containsKey(tok)) { @@ -168,15 +174,9 @@ public class SequenceDelimiter { final String ctxList = StringUtils.toEnglishList(groupStack.toArray(), "then"); - msgBuilder.append("Group '"); - msgBuilder.append(group); - msgBuilder.append("' can't be opened in this context."); - msgBuilder.append(" (forbidden by '"); - msgBuilder.append(forbiddenBy); - msgBuilder.append("')\nContext stack: "); - msgBuilder.append(ctxList); + final String fmt = "Group '%s' can't be opened in this context. (forbidden by '%s')\nContext Stack: %s"; - throw new DelimiterException(msgBuilder.toString()); + throw new DelimiterException(String.format(fmt, group, forbiddenBy, ctxList)); } /* @@ -244,8 +244,14 @@ public class SequenceDelimiter { forbiddenDelimiters.drop(); } } else if (!groupStack.empty() && groupStack.top().marksSubgroup(tok)) { + /* + * Mark a subgroup. + */ groupStack.top().markSubgroup(tok, chars); } else { + /* + * Add an item to the group. + */ groupStack.top().addItem(new Tree<>(tok)); } } @@ -270,16 +276,24 @@ public class SequenceDelimiter { msgBuilder.append(" to close it\nOpen groups: "); msgBuilder.append(ctxList); - throw new DelimiterException(msgBuilder.toString()); + final String fmt = "Unclosed group '%s'. Expected one of %s to close it.\nOpen groups: %n"; + + throw new DelimiterException(String.format(fmt, group.getName(), closingDelims, ctxList)); } return groupStack.pop().toTree(chars.root, chars); } + /* + * Check if a group is forbidden to open in a context. + */ private boolean isForbidden(final Stack.OpenGroup> groupStack, final Stack> forbiddenDelimiters, final T groupName) { boolean localForbid; + /* + * Check if a delimiter is locally forbidden. + */ if (groupStack.empty()) { localForbid = false; } else { diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java index d483f7a..c357886 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java @@ -43,4 +43,4 @@ public class SimpleTokenSplitter implements TokenSplitter { public String toString() { return String.format("SimpleTokenSplitter [spliter=%s, keepDelim=%s]", spliter, keepDelim); } -} \ No newline at end of file +} -- cgit v1.2.3