diff options
Diffstat (limited to 'BJC-Utils2/src/main/java/bjc/utils/parserutils')
9 files changed, 210 insertions, 89 deletions
diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java index 888ea7a..a885808 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java @@ -36,9 +36,11 @@ class DoubleMatcher { * Floating point components. */ private static final String rFPLeader = getRegex("fpLeader"); - private static final String rFPNum = applyFormat("fpNumber", rSimpleIntDec, rSimpleDec, - rHexString); + private static final String rFPNum = applyFormat("fpNumber", rSimpleIntDec, rSimpleDec, rHexString); + /* + * Full double. + */ private static final String rDouble = applyFormat("fpDouble", rFPLeader, rFPNum); public static final Pattern doubleLiteral = Pattern.compile("\\A" + rDouble + "\\Z"); -}
\ No newline at end of file +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java index 44744f5..a1b5feb 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java @@ -57,6 +57,9 @@ public class ShuntingYard<TokenType> { } } + /* + * Function that shunts tokens. + */ private final class TokenShunter implements Consumer<String> { private final IList<TokenType> output; private final Deque<String> stack; @@ -71,34 +74,47 @@ public class ShuntingYard<TokenType> { @Override public void accept(final String token) { - // Handle operators + /* + * Handle operators + */ if (operators.containsKey(token)) { - // Pop operators while there isn't a higher - // precedence one + /* + * Pop operators while there isn't a higher precedence one + */ while (!stack.isEmpty() && isHigherPrec(token, stack.peek())) { output.add(transformer.apply(stack.pop())); } - // Put this operator onto the stack + /* + * Put this operator onto the stack + */ stack.push(token); } else if (StringUtils.containsOnly(token, "\\(")) { - // Handle groups of parenthesis for multiple - // nesting levels + /* + * Handle groups of parenthesis for multiple nesting levels + */ stack.push(token); } else if (StringUtils.containsOnly(token, "\\)")) { - // Handle groups of parenthesis for multiple - // nesting levels + /* + * Handle groups of parenthesis for multiple nesting levels + */ final String swappedToken = token.replace(')', '('); - // Remove tokens up to a matching parenthesis + /* + * Remove tokens up to a matching parenthesis + */ while (!stack.peek().equals(swappedToken)) { output.add(transformer.apply(stack.pop())); } - // Remove the parenthesis + /* + * Remove the parenthesis + */ stack.pop(); } else { - // Just add the transformed token + /* + * Just add the transformed token + */ output.add(transformer.apply(token)); } } @@ -119,7 +135,9 @@ public class ShuntingYard<TokenType> { public ShuntingYard(final boolean configureBasics) { operators = new FunctionalMap<>(); - // Add basic operators if we're configured to do so + /* + * Add basic operators if we're configured to do so + */ if (configureBasics) { operators.put("+", Operator.ADD); operators.put("-", Operator.SUBTRACT); @@ -170,17 +188,25 @@ public class ShuntingYard<TokenType> { } private boolean isHigherPrec(final String left, final String right) { - // Check if the right operator exists + /* + * Check if the right operator exists + */ final boolean exists = operators.containsKey(right); - // If it doesn't, the left is higher precedence. + /* + * If it doesn't, the left is higher precedence. + */ if (!exists) return false; - // Get the precedence of operators + /* + * Get the precedence of operators + */ final int rightPrecedence = operators.get(right).getPrecedence(); final int leftPrecedence = operators.get(left).getPrecedence(); - // Evaluate what we were asked + /* + * Evaluate what we were asked + */ return rightPrecedence >= leftPrecedence; } @@ -196,21 +222,31 @@ public class ShuntingYard<TokenType> { * @return A list of tokens in postfix notation. */ public IList<TokenType> postfix(final IList<String> input, final Function<String, TokenType> transformer) { - // Check our input + /* + * Check our input + */ if (input == null) throw new NullPointerException("Input must not be null"); else if (transformer == null) throw new NullPointerException("Transformer must not be null"); - // Here's what we're handing back + /* + * Here's what we're handing back + */ final IList<TokenType> output = new FunctionalList<>(); - // The stack to put operators on + /* + * The stack to put operators on + */ final Deque<String> stack = new LinkedList<>(); - // Shunt the tokens + /* + * Shunt the tokens + */ input.forEach(new TokenShunter(output, stack, transformer)); - // Transform any resulting tokens + /* + * Transform any resulting tokens + */ stack.forEach(token -> { output.add(transformer.apply(token)); }); @@ -226,11 +262,13 @@ public class ShuntingYard<TokenType> { * all operators. */ public void removeOp(final String operator) { - // Check if we want to remove all operators + /* + * Check if we want to remove all operators + */ if (operator == null) { operators = new FunctionalMap<>(); } else { operators.remove(operator); } } -}
\ No newline at end of file +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java index 89dc35f..30ccc5a 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java @@ -13,8 +13,13 @@ import bjc.utils.data.Tree; import bjc.utils.parserutils.TreeConstructor.ConstructorState; import bjc.utils.parserutils.TreeConstructor.QueueFlattener; +/* + * Handle creating ASTs from tokens. + */ final class TokenTransformer<TokenType> implements Consumer<TokenType> { - // Handle operators + /* + * Handle operators + */ private final class OperatorHandler implements UnaryOperator<ConstructorState<TokenType>> { private final TokenType element; @@ -24,23 +29,29 @@ final class TokenTransformer<TokenType> implements Consumer<TokenType> { @Override public ConstructorState<TokenType> apply(final ConstructorState<TokenType> pair) { - // Replace the current AST with the result of handling - // an operator + /* + * Replace the current AST with the result of handling an operator + */ return new ConstructorState<>(pair.bindLeft(queuedASTs -> { return handleOperator(queuedASTs); })); } private ConstructorState<TokenType> handleOperator(final Deque<ITree<TokenType>> queuedASTs) { - // The AST we're going to hand back + /* + * The AST we're going to hand back + */ ITree<TokenType> newAST; - // Handle special operators + /* + * Handle special operators + */ if (isSpecialOperator.test(element)) { newAST = handleSpecialOperator.apply(element).apply(queuedASTs); } else { - // Error if we don't have enough for a binary - // operator + /* + * Error if we don't have enough for a binary operator + */ if (queuedASTs.size() < 2) { final String msg = String.format( "Attempted to parse binary operator without enough operands\n\tProblem operator is: %s\n\tPossible operand is: %s", @@ -49,18 +60,26 @@ final class TokenTransformer<TokenType> implements Consumer<TokenType> { throw new IllegalStateException(msg); } - // Grab the two operands + /* + * Grab the two operands + */ final ITree<TokenType> right = queuedASTs.pop(); final ITree<TokenType> left = queuedASTs.pop(); - // Create a new AST + /* + * Create a new AST + */ newAST = new Tree<>(element, left, right); } - // Stick it onto the stack + /* + * Stick it onto the stack + */ queuedASTs.push(newAST); - // Hand back the state + /* + * Hand back the state + */ return new ConstructorState<>(queuedASTs, newAST); } } @@ -72,7 +91,9 @@ final class TokenTransformer<TokenType> implements Consumer<TokenType> { private final Predicate<TokenType> isSpecialOperator; private final Function<TokenType, QueueFlattener<TokenType>> handleSpecialOperator; - // Create a new transformer + /* + * Create a new transformer + */ public TokenTransformer(final IHolder<ConstructorState<TokenType>> initialState, final Predicate<TokenType> operatorPredicate, final Predicate<TokenType> isSpecialOperator, final Function<TokenType, QueueFlattener<TokenType>> handleSpecialOperator) { @@ -84,17 +105,21 @@ final class TokenTransformer<TokenType> implements Consumer<TokenType> { @Override public void accept(final TokenType element) { - // Handle operators + /* + * Handle operators + */ if (operatorPredicate.test(element)) { initialState.transform(new OperatorHandler(element)); } else { final ITree<TokenType> newAST = new Tree<>(element); - // Insert the new tree into the AST + /* + * Insert the new tree into the AST + */ initialState.transform(pair -> { - // Transform the pair, ignoring the current AST - // in favor of the - // one consisting of the current element + /* + * Transform the pair, ignoring the current AST in favor of the one consisting of the current element + */ return new ConstructorState<>(pair.bindLeft(queue -> { queue.push(newAST); diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java index a8f3db2..33dd1a2 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java @@ -155,13 +155,19 @@ public class TokenUtils { public static String descapeString(final String inp) { if (inp == null) throw new NullPointerException("inp must not be null"); + /* + * Prepare the buffer and escape finder. + */ final StringBuffer work = new StringBuffer(); - final Matcher possibleEscapeFinder = possibleEscapePatt.matcher(inp); final Matcher escapeFinder = escapePatt.matcher(inp); while (possibleEscapeFinder.find()) { if (!escapeFinder.find()) { + /* + * Found a possible escape that isn't actually an + * escape. + */ final String msg = String.format("Illegal escape sequence '%s' at position %d", possibleEscapeFinder.group(), possibleEscapeFinder.start()); @@ -170,6 +176,9 @@ public class TokenUtils { final String escapeSeq = escapeFinder.group(); + /* + * Convert the escape to a string. + */ String escapeRep = ""; switch (escapeSeq) { case "\\b": @@ -216,6 +225,9 @@ public class TokenUtils { return work.toString(); } + /* + * Handle a unicode codepoint. + */ private static String handleUnicodeEscape(final String seq) { try { final int codepoint = Integer.parseInt(seq, 16); @@ -232,6 +244,9 @@ public class TokenUtils { } } + /* + * Handle a octal codepoint. + */ private static String handleOctalEscape(final String seq) { try { final int codepoint = Integer.parseInt(seq, 8); diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java index d7ed5b0..90141ef 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java @@ -58,7 +58,9 @@ public class TreeConstructor { */ public static <TokenType> ITree<TokenType> constructTree(final IList<TokenType> tokens, final Predicate<TokenType> isOperator) { - // Construct a tree with no special operators + /* + * Construct a tree with no special operators + */ return constructTree(tokens, isOperator, op -> false, null); } @@ -91,7 +93,9 @@ public class TreeConstructor { public static <TokenType> ITree<TokenType> constructTree(final IList<TokenType> tokens, final Predicate<TokenType> isOperator, final Predicate<TokenType> isSpecialOperator, final Function<TokenType, QueueFlattener<TokenType>> handleSpecialOperator) { - // Make sure our parameters are valid + /* + * Make sure our parameters are valid + */ if (tokens == null) throw new NullPointerException("Tokens must not be null"); else if (isOperator == null) @@ -99,17 +103,23 @@ public class TreeConstructor { else if (isSpecialOperator == null) throw new NullPointerException("Special operator determiner must not be null"); - // Here is the state for the tree construction + /* + * Here is the state for the tree construction + */ final IHolder<ConstructorState<TokenType>> initialState = new Identity<>( new ConstructorState<>(new LinkedList<>(), null)); - // Transform each of the tokens + /* + * Transform each of the tokens + */ tokens.forEach(new TokenTransformer<>(initialState, isOperator, isSpecialOperator, handleSpecialOperator)); - // Grab the tree from the state + /* + * Grab the tree from the state + */ return initialState.unwrap(pair -> { return pair.getRight(); }); } -}
\ No newline at end of file +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/DelimiterGroup.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/DelimiterGroup.java index 85d4038..b1d8597 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/DelimiterGroup.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/DelimiterGroup.java @@ -34,12 +34,20 @@ public class DelimiterGroup<T> { * */ public class OpenGroup { + /* + * The contents of this group. + */ private final Deque<ITree<T>> contents; + /* + * The contents of the current subgroup. + */ private IList<ITree<T>> currentGroup; + /* + * The token that opened the group, and any opening parameters. + */ private final T opener; - private final T[] params; /** @@ -80,14 +88,23 @@ public class DelimiterGroup<T> { * The characteristics for building the tree. */ public void markSubgroup(final T marker, final SequenceCharacteristics<T> chars) { + /* + * Add all of the contents to the subgroup. + */ final ITree<T> subgroupContents = new Tree<>(chars.contents); for (final ITree<T> itm : currentGroup) { subgroupContents.addChild(itm); } + /* + * Handle subordinate sub-groups. + */ while (!contents.isEmpty()) { final ITree<T> possibleSubordinate = contents.peek(); + /* + * Subordinate lower priority subgroups. + */ if (possibleSubordinate.getHead().equals(chars.subgroup)) { final T otherMarker = possibleSubordinate.getChild(1).getHead(); @@ -103,7 +120,6 @@ public class DelimiterGroup<T> { final Tree<T> subgroup = new Tree<>(chars.subgroup, subgroupContents, new Tree<>(marker)); - //System.out.println("\tTRACE: generated subgroup\n" + subgroup + "\n\n"); contents.push(subgroup); currentGroup = new FunctionalList<>(); @@ -121,12 +137,19 @@ public class DelimiterGroup<T> { * @return This group as a tree. */ public ITree<T> toTree(final T closer, final SequenceCharacteristics<T> chars) { + /* + * Mark any implied subgroups. + */ if (impliedSubgroups.containsKey(closer)) { markSubgroup(impliedSubgroups.get(closer), chars); } final ITree<T> res = new Tree<>(chars.contents); + /* + * Add either the contents of the current group, + * or subgroups if they're their. + */ if (contents.isEmpty()) { currentGroup.forEach(res::addChild); } else { @@ -440,8 +463,7 @@ public class DelimiterGroup<T> { * The group opened by the marker. */ public void addOpener(final T opener, final T group) { - if (opener == null) - throw new NullPointerException("Opener must not be null"); + if (opener == null) throw new NullPointerException("Opener must not be null"); else if (group == null) throw new NullPointerException("Group to open must not be null"); openDelimiters.put(opener, group); @@ -457,8 +479,7 @@ public class DelimiterGroup<T> { * The group opened by the marker. */ public void addNestedOpener(final T opener, final T group) { - if (opener == null) - throw new NullPointerException("Opener must not be null"); + if (opener == null) throw new NullPointerException("Opener must not be null"); else if (group == null) throw new NullPointerException("Group to open must not be null"); nestedOpenDelimiters.put(opener, group); @@ -474,14 +495,10 @@ public class DelimiterGroup<T> { * The subgroup to imply. */ public void implySubgroup(final T closer, final T subgroup) { - if (closer == null) - throw new NullPointerException("Closer must not be null"); - else if (subgroup == null) - throw new NullPointerException("Subgroup must not be null"); - else if (!closingDelimiters.contains(closer)) - throw new IllegalArgumentException(String.format("No closing delimiter '%s' defined", closer)); - else if (!subgroups.containsKey(subgroup)) - throw new IllegalArgumentException(String.format("No subgroup '%s' defined", subgroup)); + if (closer == null) throw new NullPointerException("Closer must not be null"); + else if (subgroup == null) throw new NullPointerException("Subgroup must not be null"); + else if (!closingDelimiters.contains(closer)) throw new IllegalArgumentException(String.format("No closing delimiter '%s' defined", closer)); + else if (!subgroups.containsKey(subgroup)) throw new IllegalArgumentException(String.format("No subgroup '%s' defined", subgroup)); impliedSubgroups.put(closer, subgroup); } @@ -573,4 +590,4 @@ public class DelimiterGroup<T> { public void setForgetful(final boolean forgetful) { this.forgetful = forgetful; } -}
\ No newline at end of file +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/RegexOpener.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/RegexOpener.java index 98c1dc1..ee93b73 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/RegexOpener.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/RegexOpener.java @@ -8,7 +8,7 @@ import bjc.utils.data.IPair; import bjc.utils.data.Pair; /** - * A predicated opener for use with {@link RegexOpener} + * A predicated opener for use with {@link RegexCloser} * * @author bjculkin * @@ -51,4 +51,4 @@ public class RegexOpener implements Function<String, IPair<String, String[]>> { return new Pair<>(null, null); } -}
\ No newline at end of file +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/SequenceDelimiter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/SequenceDelimiter.java index 48d85c1..ccfaffb 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/SequenceDelimiter.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/delims/SequenceDelimiter.java @@ -32,6 +32,9 @@ public class SequenceDelimiter<T> { */ private final Map<T, DelimiterGroup<T>> groups; + /* + * The initial group to start with. + */ private DelimiterGroup<T> initialGroup; /** @@ -49,14 +52,14 @@ public class SequenceDelimiter<T> { * following grammar while obeying the defined grouping rules. * * <pre> - * <tree> -> (<data> | <subgroup> | <group>)* - * <subgroup> -> <tree> <marker> - * <group> -> <open> <tree> <close> + * <tree> → (<data> | <subgroup> | <group>)* + * <subgroup> → <tree> <marker> + * <group> → <open> <tree> <close> * - * <data> -> STRING - * <open> -> STRING - * <close> -> STRING - * <marker> -> STRING + * <data> → STRING + * <open> → STRING + * <close> → STRING + * <marker> → STRING * </pre> * * @param chars @@ -92,9 +95,8 @@ public class SequenceDelimiter<T> { */ public ITree<T> delimitSequence(final SequenceCharacteristics<T> chars, @SuppressWarnings("unchecked") final T... seq) throws DelimiterException { - if (initialGroup == null) - throw new NullPointerException("Initial group must be specified."); - else if (chars == null) throw new NullPointerException("Sequence characteristics must not be null"); + if (initialGroup == null) throw new NullPointerException("Initial group must be specified."); + else if (chars == null) throw new NullPointerException("Sequence characteristics must not be null"); /* * The stack of opened and not yet closed groups. @@ -123,9 +125,15 @@ public class SequenceDelimiter<T> { */ final IMap<T, T> whoForbid = new PushdownMap<>(); + /* + * Process each member of the sequence. + */ for (int i = 0; i < seq.length; i++) { final T tok = seq[i]; + /* + * Check if this token could open a group. + */ final IPair<T, T[]> possibleOpenPar = groupStack.top().doesOpen(tok); T possibleOpen = possibleOpenPar.getLeft(); @@ -156,8 +164,6 @@ public class SequenceDelimiter<T> { * exclusions from all enclosing groups. */ if (isForbidden(groupStack, forbiddenDelimiters, possibleOpen)) { - final StringBuilder msgBuilder = new StringBuilder(); - T forbiddenBy; if (whoForbid.containsKey(tok)) { @@ -168,15 +174,9 @@ public class SequenceDelimiter<T> { final String ctxList = StringUtils.toEnglishList(groupStack.toArray(), "then"); - msgBuilder.append("Group '"); - msgBuilder.append(group); - msgBuilder.append("' can't be opened in this context."); - msgBuilder.append(" (forbidden by '"); - msgBuilder.append(forbiddenBy); - msgBuilder.append("')\nContext stack: "); - msgBuilder.append(ctxList); + final String fmt = "Group '%s' can't be opened in this context. (forbidden by '%s')\nContext Stack: %s"; - throw new DelimiterException(msgBuilder.toString()); + throw new DelimiterException(String.format(fmt, group, forbiddenBy, ctxList)); } /* @@ -244,8 +244,14 @@ public class SequenceDelimiter<T> { forbiddenDelimiters.drop(); } } else if (!groupStack.empty() && groupStack.top().marksSubgroup(tok)) { + /* + * Mark a subgroup. + */ groupStack.top().markSubgroup(tok, chars); } else { + /* + * Add an item to the group. + */ groupStack.top().addItem(new Tree<>(tok)); } } @@ -270,16 +276,24 @@ public class SequenceDelimiter<T> { msgBuilder.append(" to close it\nOpen groups: "); msgBuilder.append(ctxList); - throw new DelimiterException(msgBuilder.toString()); + final String fmt = "Unclosed group '%s'. Expected one of %s to close it.\nOpen groups: %n"; + + throw new DelimiterException(String.format(fmt, group.getName(), closingDelims, ctxList)); } return groupStack.pop().toTree(chars.root, chars); } + /* + * Check if a group is forbidden to open in a context. + */ private boolean isForbidden(final Stack<DelimiterGroup<T>.OpenGroup> groupStack, final Stack<Multiset<T>> forbiddenDelimiters, final T groupName) { boolean localForbid; + /* + * Check if a delimiter is locally forbidden. + */ if (groupStack.empty()) { localForbid = false; } else { diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java index d483f7a..c357886 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java @@ -43,4 +43,4 @@ public class SimpleTokenSplitter implements TokenSplitter { public String toString() { return String.format("SimpleTokenSplitter [spliter=%s, keepDelim=%s]", spliter, keepDelim); } -}
\ No newline at end of file +} |