diff options
6 files changed, 613 insertions, 38 deletions
diff --git a/BJC-Utils2/src/examples/java/bjc/utils/examples/DelimSplitterTest.java b/BJC-Utils2/src/examples/java/bjc/utils/examples/DelimSplitterTest.java index 4cd939e..a127caa 100644 --- a/BJC-Utils2/src/examples/java/bjc/utils/examples/DelimSplitterTest.java +++ b/BJC-Utils2/src/examples/java/bjc/utils/examples/DelimSplitterTest.java @@ -1,16 +1,13 @@ package bjc.utils.examples; import bjc.utils.data.ITree; -import bjc.utils.data.TopDownTransformResult; -import bjc.utils.funcdata.bst.TreeLinearizationMethod; import bjc.utils.funcutils.StringUtils; +import bjc.utils.parserutils.DelimiterException; +import bjc.utils.parserutils.DelimiterGroup; import bjc.utils.parserutils.SequenceDelimiter; import bjc.utils.parserutils.StringDelimiter; import bjc.utils.parserutils.TokenSplitter; -import bjc.utils.parserutils.SequenceDelimiter.DelimiterException; -import bjc.utils.parserutils.SequenceDelimiter.DelimiterGroup; - import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; @@ -37,6 +34,8 @@ public class DelimSplitterTest { private Map<String, DelimiterGroup<String>> groups; + boolean verbose; + /* * Create a new tester. */ @@ -48,6 +47,8 @@ public class DelimSplitterTest { split = new TokenSplitter(); dlm = new StringDelimiter(); + + verbose = true; } private void loadMirrorDB() { @@ -133,42 +134,67 @@ public class DelimSplitterTest { break; case "splitter-compile": split.compile(); - System.out.println("Compiled splitter"); + if(verbose) { + System.out.println("Compiled splitter"); + } break; case "splitter-add": split.addDelimiter(argArray); - System.out.println("Added delimiters " + StringUtils.toEnglishList(argArray, true)); + if(verbose) { + System.out.println("Added delimiters " + StringUtils.toEnglishList(argArray, true)); + } break; case "splitter-addmulti": split.addMultiDelimiter(argArray); - System.out.println("Added multi-delimiters " + StringUtils.toEnglishList(argArray, true)); + if(verbose) { + System.out.println("Added multi-delimiters " + + StringUtils.toEnglishList(argArray, true)); + } break; case "splitter-addnon": split.addNonMatcher(argArray); - System.out.println("Added non-splitters " + StringUtils.toEnglishList(argArray, true)); + if(verbose) { + System.out.println("Added non-splitters " + + StringUtils.toEnglishList(argArray, true)); + } break; case "splitter-addmatch": for(String arg : argArray) { split.addDelimiter(arg, mirrored.get(arg)); } - System.out.println("Added matched delimiters " + StringUtils.toEnglishList(argArray, true)); + if(verbose) { + System.out.println("Added matched delimiters " + + StringUtils.toEnglishList(argArray, true)); + } break; case "splitter-debug": System.out.println(split.toString()); break; case "splitter-reset": split = new TokenSplitter(); - System.out.println("Reset splitter"); + if(verbose) { + System.out.println("Reset splitter"); + } break; case "delims-addopen": dlm.addOpener(argArray[0], argArray[1]); - System.out.printf("Added opener '%s' for group '%s'\n", argArray[0], argArray[1]); + if(verbose) { + System.out.printf("Added opener '%s' for group '%s'\n", argArray[0], argArray[1]); + } break; case "delims-addgroup": for(String arg : argArray) { dlm.addGroup(groups.get(arg)); } - System.out.println("Added groups " + StringUtils.toEnglishList(argArray, true)); + if(verbose) { + System.out.println("Added groups " + StringUtils.toEnglishList(argArray, true)); + } + break; + case "delims-setinitial": + dlm.setInitialGroup(groups.get(argArray[0])); + if(verbose) { + System.out.println("Set initial group"); + } break; case "delims-debug": System.out.println(dlm.toString()); @@ -178,13 +204,17 @@ public class DelimSplitterTest { break; case "delims-reset": dlm = new StringDelimiter(); - System.out.println("Reset delimiter"); + if(verbose) { + System.out.println("Reset delimiter"); + } break; case "delimgroups-new": for(String arg : argArray) { groups.put(arg, new DelimiterGroup<>(arg)); } - System.out.println("Created groups " + StringUtils.toEnglishList(argArray, true)); + if(verbose) { + System.out.println("Created groups " + StringUtils.toEnglishList(argArray, true)); + } break; case "delimgroups-edit": for(String arg : argArray) { @@ -199,7 +229,9 @@ public class DelimSplitterTest { case "delimgroups-reset": dlm = new StringDelimiter(); groups = new HashMap<>(); - System.out.println("Reset delimiter groups + delimiter"); + if(verbose) { + System.out.println("Reset delimiter groups + delimiter"); + } break; case "load-file": handleLoadFile(args); @@ -214,7 +246,13 @@ public class DelimSplitterTest { * Load script commands from a file. */ private void handleLoadFile(String args) { - try(FileInputStream fis = new FileInputStream(args)) { + String pth = args; + + if(args.startsWith("\"")) { + pth = args.substring(1, args.length() - 1); + } + + try(FileInputStream fis = new FileInputStream(pth)) { Scanner scn = new Scanner(fis); while(scn.hasNextLine()) { @@ -223,7 +261,9 @@ public class DelimSplitterTest { if(ln.equals("")) continue; if(ln.startsWith("#")) continue; - System.out.println("\nRead command '" + ln + "' from file\n"); + if(verbose) { + System.out.println("\nRead command '" + ln + "' from file\n"); + } handleCommand(ln, scn, false); } @@ -246,8 +286,9 @@ public class DelimSplitterTest { DelimiterGroup<String> group = groups.get(arg); - System.out.println("Editing group '" + arg + "'"); - + if(verbose) { + System.out.println("Editing group '" + arg + "'"); + } if(isInteractive) { System.out.println("Enter command (blank line to stop editing): "); } @@ -269,21 +310,31 @@ public class DelimSplitterTest { switch(command) { case "add-closing": group.addClosing(argArray); - System.out.println("Added closers " + StringUtils.toEnglishList(argArray, true)); + if(verbose) { + System.out.println("Added closers " + + StringUtils.toEnglishList(argArray, true)); + } break; case "add-tlexclude": group.addTopLevelForbid(argArray); - System.out.println("Added top-level exclusions " - + StringUtils.toEnglishList(argArray, true)); + if(verbose) { + System.out.println("Added top-level exclusions " + + StringUtils.toEnglishList(argArray, true)); + } break; case "add-exclude": group.addTopLevelForbid(argArray); - System.out.println( - "Added nested exclusions " + StringUtils.toEnglishList(argArray, true)); + if(verbose) { + System.out.println("Added nested exclusions " + + StringUtils.toEnglishList(argArray, true)); + } break; case "add-subgroup": - group.addSubgroup(argArray[0], Arrays.copyOfRange(argArray, 1, argArray.length)); - System.out.println("Added subgroups"); + group.addSubgroup(argArray[0], Integer.parseInt(argArray[1])); + if(verbose) { + System.out.println(String.format("Added subgroup %s with priority %s", + argArray[0], argArray[1])); + } break; case "debug": System.out.println(group.toString()); @@ -299,7 +350,9 @@ public class DelimSplitterTest { ln = scn.nextLine().trim(); } - System.out.println("Finished editing group '" + arg + "'"); + if(verbose) { + System.out.println("Finished editing group '" + arg + "'"); + } } private void handleDelim(String args) { @@ -354,24 +407,29 @@ public class DelimSplitterTest { } private void printDelimSeq(ITree<String> delim) { - System.out.println("Delimited tokens:\n" + delim.toString()); - System.out.print("Delimited expr:\n"); + System.out.println("Delimited tokens:\n" + delim.getChild(1).toString()); + System.out.print("Delimited expr: "); printDelimTree(delim); System.out.println(); + System.out.println(); + /* ITree<String> transform = delim.topDownTransform(this::pickNode, this::transformNode); - System.out.println("Transformed tree: " + transform); + System.out.println("Transformed tree:\n" + transform.getChild(1)); + System.out.println(); System.out.println(); - System.out.print("Transformed expr:\n"); + System.out.print("Transformed expr: "); printDelimTree(transform); + */ + System.out.println(); } private void printDelimTree(ITree<String> tree) { StringBuilder sb = new StringBuilder(); - intPrintDelimTree(tree, sb); + intPrintDelimTree(tree.getChild(1), sb); System.out.println(sb.toString().replaceAll("\\s+", " ")); } @@ -415,6 +473,7 @@ public class DelimSplitterTest { } } + /* private TopDownTransformResult pickNode(String node) { if(groups.containsKey(node) || node.equals("subgroup")) return TopDownTransformResult.PUSHDOWN; @@ -429,7 +488,7 @@ public class DelimSplitterTest { return tree; } - +*/ /** * Main method * diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/DelimiterException.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DelimiterException.java new file mode 100644 index 0000000..667e2fe --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DelimiterException.java @@ -0,0 +1,16 @@ +package bjc.utils.parserutils; + +/** + * The superclass for exceptions thrown during sequence delimitation. + */ +public class DelimiterException extends RuntimeException { + /** + * Create a new generic delimiter exception. + * + * @param res + * The reason for this exception. + */ + public DelimiterException(String res) { + super(res); + } +}
\ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/DelimiterGroup.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DelimiterGroup.java new file mode 100644 index 0000000..37a8726 --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DelimiterGroup.java @@ -0,0 +1,392 @@ +package bjc.utils.parserutils; + +import bjc.utils.data.ITree; +import bjc.utils.data.Tree; +import bjc.utils.funcdata.FunctionalList; +import bjc.utils.funcdata.IList; + +import java.util.Arrays; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Represents a possible delimiter group to match. + * + * @author EVE + * + * @param <T> + * The type of items in the sequence. + */ +public class DelimiterGroup<T> { + /** + * Represents an instance of a delimiter group. + * + * @author EVE + * + */ + public class OpenGroup { + private Deque<ITree<T>> contents; + + private IList<ITree<T>> currentGroup; + + private T opener; + + /** + * Create a new instance of a delimiter group. + * + * @param open + * The item that opened this group. + */ + public OpenGroup(T open) { + opener = open; + + contents = new LinkedList<>(); + + currentGroup = new FunctionalList<>(); + } + + /** + * Add an item to this group instance. + * + * @param itm + * The item to add to this group instance. + */ + public void addItem(ITree<T> itm) { + currentGroup.add(itm); + } + + /** + * Mark a subgroup. + * + * @param marker + * The item that indicated this subgroup. + * + * @param chars + * The characteristics for building the tree. + */ + public void markSubgroup(T marker, SequenceCharacteristics<T> chars) { + ITree<T> subgroupContents = new Tree<>(chars.contents); + for(ITree<T> itm : currentGroup) { + subgroupContents.addChild(itm); + } + + while(!contents.isEmpty()) { + ITree<T> possibleSubordinate = contents.peek(); + + if(possibleSubordinate.getHead().equals(chars.subgroup)) { + T otherMarker = possibleSubordinate.getChild(1).getHead(); + + if(subgroups.get(marker) > subgroups.get(otherMarker)) { + subgroupContents.prependChild(contents.pop()); + } else { + break; + } + } else { + subgroupContents.prependChild(contents.pop()); + } + } + + Tree<T> subgroup = new Tree<>(chars.subgroup, subgroupContents, new Tree<>(marker)); + + //System.out.println("\tTRACE: generated subgroup\n" + subgroup + "\n\n"); + contents.push(subgroup); + + currentGroup = new FunctionalList<>(); + } + + /** + * Convert this group into a tree. + * + * @param closer + * The item that closed this group. + * + * @param chars + * The characteristics for building the tree. + * + * @return This group as a tree. + */ + public ITree<T> toTree(T closer, SequenceCharacteristics<T> chars) { + ITree<T> res = new Tree<>(chars.contents); + + if(contents.isEmpty()) { + currentGroup.forEach(res::addChild); + } else { + while(!contents.isEmpty()) { + res.prependChild(contents.poll()); + } + + currentGroup.forEach(res::addChild); + } + + return new Tree<>(groupName, new Tree<>(opener), res, new Tree<>(closer)); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + + builder.append("OpenGroup [contents="); + builder.append(contents); + builder.append(", currentGroup="); + builder.append(currentGroup); + builder.append(", opener="); + builder.append(opener); + builder.append("]"); + + return builder.toString(); + } + + /** + * Check if a group is excluded at the top level of this group. + * + * @param groupName + * The group to check. + * + * @return Whether or not the provided group is excluded. + */ + public boolean excludes(T groupName) { + return topLevelExclusions.contains(groupName); + } + + /** + * Check if the provided delimiter would close this group. + * + * @param del + * The string to check as a closing delimiter. + * + * @return Whether or not the provided delimiter closes this + * group. + */ + public boolean isClosing(T del) { + return closingDelimiters.contains(del); + } + + /** + * Get the name of the group this is an instance of. + * + * @return The name of the group this is an instance of. + */ + public T getName() { + return groupName; + } + + /** + * Get the groups that aren't allowed at all in this group. + * + * @return The groups that aren't allowed at all in this group. + */ + public Set<T> getNestingExclusions() { + return groupExclusions; + } + + /** + * Checks if a given token marks a subgroup. + * + * @param tok + * The token to check. + * + * @return Whether or not the token marks a subgroup. + */ + public boolean marksSubgroup(T tok) { + return subgroups.containsKey(tok); + } + } + + /** + * The name of this delimiter group. + */ + public final T groupName; + + /* + * The delimiters that close this group. + */ + private Set<T> closingDelimiters; + + /* + * The groups that can't occur in the top level of this group. + */ + private Set<T> topLevelExclusions; + + /* + * The groups that can't occur anywhere inside this group. + */ + private Set<T> groupExclusions; + + /* + * Mapping from sub-group delimiters, to any sub-groups enclosed in + * them. + */ + private Map<T, Integer> subgroups; + + /** + * Create a new empty delimiter group. + * + * @param name + * The name of the delimiter group + */ + public DelimiterGroup(T name) { + if(name == null) throw new NullPointerException("Group name must not be null"); + + groupName = name; + + closingDelimiters = new HashSet<>(); + topLevelExclusions = new HashSet<>(); + groupExclusions = new HashSet<>(); + subgroups = new HashMap<>(); + } + + /** + * Adds one or more delimiters that close this group. + * + * @param closers + * Delimiters that close this group. + */ + @SafeVarargs + public final void addClosing(T... closers) { + List<T> closerList = Arrays.asList(closers); + + for(T closer : closerList) { + if(closer == null) { + throw new NullPointerException("Closing delimiter must not be null"); + } else if(closer.equals("")) { + /* + * We can do this because equals works on + * arbitrary objects, not just those of the same + * type. + */ + throw new IllegalArgumentException("Empty string is not a valid exclusion"); + } else { + closingDelimiters.add(closer); + } + } + } + + /** + * Adds one or more groups that cannot occur in the top level of this + * group. + * + * @param exclusions + * The groups forbidden in the top level of this group. + */ + @SafeVarargs + public final void addTopLevelForbid(T... exclusions) { + for(T exclusion : exclusions) { + if(exclusion == null) { + throw new NullPointerException("Exclusion must not be null"); + } else if(exclusion.equals("")) { + /* + * We can do this because equals works on + * arbitrary objects, not just those of the same + * type. + */ + throw new IllegalArgumentException("Empty string is not a valid exclusion"); + } else { + topLevelExclusions.add(exclusion); + } + } + } + + /** + * Adds one or more groups that cannot occur at all in this group. + * + * @param exclusions + * The groups forbidden inside this group. + */ + @SafeVarargs + public final void addGroupForbid(T... exclusions) { + for(T exclusion : exclusions) { + if(exclusion == null) { + throw new NullPointerException("Exclusion must not be null"); + } else if(exclusion.equals("")) { + /* + * We can do this because equals works on + * arbitrary objects, not just those of the same + * type. + */ + throw new IllegalArgumentException("Empty string is not a valid exclusion"); + } else { + groupExclusions.add(exclusion); + } + } + } + + /** + * Adds sub-group markers to this group. + * + * @param subgroup + * The token to mark a sub-group. + * + * @param priority + * The priority of this sub-group. + * + * @param contained + * Any sub-groups to enclose in this group. + */ + public void addSubgroup(T subgroup, int priority) { + if(subgroup == null) { + throw new NullPointerException("Subgroup marker must not be null"); + } + + subgroups.put(subgroup, priority); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + + builder.append("("); + + builder.append("groupName=["); + builder.append(groupName); + builder.append("], "); + + builder.append("closingDelimiters=["); + for(T closer : closingDelimiters) { + builder.append(closer + ","); + } + builder.deleteCharAt(builder.length() - 1); + builder.append("]"); + + if(topLevelExclusions != null && !topLevelExclusions.isEmpty()) { + builder.append(", "); + builder.append("topLevelExclusions=["); + for(T exclusion : topLevelExclusions) { + builder.append(exclusion + ","); + } + builder.deleteCharAt(builder.length() - 1); + builder.append("]"); + } + + if(groupExclusions != null && !groupExclusions.isEmpty()) { + builder.append(", "); + builder.append("groupExclusions=["); + for(T exclusion : groupExclusions) { + builder.append(exclusion + ","); + } + builder.deleteCharAt(builder.length() - 1); + builder.append("]"); + } + + builder.append(" )"); + + return builder.toString(); + } + + /** + * Open an instance of this group. + * + * @param opener + * The item that opened this group. + * + * @return An opened instance of this group. + */ + public OpenGroup open(T opener) { + return new OpenGroup(opener); + } + +}
\ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/SequenceCharacteristics.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/SequenceCharacteristics.java new file mode 100644 index 0000000..c28e42b --- /dev/null +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/SequenceCharacteristics.java @@ -0,0 +1,101 @@ +package bjc.utils.parserutils; + +/** + * Marks the parameters for building a sequence tree. + * + * @author EVE + * + * @param <T> + * The type of item in the tree. + */ +public class SequenceCharacteristics<T> { + /** + * The item to mark the root of the tree. + */ + public final T root; + + /** + * The item to mark the contents of a group/subgroup. + */ + + public final T contents; + + /** + * The item to mark a subgroup. + */ + public final T subgroup; + + /** + * Create a new set of parameters for building a tree. + * + * @param root + * The root marker. + * @param contents + * The group/subgroup contents marker. + * @param subgroup + * The subgroup marker. + */ + public SequenceCharacteristics(T root, T contents, T subgroup) { + this.root = root; + this.contents = contents; + this.subgroup = subgroup; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#hashCode() + */ + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((contents == null) ? 0 : contents.hashCode()); + result = prime * result + ((root == null) ? 0 : root.hashCode()); + result = prime * result + ((subgroup == null) ? 0 : subgroup.hashCode()); + return result; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override + public boolean equals(Object obj) { + if(this == obj) return true; + if(obj == null) return false; + if(!(obj instanceof SequenceCharacteristics)) return false; + + SequenceCharacteristics<?> other = (SequenceCharacteristics<?>) obj; + + if(contents == null) { + if(other.contents != null) return false; + } else if(!contents.equals(other.contents)) return false; + + if(root == null) { + if(other.root != null) return false; + } else if(!root.equals(other.root)) return false; + + if(subgroup == null) { + if(other.subgroup != null) return false; + } else if(!subgroup.equals(other.subgroup)) return false; + + return true; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + + builder.append("SequenceCharacteristics [root="); + builder.append(root == null ? "(null)" : root); + builder.append(", contents="); + builder.append(contents == null ? "(null)" : contents); + builder.append(", subgroup="); + builder.append(subgroup == null ? "(null)" : subgroup); + builder.append("]"); + + return builder.toString(); + } +}
\ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java index ec69ade..db2c288 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java @@ -104,6 +104,8 @@ public class TokenSplitter { */ public void addDelimiter(String... delims) { for(String delim : delims) { + if(delim == null) throw new NullPointerException("Delim must not be null"); + String quoteDelim = Pattern.quote(delim); String delimPat = String.format(WITH_DELIM, quoteDelim); @@ -133,6 +135,8 @@ public class TokenSplitter { */ public void addMultiDelimiter(String... delims) { for(String delim : delims) { + if(delim == null) throw new NullPointerException("Delim must not be null"); + String delimPat = String.format(WITH_MULTI_DELIM, "(?:" + delim + ")"); if(currPatt == null) { @@ -154,11 +158,13 @@ public class TokenSplitter { /** * Marks strings matching the pattern delim as non-splittable. * - * @param delimSet + * @param delims * The regex to not splitting matching strings. */ public void addNonMatcher(String... delims) { for(String delim : delims) { + if(delim == null) throw new NullPointerException("Delim must not be null"); + if(currPatt == null) { currPatt = new StringBuilder(); currExclusionPatt = new StringBuilder(); diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java index 8224928..ce975f1 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java @@ -20,11 +20,12 @@ public class TokenUtils { * situation that indicates its use as an infix operator. * * @param expression - * The expression to check + * The expression to check. * @param operator - * The operator to see if it is contained + * The operator to see if it is contained. + * * @return Whether or not the given expression contains the specified - * operator as a infix operator + * operator as a infix operator. */ public static boolean containsInfixOperator(String expression, String operator) { return StringUtils.countMatches(expression, operator) == 1 && !expression.equalsIgnoreCase(operator) |