Add more features to sequence delimiter.

Two main features were added.

One, various sequence closers can imply a subgroup. This is mainly useful
in contexts like arrays, where you want one subgroup per array element.

Two, predicated opening/closing delimiters. These allow having both an
infinite set of opening delimiters, as well as having a set of closers
that is both infinite and dependant on what the opener was.

Note, however, that predicated openers and closers will be slower than
using normal openers/closers, since every one has to be tried to check if
a token is a opener/closer.

3 files changed, 99 insertions, 18 deletions

diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/DelimiterGroup.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DelimiterGroup.java
index f3d7c3f..1d13565 100644
--- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/DelimiterGroup.java
+++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DelimiterGroup.java
@@ -1,6 +1,8 @@
 package bjc.utils.parserutils;
 
+import bjc.utils.data.IPair;
 import bjc.utils.data.ITree;
+import bjc.utils.data.Pair;
 import bjc.utils.data.Tree;
 import bjc.utils.funcdata.FunctionalList;
 import bjc.utils.funcdata.IList;
@@ -12,7 +14,10 @@ import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
+import java.util.function.BiPredicate;
+import java.util.function.Function;
 
 /**
  * Represents a possible delimiter group to match.
@@ -36,14 +41,20 @@ public class DelimiterGroup<T> {
 
 		private T opener;
 
+		private T[] params;
+
 		/**
 		 * Create a new instance of a delimiter group.
 		 * 
 		 * @param open
 		 *                The item that opened this group.
+		 * 
+		 * @param parms
+		 *                Any parameters from the opener.
 		 */
-		public OpenGroup(T open) {
+		public OpenGroup(T open, T[] parms) {
 			opener = open;
+			params = parms;
 
 			contents = new LinkedList<>();
 
@@ -111,6 +122,10 @@ public class DelimiterGroup<T> {
 		 * @return This group as a tree.
 		 */
 		public ITree<T> toTree(T closer, SequenceCharacteristics<T> chars) {
+			if(impliedSubgroups.containsKey(closer)) {
+				markSubgroup(impliedSubgroups.get(closer), chars);
+			}
+
 			ITree<T> res = new Tree<>(chars.contents);
 
 			if(contents.isEmpty()) {
@@ -163,6 +178,16 @@ public class DelimiterGroup<T> {
 		 *         group.
 		 */
 		public boolean isClosing(T del) {
+			if(closingDelimiters.contains(del)) {
+				return true;
+			}
+
+			for(BiPredicate<T, T[]> pred : predClosers) {
+				if(pred.test(del, params)) {
+					return true;
+				}
+			}
+
 			return closingDelimiters.contains(del);
 		}
 
@@ -195,13 +220,30 @@ public class DelimiterGroup<T> {
 		public boolean marksSubgroup(T tok) {
 			return subgroups.containsKey(tok);
 		}
-		
-		public T doesOpen(T name) {
-			if(openDelimiters.containsKey(name)) {
-				return openDelimiters.get(name);
+
+		/**
+		 * Checks if a given token opens a group.
+		 * 
+		 * @param marker
+		 *                The token to check.
+		 * 
+		 * @return The name of the group T opens, or null if it doesn't
+		 *         open one.
+		 */
+		public IPair<T, T[]> doesOpen(T marker) {
+			if(openDelimiters.containsKey(marker)) {
+				return new Pair<>(openDelimiters.get(marker), null);
 			}
-			
-			return null;
+
+			for(Function<T, IPair<T, T[]>> pred : predOpeners) {
+				IPair<T, T[]> par = pred.apply(marker);
+
+				if(par.getLeft() != null) {
+					return par;
+				}
+			}
+
+			return new Pair<>(null, null);
 		}
 	}
 
@@ -211,10 +253,10 @@ public class DelimiterGroup<T> {
 	public final T groupName;
 
 	/*
-	 * The delimiters that open groups in this group,
+	 * The delimiters that open groups at the top level of this group.
 	 */
 	private Map<T, T> openDelimiters;
-	
+
 	/*
 	 * The delimiters that close this group.
 	 */
@@ -236,6 +278,21 @@ public class DelimiterGroup<T> {
 	 */
 	private Map<T, Integer> subgroups;
 
+	/*
+	 * Subgroups implied by a particular closing delimiter
+	 */
+	private Map<T, T> impliedSubgroups;
+
+	/*
+	 * Allows more complex openings
+	 */
+	private List<Function<T, IPair<T, T[]>>> predOpeners;
+
+	/*
+	 * Allow more complex closings
+	 */
+	private List<BiPredicate<T, T[]>> predClosers;
+
 	/**
 	 * Create a new empty delimiter group.
 	 * 
@@ -249,9 +306,15 @@ public class DelimiterGroup<T> {
 
 		openDelimiters = new HashMap<>();
 		closingDelimiters = new HashSet<>();
+
 		topLevelExclusions = new HashSet<>();
 		groupExclusions = new HashSet<>();
+
 		subgroups = new HashMap<>();
+		impliedSubgroups = new HashMap<>();
+
+		predOpeners = new LinkedList<>();
+		predClosers = new LinkedList<>();
 	}
 
 	/**
@@ -348,11 +411,15 @@ public class DelimiterGroup<T> {
 
 		subgroups.put(subgroup, priority);
 	}
-	
+
 	public void addOpener(T opener, T group) {
 		openDelimiters.put(opener, group);
 	}
 
+	public void implySubgroup(T closer, T subgroup) {
+		impliedSubgroups.put(closer, subgroup);
+	}
+
 	@Override
 	public String toString() {
 		StringBuilder builder = new StringBuilder();
@@ -403,8 +470,16 @@ public class DelimiterGroup<T> {
 	 * 
 	 * @return An opened instance of this group.
 	 */
-	public OpenGroup open(T opener) {
-		return new OpenGroup(opener);
+	public OpenGroup open(T opener, T[] parms) {
+		return new OpenGroup(opener, parms);
+	}
+
+	public void addPredOpener(Function<T, IPair<T, T[]>> pred) {
+		predOpeners.add(pred);
+	}
+
+	public void addPredCloser(BiPredicate<T, T[]> pred) {
+		predClosers.add(pred);
 	}
 
 }
\ No newline at end of file
diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/SequenceDelimiter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/SequenceDelimiter.java
index 9a3bac6..771933a 100644
--- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/SequenceDelimiter.java
+++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/SequenceDelimiter.java
@@ -1,5 +1,6 @@
 package bjc.utils.parserutils;
 
+import bjc.utils.data.IPair;
 import bjc.utils.data.ITree;
 import bjc.utils.data.Tree;
 import bjc.utils.esodata.PushdownMap;
@@ -104,7 +105,7 @@ public class SequenceDelimiter<T> {
 		/*
 		 * Open initial group.
 		 */
-		groupStack.push(initialGroup.open(chars.root));
+		groupStack.push(initialGroup.open(chars.root, null));
 		
 		/*
 		 * Groups that aren't allowed to be opened at the moment.
@@ -119,7 +120,9 @@ public class SequenceDelimiter<T> {
 		for(int i = 0; i < seq.length; i++) {
 			T tok = seq[i];
 
-			T possibleOpen = groupStack.top().doesOpen(tok);
+			IPair<T, T[]> possibleOpenPar = groupStack.top().doesOpen(tok);
+			T possibleOpen = possibleOpenPar.getLeft();
+			
 			/*
 			 * If we have an opening delimiter, handle it.
 			 */
@@ -161,7 +164,7 @@ public class SequenceDelimiter<T> {
 				/*
 				 * Add an open group.
 				 */
-				DelimiterGroup<T>.OpenGroup open = group.open(tok);
+				DelimiterGroup<T>.OpenGroup open = group.open(tok, possibleOpenPar.getRight());
 				groupStack.push(open);
 
 				/*
diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java
index db2c288..d2569d9 100644
--- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java
+++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenSplitter.java
@@ -105,7 +105,7 @@ public class TokenSplitter {
 	public void addDelimiter(String... delims) {
 		for(String delim : delims) {
 			if(delim == null) throw new NullPointerException("Delim must not be null");
-			
+
 			String quoteDelim = Pattern.quote(delim);
 			String delimPat = String.format(WITH_DELIM, quoteDelim);
 
@@ -136,7 +136,7 @@ public class TokenSplitter {
 	public void addMultiDelimiter(String... delims) {
 		for(String delim : delims) {
 			if(delim == null) throw new NullPointerException("Delim must not be null");
-			
+
 			String delimPat = String.format(WITH_MULTI_DELIM, "(?:" + delim + ")");
 
 			if(currPatt == null) {
@@ -164,7 +164,7 @@ public class TokenSplitter {
 	public void addNonMatcher(String... delims) {
 		for(String delim : delims) {
 			if(delim == null) throw new NullPointerException("Delim must not be null");
-			
+
 			if(currPatt == null) {
 				currPatt = new StringBuilder();
 				currExclusionPatt = new StringBuilder();
@@ -184,6 +184,9 @@ public class TokenSplitter {
 	 * Makes this splitter ready to use.
 	 */
 	public void compile() {
+		if(currPatt == null) currPatt = new StringBuilder();
+		if(currExclusionPatt == null) currExclusionPatt = new StringBuilder();
+
 		compPatt = Pattern.compile(currPatt.toString());
 		exclusionPatt = Pattern.compile(currExclusionPatt.toString());
 	}