Begin conversion to iterator

8 files changed, 225 insertions, 124 deletions

diff --git a/clformat/src/main/java/bjc/utils/ioutils/format/CLFormatter.java b/clformat/src/main/java/bjc/utils/ioutils/format/CLFormatter.java
index f330e42..56fc69b 100644
--- a/clformat/src/main/java/bjc/utils/ioutils/format/CLFormatter.java
+++ b/clformat/src/main/java/bjc/utils/ioutils/format/CLFormatter.java
@@ -13,22 +13,7 @@ import bjc.utils.esodata.SingleTape;
 import bjc.utils.esodata.Tape;
 import bjc.utils.ioutils.ReportWriter;
 import bjc.utils.ioutils.SimpleProperties;
-import bjc.utils.ioutils.format.directives.AestheticDirective;
-import bjc.utils.ioutils.format.directives.CaseDirective;
-import bjc.utils.ioutils.format.directives.CharacterDirective;
-import bjc.utils.ioutils.format.directives.ConditionalDirective;
-import bjc.utils.ioutils.format.directives.Directive;
-import bjc.utils.ioutils.format.directives.EscapeDirective;
-import bjc.utils.ioutils.format.directives.FormatParameters;
-import bjc.utils.ioutils.format.directives.FreshlineDirective;
-import bjc.utils.ioutils.format.directives.GotoDirective;
-import bjc.utils.ioutils.format.directives.InflectDirective;
-import bjc.utils.ioutils.format.directives.IterationDirective;
-import bjc.utils.ioutils.format.directives.LiteralDirective;
-import bjc.utils.ioutils.format.directives.NumberDirective;
-import bjc.utils.ioutils.format.directives.RadixDirective;
-import bjc.utils.ioutils.format.directives.RecursiveDirective;
-import bjc.utils.ioutils.format.directives.TabulateDirective;
+import bjc.utils.ioutils.format.directives.*;
 
 /**
  * An implementation of CL's FORMAT.
@@ -37,39 +22,10 @@ import bjc.utils.ioutils.format.directives.TabulateDirective;
  *
  */
 public class CLFormatter {
-	private static String prefixParam;
-	private static String formatMod;
-	private static String directiveName;
-
-	private static String prefixList;
-	private static String formatDirective;
-
-	private static Pattern pFormatDirective;
-
 	private static Map<String, Directive> builtinDirectives;
 	private        Map<String, Directive> extraDirectives;
 
 	static {
-		SimpleProperties props = new SimpleProperties();
-
-		try (InputStream is = CLFormatter.class.getResourceAsStream("/formats.sprop")) {
-			props.loadFrom(is, false);
-		} catch (IOException ioex) {
-			// WELP, we failed. Bail
-			throw new RuntimeException("Couldn't load formats for formatter");
-		}
-		
-		String seqPrefixParam   = props.get("clFormatPrefixParam");
-
-		prefixParam   = String.format(props.get("clFormatPrefix"), seqPrefixParam);
-		formatMod     = props.get("clFormatModifier");
-		directiveName = props.get("clFormatName");
-
-		prefixList      = String.format(props.get("delimSeparatedList"), prefixParam, ",");
-		formatDirective = String.format(props.get("clFormatDirective"), prefixList, formatMod, directiveName);
-
-		pFormatDirective = Pattern.compile(formatDirective);
-
 		builtinDirectives = new HashMap<>();
 
 		builtinDirectives.put("A", new AestheticDirective());
@@ -229,20 +185,22 @@ public class CLFormatter {
 	 * @throws IOException If something goes wrong
 	 */
 	public void doFormatString(String format, ReportWriter rw, Tape<Object> tParams, boolean isToplevel) throws IOException {
-		Matcher dirMatcher = pFormatDirective.matcher(format);
-
-		// We need this StringBuffer to use appendReplacement and stuff
-		// from Matcher. The fact that for some reason, StringBuffer is
-		// final prevents us from using our own dummy StringBuffer that
-		// auto-flushes to our output stream, so we have to do it
-		// ourselves.
-		StringBuffer sb = new StringBuffer();
+		CLTokenizer cltok = new CLTokenizer(format);
 
 		boolean doTail = true;
 		try {
-			while(dirMatcher.find()) {
-				dirMatcher.appendReplacement(sb, "");
-				rw.writeBuffer(sb);
+			while (cltok.hasNext()) {
+				String direc = cltok.next();
+
+				if (!direc.startsWith("~")) {
+					rw.write(direc);
+					continue;
+				}
+
+				Matcher dirMatcher = CLPattern.getDirectiveMatcher(direc);
+				if (!dirMatcher.find()) {
+					throw new IllegalArgumentException("Unable to match directive found from tokenizer");
+				}
 
 				String dirName   = dirMatcher.group("name");
 				String dirFunc   = dirMatcher.group("funcname");
@@ -266,15 +224,13 @@ public class CLFormatter {
 				}
 
 				if(extraDirectives.containsKey(dirName)) {
-					extraDirectives.get(dirName).format(new FormatParameters(rw, item, mods, arrParams, tParams, dirMatcher, this));
+					extraDirectives.get(dirName).format(new FormatParameters(rw, item, mods, arrParams, tParams, cltok, this));
 
 					continue;
 				}
 
 				if(builtinDirectives.containsKey(dirName)) {
-					// System.err.printf("Executing directive %s (%s) (%d to %d) from string %s\n", dirName, dirMatcher.group(), dirMatcher.start(), dirMatcher.end(), format);
-
-					builtinDirectives.get(dirName).format(new FormatParameters(rw, item, mods, arrParams, tParams, dirMatcher, this));
+					builtinDirectives.get(dirName).format(new FormatParameters(rw, item, mods, arrParams, tParams, cltok, this));
 
 					continue;
 				}
@@ -303,12 +259,18 @@ public class CLFormatter {
 					case "E":
 					case "G":
 					case "$":
-						/* @TODO implement floating point directives. */
+						/* @TODO 
+						 *
+						 * implement floating point directives.
+						 */
 						throw new IllegalArgumentException("Floating-point directives aren't implemented yet.");
 					case "W":
 						/*
-						 * @TODO figure out if we want to implement
-						 * someting for these directives instead of
+						 * @TODO 
+						 *
+						 * figure out if we want to
+						 * implement someting for these
+						 * directives instead of
 						 * punting.
 						 */
 						throw new IllegalArgumentException("S and W aren't implemented. Use A instead");
@@ -329,8 +291,5 @@ public class CLFormatter {
 
 			doTail = false;
 		}
-
-		if (doTail) dirMatcher.appendTail(sb);
-		rw.writeBuffer(sb);
 	}
 }
diff --git a/clformat/src/main/java/bjc/utils/ioutils/format/CLPattern.java b/clformat/src/main/java/bjc/utils/ioutils/format/CLPattern.java
new file mode 100644
index 0000000..aed9aea
--- /dev/null
+++ b/clformat/src/main/java/bjc/utils/ioutils/format/CLPattern.java
@@ -0,0 +1,51 @@
+package bjc.utils.ioutils.format;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import bjc.utils.ioutils.SimpleProperties;
+
+public class CLPattern {
+	private static String prefixParam;
+	private static String formatMod;
+	private static String directiveName;
+
+	private static String prefixList;
+	private static String formatDirective;
+
+	private static Pattern pFormatDirective;
+
+	static {
+		SimpleProperties props = new SimpleProperties();
+
+		try (InputStream is = CLFormatter.class.getResourceAsStream("/formats.sprop")) {
+			props.loadFrom(is, false);
+		} catch (IOException ioex) {
+			// WELP, we failed. Bail
+			throw new RuntimeException("Couldn't load formats for formatter");
+		}
+		
+		String seqPrefixParam   = props.get("clFormatPrefixParam");
+
+		prefixParam   = String.format(props.get("clFormatPrefix"), seqPrefixParam);
+		formatMod     = props.get("clFormatModifier");
+		directiveName = props.get("clFormatName");
+
+		prefixList      = String.format(props.get("delimSeparatedList"), prefixParam, ",");
+		formatDirective = String.format(props.get("clFormatDirective"), prefixList, formatMod, directiveName);
+
+		pFormatDirective = Pattern.compile(formatDirective);
+	}
+
+	/**
+	 * Get a matcher for FORMAT directives.
+	 *
+	 * @param inp
+	 * 	The string to parse directives from.
+	 */
+	public static Matcher getDirectiveMatcher(String inp) {
+		return pFormatDirective.matcher(inp);
+	}
+}
diff --git a/clformat/src/main/java/bjc/utils/ioutils/format/CLTokenizer.java b/clformat/src/main/java/bjc/utils/ioutils/format/CLTokenizer.java
new file mode 100644
index 0000000..64f45c0
--- /dev/null
+++ b/clformat/src/main/java/bjc/utils/ioutils/format/CLTokenizer.java
@@ -0,0 +1,63 @@
+package bjc.utils.ioutils.format;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.regex.Matcher;
+
+import bjc.utils.ioutils.SimpleProperties;
+import bjc.utils.ioutils.format.directives.*;
+
+public class CLTokenizer implements Iterator<String> {
+	private Matcher mat;
+
+	private String dir;
+
+	public CLTokenizer(String strang) {
+		this.mat = CLPattern.getDirectiveMatcher(strang);
+	}
+
+	@Override
+	public boolean hasNext() {
+		return !mat.hitEnd();
+	}
+
+	@Override
+	public String next() {
+		String tk = getNext();
+		System.out.printf("\tToken: %s\n", tk);
+
+		return tk;
+	}
+
+	private String getNext() {
+		if (!hasNext()) return "";
+
+		if (dir != null) {
+			String tmp = dir;
+
+			dir = null;
+			return tmp;
+		}
+
+		StringBuffer sb = new StringBuffer();
+
+		while (mat.find()) {
+			mat.appendReplacement(sb, "");
+			
+			dir = mat.group();
+
+			String tmp = sb.toString();
+			if (tmp.equals("")) {
+				dir = null;
+
+				return mat.group();
+			}
+
+			return sb.toString();
+		}
+
+		mat.appendTail(sb);
+
+		return sb.toString();
+	}
+}
diff --git a/clformat/src/main/java/bjc/utils/ioutils/format/directives/CaseDirective.java b/clformat/src/main/java/bjc/utils/ioutils/format/directives/CaseDirective.java
index a21fd37..a7e2b08 100644
--- a/clformat/src/main/java/bjc/utils/ioutils/format/directives/CaseDirective.java
+++ b/clformat/src/main/java/bjc/utils/ioutils/format/directives/CaseDirective.java
@@ -2,38 +2,47 @@ package bjc.utils.ioutils.format.directives;
 
 import java.io.IOException;
 import java.io.StringWriter;
+import java.util.Iterator;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import bjc.utils.ioutils.ReportWriter;
+import bjc.utils.ioutils.format.CLPattern;
 
 public class CaseDirective implements Directive {
 	private static final Pattern wordPattern = Pattern.compile("(\\w+)(\\b*)");
 
 	@Override
 	public void format(FormatParameters dirParams) throws IOException {
-		StringBuffer condBody = new StringBuffer();
+		StringBuilder condBody = new StringBuilder();
 
 		int nestLevel = 1;
 
-		while (dirParams.dirMatcher.find()) {
+		Iterator<String> dirIter = dirParams.dirIter;
+		while (dirParams.dirIter.hasNext()) {
+			String direc = dirIter.next();
+			if (!direc.startsWith("~")) {
+				condBody.append(direc);
+				continue;
+			}
+
+			Matcher dirMat = CLPattern.getDirectiveMatcher(direc);
+			dirMat.find();
+
 			/* Process a list of clauses. */
-			String dirName = dirParams.dirMatcher.group("name");
+			String dirName = dirMat.group("name");
 
 			if (dirName != null) {
-				/* Append everything up to this directive. */
-				dirParams.dirMatcher.appendReplacement(condBody, "");
-
 				if (dirName.equals("(")) {
 					if (nestLevel > 0) {
-						condBody.append(dirParams.dirMatcher.group());
+						condBody.append(dirMat.group());
 					}
 
 					nestLevel += 1;
 				} else if (Directive.isOpening(dirName)) {
 					nestLevel += 1;
 
-					condBody.append(dirParams.dirMatcher.group());
+					condBody.append(dirMat.group());
 				} else if (dirName.equals(")")) {
 					nestLevel = Math.max(0, nestLevel - 1);
 
@@ -43,7 +52,7 @@ public class CaseDirective implements Directive {
 					nestLevel = Math.max(0, nestLevel - 1);
 				} else {
 					/* Not a special directive. */
-					condBody.append(dirParams.dirMatcher.group());
+					condBody.append(dirMat.group());
 				}
 			}
 		}
diff --git a/clformat/src/main/java/bjc/utils/ioutils/format/directives/ConditionalDirective.java b/clformat/src/main/java/bjc/utils/ioutils/format/directives/ConditionalDirective.java
index d245e6a..2cbd513 100644
--- a/clformat/src/main/java/bjc/utils/ioutils/format/directives/ConditionalDirective.java
+++ b/clformat/src/main/java/bjc/utils/ioutils/format/directives/ConditionalDirective.java
@@ -3,7 +3,11 @@ package bjc.utils.ioutils.format.directives;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.IllegalFormatConversionException;
+import java.util.Iterator;
 import java.util.List;
+import java.util.regex.Matcher;
+
+import bjc.utils.ioutils.format.CLPattern;
 import bjc.utils.ioutils.format.EscapeException;
 
 /**
@@ -15,7 +19,7 @@ import bjc.utils.ioutils.format.EscapeException;
 public class ConditionalDirective implements Directive {
 	@Override
 	public void format(FormatParameters dirParams) throws IOException {
-		StringBuffer condBody = new StringBuffer();
+		StringBuilder condBody = new StringBuilder();
 
 		List<String> clauses = new ArrayList<>();
 
@@ -24,33 +28,36 @@ public class ConditionalDirective implements Directive {
 
 		int nestLevel = 1;
 
-		while (dirParams.dirMatcher.find()) {
+		Iterator<String> dirIter = dirParams.dirIter;
+		while (dirIter.hasNext()) {
+			String direc = dirIter.next();
+			if (!direc.startsWith("~")) {
+				condBody.append(direc);
+				continue;
+			}
+
+			Matcher dirMat = CLPattern.getDirectiveMatcher(direc);
 			/* Process a list of clauses. */
-			String dirName = dirParams.dirMatcher.group("name");
-			String dirMods = dirParams.dirMatcher.group("modifiers");
+			String dirName = dirMat.group("name");
+			String dirMods = dirMat.group("modifiers");
 
-			//System.err.printf("Found conditional directive %s with %s mods and level %d\n", dirName, dirMods, nestLevel);
 			if (dirName != null) {
-				/* Append everything up to this directive. */
-				dirParams.dirMatcher.appendReplacement(condBody, "");
-
 				if (dirName.equals("[")) {
 					if (nestLevel > 0) {
-						condBody.append(dirParams.dirMatcher.group());
+						condBody.append(dirMat.group());
 					}
 					nestLevel += 1;
 				} else if (Directive.isOpening(dirName)) {
 					nestLevel += 1;
 
-					condBody.append(dirParams.dirMatcher.group());
+					condBody.append(dirMat.group());
 				} else if (dirName.equals("]")) {
 					nestLevel = Math.max(0, nestLevel - 1);
 
 					if (nestLevel == 0) {
 						/* End the conditional. */
 						String clause = condBody.toString();
-						// System.err.printf("Found clause \"%s]\"\n", clause);
-						condBody      = new StringBuffer();
+						condBody      = new StringBuilder();
 
 						if (isDefault) {
 							defClause = clause;
@@ -60,18 +67,17 @@ public class ConditionalDirective implements Directive {
 						break;
 					} else {
 						/* Not a special directive. */
-						condBody.append(dirParams.dirMatcher.group());
+						condBody.append(dirMat.group());
 					}
 				} else if (Directive.isClosing(dirName)) {
 					nestLevel = Math.max(0, nestLevel - 1);
 
-					condBody.append(dirParams.dirMatcher.group());
+					condBody.append(dirMat.group());
 				} else if (dirName.equals(";")) {
 					if (nestLevel == 1) {
 						/* End the clause. */
 						String clause = condBody.toString();
-						// System.err.printf("Found clause \"%s;\"\n", clause);
-						condBody      = new StringBuffer();
+						condBody      = new StringBuilder();
 
 						if (isDefault) {
 							defClause = clause;
@@ -86,11 +92,11 @@ public class ConditionalDirective implements Directive {
 						}
 					} else {
 						/* Not a special directive. */
-						condBody.append(dirParams.dirMatcher.group());
+						condBody.append(dirMat.group());
 					}
 				} else {
 					/* Not a special directive. */
-					condBody.append(dirParams.dirMatcher.group());
+					condBody.append(dirMat.group());
 				}
 			}
 		}
diff --git a/clformat/src/main/java/bjc/utils/ioutils/format/directives/FormatParameters.java b/clformat/src/main/java/bjc/utils/ioutils/format/directives/FormatParameters.java
index b49933d..c5f3331 100644
--- a/clformat/src/main/java/bjc/utils/ioutils/format/directives/FormatParameters.java
+++ b/clformat/src/main/java/bjc/utils/ioutils/format/directives/FormatParameters.java
@@ -1,6 +1,6 @@
 package bjc.utils.ioutils.format.directives;
 
-import java.util.regex.Matcher;
+import java.util.Iterator;
 
 import bjc.utils.esodata.Tape;
 import bjc.utils.ioutils.ReportWriter;
@@ -14,17 +14,17 @@ public class FormatParameters {
 	public CLModifiers	mods;
 	public CLParameters	arrParams;
 	public Tape<Object>	tParams;
-	public Matcher		dirMatcher;
+	public Iterator<String>	dirIter;
 	public CLFormatter	fmt;
 
 	public FormatParameters(ReportWriter rw, Object item, CLModifiers mods, CLParameters arrParams,
-			Tape<Object> tParams, Matcher dirMatcher, CLFormatter fmt) {
-		this.rw = rw;
-		this.item = item;
-		this.mods = mods;
+			Tape<Object> tParams, Iterator<String> dirIter, CLFormatter fmt) {
+		this.rw        = rw;
+		this.item      = item;
+		this.mods      = mods;
 		this.arrParams = arrParams;
-		this.tParams = tParams;
-		this.dirMatcher = dirMatcher;
-		this.fmt = fmt;
+		this.tParams   = tParams;
+		this.dirIter   = dirIter;
+		this.fmt       = fmt;
 	}
-}
\ No newline at end of file
+}
diff --git a/clformat/src/main/java/bjc/utils/ioutils/format/directives/InflectDirective.java b/clformat/src/main/java/bjc/utils/ioutils/format/directives/InflectDirective.java
index 8790cf2..5cbde64 100644
--- a/clformat/src/main/java/bjc/utils/ioutils/format/directives/InflectDirective.java
+++ b/clformat/src/main/java/bjc/utils/ioutils/format/directives/InflectDirective.java
@@ -2,8 +2,13 @@ package bjc.utils.ioutils.format.directives;
 
 import java.io.IOException;
 import java.io.StringWriter;
+import java.util.Iterator;
+import java.util.regex.Matcher;
+
 import bjc.inflexion.InflectionML;
+
 import bjc.utils.ioutils.ReportWriter;
+import bjc.utils.ioutils.format.CLPattern;
 
 /**
  * Inflection directive.
@@ -11,32 +16,36 @@ import bjc.utils.ioutils.ReportWriter;
  *
  */
 public class InflectDirective implements Directive {
-	//private static final Pattern wordPattern = Pattern.compile("(\\w+)(\\b*)");
-
 	@Override
 	public void format(FormatParameters dirParams) throws IOException {
-		StringBuffer condBody = new StringBuffer();
+		StringBuilder condBody = new StringBuilder();
 
 		int nestLevel = 1;
 
-		while (dirParams.dirMatcher.find()) {
+		Iterator<String> dirIter = dirParams.dirIter;
+		while (dirIter.hasNext()) {
+			String direc = dirIter.next();
+			if (!direc.startsWith("~")) {
+				condBody.append(direc);
+				continue;
+			}
+
+			Matcher dirMat = CLPattern.getDirectiveMatcher(direc);
+
 			/* Process a list of clauses. */
-			String dirName = dirParams.dirMatcher.group("name");
+			String dirName = dirMat.group("name");
 
 			if (dirName != null) {
-				/* Append everything up to this directive. */
-				dirParams.dirMatcher.appendReplacement(condBody, "");
-
 				if (dirName.equals("`[")) {
 					if (nestLevel > 0) {
-						condBody.append(dirParams.dirMatcher.group());
+						condBody.append(dirMat.group());
 					}
 
 					nestLevel += 1;
 				} else if (Directive.isOpening(dirName)) {
 					nestLevel += 1;
 
-					condBody.append(dirParams.dirMatcher.group());
+					condBody.append(dirMat.group());
 				} else if (dirName.equals("`]")) {
 					nestLevel = Math.max(0, nestLevel - 1);
 
@@ -46,7 +55,7 @@ public class InflectDirective implements Directive {
 					nestLevel = Math.max(0, nestLevel - 1);
 				} else {
 					/* Not a special directive. */
-					condBody.append(dirParams.dirMatcher.group());
+					condBody.append(dirMat.group());
 				}
 			}
 		}
diff --git a/clformat/src/main/java/bjc/utils/ioutils/format/directives/IterationDirective.java b/clformat/src/main/java/bjc/utils/ioutils/format/directives/IterationDirective.java
index afc1c6b..4184c76 100644
--- a/clformat/src/main/java/bjc/utils/ioutils/format/directives/IterationDirective.java
+++ b/clformat/src/main/java/bjc/utils/ioutils/format/directives/IterationDirective.java
@@ -3,12 +3,11 @@ package bjc.utils.ioutils.format.directives;
 import java.io.IOException;
 import java.util.IllegalFormatConversionException;
 import java.util.Iterator;
+import java.util.regex.Matcher;
 
 import bjc.utils.esodata.SingleTape;
 import bjc.utils.esodata.Tape;
-import bjc.utils.ioutils.format.CLFormatter;
-import bjc.utils.ioutils.format.CLParameters;
-import bjc.utils.ioutils.format.EscapeException;
+import bjc.utils.ioutils.format.*;
 
 /**
  * Implements the { directive.
@@ -22,21 +21,26 @@ public class IterationDirective implements Directive {
 	public void format(FormatParameters dirParams) throws IOException {
 		CLFormatter.checkItem(dirParams.item, '{');
 
-		StringBuffer condBody = new StringBuffer();
+		StringBuilder condBody = new StringBuilder();
 
-		while (dirParams.dirMatcher.find()) {
+		Iterator<String> dirIter = dirParams.dirIter;
+		while (dirIter.hasNext()) {
+			String direc = dirIter.next();
+			if (!direc.startsWith("~")) {
+				condBody.append(direc);
+				continue;
+			}
+
+			Matcher dirMat = CLPattern.getDirectiveMatcher(direc);
 			/* Process a list of clauses. */
-			String dirName = dirParams.dirMatcher.group("name");
+			String dirName = dirMat.group("name");
 
 			if (dirName != null) {
-				/* Append everything up to this directive. */
-				dirParams.dirMatcher.appendReplacement(condBody, "");
-
 				if (dirName.equals("}")) {
 					break;
 				} else {
 					/* Not a special directive. */
-					condBody.append(dirParams.dirMatcher.group());
+					condBody.append(dirMat.group());
 				}
 			}
 		}