Move SCL into new project

SCL is now independant of dicelang, and thus deserving of its own repo.

19 files changed, 1628 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..e8c76f2
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+/bin/
+/target/
+/tags
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..67344f0
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+<project
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+	xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>bjc</groupId>
+	<artifactId>dicelang-scl</artifactId>
+	<version>1.0-SNAPSHOT</version>
+	<name>scl</name>
+	<url>http://maven.apache.org</url>
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+	</properties>
+	<dependencies>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>3.8.1</version>
+			<scope>test</scope>
+		</dependency>
+
+
+		<dependency>
+			<groupId>bjc</groupId>
+			<artifactId>BJC-Utils2</artifactId>
+			<version>1.0.0</version>
+		</dependency>
+		<dependency>
+			<groupId>bjc</groupId>
+			<artifactId>dicelang-dice</artifactId>
+			<version>1.0.0</version>
+		</dependency>
+	</dependencies>
+</project>
diff --git a/src/main/java/bjc/dicelang/scl/Errors.java b/src/main/java/bjc/dicelang/scl/Errors.java
new file mode 100644
index 0000000..8d3faff
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/Errors.java
@@ -0,0 +1,149 @@
+package bjc.dicelang.scl;
+
+/**
+ * Repository for error messages.
+ *
+ *
+ * @author EVE
+ */
+/*
+ * @TODO 10/08/17 Ben Culkin :ErrorRefactor
+ * 
+ * This way of handling error messages is not easy to deal with. Something else
+ * needs to be done, but I'm not sure what at the moment.
+ *
+ *
+ */
+public class Errors {
+	/**
+	 * The types of error message.
+	 *
+	 * @author EVE
+	 *
+	 */
+	public static enum ErrorKey {
+		/* Stream Errors */
+		/**
+		 * Attempted to switch to a non-existant stream
+		 */
+		EK_STRM_NONEX,
+		/**
+		 * Can't delete the last stream
+		 */
+		EK_STRM_LAST,
+		/**
+		 * Unknown stream command
+		 */
+		EK_STRM_INVCOM,
+		/* SCL Errors */
+		/**
+		 * Unknown SCL token
+		 */
+		EK_SCL_INVTOKEN,
+		/**
+		 * Mismatched quote in SCL command
+		 */
+		EK_SCL_MMQUOTE,
+		/**
+		 * Stack underflow in SCL command
+		 */
+		EK_SCL_SUNDERFLOW,
+		/**
+		 * Unknown word in SCL command
+		 */
+		EK_SCL_UNWORD,
+		/**
+		 * Invalid argument to SCL command
+		 */
+		EK_SCL_INVARG,
+	}
+
+	/**
+	 * The mode for the type of error messages to print out.
+	 *
+	 * @author EVE
+	 *
+	 */
+	public static enum ErrorMode {
+		/**
+		 * Output error messages for wizards.
+		 */
+		WIZARD,
+		/**
+		 * Output error messages for developers.
+		 */
+		DEV
+	}
+
+	private ErrorMode mode;
+
+	/**
+	 * Print an error.
+	 *
+	 * @param key
+	 *        The key of the error.
+	 *
+	 * @param args
+	 *        The arguments for the error.
+	 */
+	public void printError(final ErrorKey key, final String... args) {
+		switch(mode) {
+		case WIZARD:
+			System.out.println("\t? " + key.ordinal());
+			break;
+
+		case DEV:
+			devError(key, args);
+			break;
+
+		default:
+			System.out.println("\tERROR ERROR: Unknown error mode " + mode);
+		}
+	}
+
+	private static void devError(final ErrorKey key, final String[] args) {
+		switch(key) {
+		case EK_STRM_NONEX:
+			System.out.printf("\tERROR: Attempted to switch to non-existent stream\n");
+			break;
+
+		case EK_STRM_LAST:
+			System.out.printf("\tERROR: Cannot delete last stream\n");
+			break;
+
+		case EK_STRM_INVCOM:
+			System.out.printf("\tERROR: Unknown stream control command %s\n", args[0]);
+			break;
+
+		case EK_SCL_INVTOKEN:
+			System.out.printf("\tERROR: Unknown SCL token %s\n", args[0]);
+			break;
+
+		case EK_SCL_MMQUOTE:
+			System.out.printf("\tERROR: Mismatched delimiter in SCL command\n");
+			break;
+
+		case EK_SCL_SUNDERFLOW:
+			System.out.printf("\tERROR: Not enough items in stack for word %s\n", args[0]);
+			break;
+
+		case EK_SCL_UNWORD:
+			System.out.printf("\tERROR: Unknown word %s\n", args[0]);
+			break;
+
+		default:
+			System.out.printf("\tERROR ERROR: Unknown error key %s\n", key);
+		}
+	}
+
+	/**
+	 * The instance of the errors.
+	 */
+	public final static Errors inst;
+
+	static {
+		inst = new Errors();
+
+		inst.mode = ErrorMode.DEV;
+	}
+}
diff --git a/src/main/java/bjc/dicelang/scl/StreamControlConsole.java b/src/main/java/bjc/dicelang/scl/StreamControlConsole.java
new file mode 100644
index 0000000..78c8c5e
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/StreamControlConsole.java
@@ -0,0 +1,73 @@
+package bjc.dicelang.scl;
+
+import bjc.utils.funcdata.FunctionalList;
+import bjc.utils.funcdata.IList;
+
+import java.util.Scanner;
+
+/**
+ * Implement a SCL REPL
+ *
+ * @author Ben Culkin
+ */
+public class StreamControlConsole {
+	/*
+	 * @TODO 10/08/17 :SCLArgs
+	 * 
+	 * Do something useful with the CLI args.
+	 *
+	 */
+	/**
+	 * Main method
+	 *
+	 * @param args
+	 *        Unused CLI args.
+	 */
+	public static void main(String[] args) {
+		/*
+		 * Initialize vars.
+		 * 
+		 */
+		StreamEngine sengine = new StreamEngine();
+		StreamControlEngine sclengine = new StreamControlEngine(sengine);
+		Scanner scn = new Scanner(System.in);
+
+		/* Get input from the user. */
+		System.out.print("Enter a SCL command string (blank to exit): ");
+
+		/* Process it. */
+		while(scn.hasNextLine()) {
+			String ln = scn.nextLine().trim();
+
+			if(ln.equals("")) {
+				/* Ignore empty lines. */
+				break;
+			}
+
+			/* Break the token into strings. */
+			IList<String> res = new FunctionalList<>();
+			String[] tokens = ln.split(" ");
+
+			/* Run the stream engine on the tokens. */
+			boolean succ = sengine.doStreams(tokens, res);
+			if(!succ) {
+				System.out.printf("ERROR: Stream engine failed for line '%s'\n", ln);
+				continue;
+			}
+
+			/* Run the command through SCL. */
+			tokens = res.toArray(new String[res.getSize()]);
+			succ = sclengine.runProgram(tokens);
+			if(!succ) {
+				System.out.printf("ERROR: SCL engine failed for line '%s'\n", ln);
+				continue;
+			}
+
+			/* Prompt again. */
+			System.out.print("Command string executed succesfully.\n\n");
+			System.out.print("Enter a SCL command string (blank to exit): ");
+		}
+
+		scn.close();
+	}
+}
diff --git a/src/main/java/bjc/dicelang/scl/StreamControlEngine.java b/src/main/java/bjc/dicelang/scl/StreamControlEngine.java
new file mode 100644
index 0000000..5c2b4de
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/StreamControlEngine.java
@@ -0,0 +1,437 @@
+package bjc.dicelang.scl;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import bjc.dicelang.scl.tokens.ArraySCLToken;
+import bjc.dicelang.scl.tokens.BooleanSCLToken;
+import bjc.dicelang.scl.tokens.IntSCLToken;
+import bjc.dicelang.scl.tokens.SCLToken;
+import bjc.dicelang.scl.tokens.StringLitSCLToken;
+import bjc.dicelang.scl.tokens.SymbolSCLToken;
+import bjc.dicelang.scl.tokens.WordListSCLToken;
+import bjc.dicelang.scl.tokens.WordSCLToken;
+import bjc.dicelang.scl.tokens.WordsSCLToken;
+import bjc.utils.esodata.SimpleStack;
+import bjc.utils.esodata.Stack;
+import bjc.utils.funcdata.FunctionalList;
+import bjc.utils.funcdata.IList;
+import bjc.utils.parserutils.TokenUtils;
+
+import static bjc.dicelang.scl.Errors.ErrorKey.*;
+import static bjc.dicelang.scl.tokens.TokenType.*;
+import static bjc.dicelang.scl.tokens.WordType.*;
+
+/*
+ * @TODO 10/08/17 Ben Culkin :SCLReorg
+ * 
+ * This is a large enough class that it should maybe be split into subclasses.
+ */
+/**
+ * Runs a Stream Control Language (SCL) program.
+ *
+ * SCL is a stack-based concatenative language based mostly off of Postscript
+ * and Factor, with inspiration from various other languages.
+ *
+ * @author Ben Culkin
+ */
+public class StreamControlEngine {
+	/* The stream engine we're hooked to. */
+	private final StreamEngine eng;
+
+	/* The current stack state. */
+	private final Stack<SCLToken> curStack;
+
+	/* Map of user defined words. */
+	private final Map<String, SCLToken> words;
+
+	/**
+	 * Create a new stream control engine.
+	 *
+	 * @param engine
+	 *            The engine to control.
+	 */
+	public StreamControlEngine(final StreamEngine engine) {
+		eng = engine;
+
+		words = new HashMap<>();
+		curStack = new SimpleStack<>();
+	}
+
+	/**
+	 * Run a SCL program.
+	 *
+	 * @param tokens
+	 *            The program to run.
+	 *
+	 * @return Whether the program executed successfully.
+	 */
+	public boolean runProgram(final String[] tokens) {
+		for (int i = 0; i < tokens.length; i++) {
+			/* Tokenize each token. */
+			final String token = tokens[i];
+			final SCLToken tok = SCLToken.tokenizeString(token);
+
+			if (tok == null) {
+				System.out.printf("ERROR: Tokenization failed for '%s'\n", token);
+				return false;
+			}
+
+			/* Handle token types. */
+			switch (tok.type) {
+			case SQUOTE:
+				/* Handle single-quotes. */
+				i = handleSingleQuote(i, tokens);
+				if (i == -1) {
+					return false;
+				}
+				break;
+
+			case OBRACKET:
+				/* Handle delimited brackets. */
+				i = handleDelim(i, tokens, "]");
+				if (i == -1) {
+					return false;
+				}
+				break;
+
+			case OBRACE:
+				/* Handle delimited braces. */
+				i = handleDelim(i, tokens, "}");
+				if (i == -1) {
+					return false;
+				}
+				final SCLToken brak = curStack.pop();
+				curStack.push(new ArraySCLToken(((WordListSCLToken) brak).tokenVals));
+				break;
+
+			case WORD:
+				/* Handle words. */
+				if (!handleWord((WordSCLToken) tok)) {
+					System.out.printf("WARNING: Execution of word '%s' failed\n", tok);
+				}
+				break;
+
+			default:
+				/* Put it onto the stack. */
+				curStack.push(tok);
+				break;
+			}
+		}
+
+		return true;
+	}
+
+	private boolean handleWord(final WordSCLToken tk) {
+		boolean succ = true;
+
+		/* Handle each type of word. */
+		/*
+		 * @NOTE This should probably use something other than a switch statement.
+		 */
+		switch (tk.wordVal) {
+		case NEWSTREAM:
+			eng.newStream();
+			break;
+		case LEFTSTREAM:
+			succ = eng.leftStream();
+			if (!succ) {
+				return false;
+			}
+			break;
+		case RIGHTSTREAM:
+			succ = eng.rightStream();
+			if (!succ) {
+				return false;
+			}
+			break;
+		case DELETESTREAM:
+			succ = eng.deleteStream();
+			if (!succ) {
+				return false;
+			}
+			break;
+		case MERGESTREAM:
+			succ = eng.mergeStream();
+			if (!succ) {
+				return false;
+			}
+			break;
+		case MAKEARRAY:
+			succ = makeArray();
+			if (!succ) {
+				return false;
+			}
+			break;
+		case MAKEEXEC:
+			succ = toggleExec(true);
+			if (!succ) {
+				return false;
+			}
+			break;
+		case MAKEUNEXEC:
+			succ = toggleExec(false);
+			if (!succ) {
+				return false;
+			}
+			break;
+		case STACKCOUNT:
+			curStack.push(new IntSCLToken(curStack.size()));
+			break;
+		case STACKEMPTY:
+			curStack.push(new BooleanSCLToken(curStack.empty()));
+			break;
+		case DROP:
+			if (curStack.size() == 0) {
+				Errors.inst.printError(EK_SCL_SUNDERFLOW, tk.toString());
+				return false;
+			}
+			curStack.drop();
+			break;
+		case NDROP:
+			succ = handleNDrop();
+			if (!succ) {
+				return false;
+			}
+			break;
+		case NIP:
+			if (curStack.size() < 2) {
+				Errors.inst.printError(EK_SCL_SUNDERFLOW, tk.toString());
+				return false;
+			}
+			curStack.nip();
+			break;
+		case NNIP:
+			succ = handleNNip();
+			if (!succ) {
+				return false;
+			}
+			break;
+		case DEFINE:
+			succ = handleDefine();
+			if (!succ) {
+				return false;
+			}
+			break;
+		default:
+			Errors.inst.printError(EK_SCL_UNWORD, tk.toString());
+			return false;
+		}
+
+		return true;
+	}
+
+	private boolean handleDefine() {
+		if (curStack.size() < 2) {
+			Errors.inst.printError(EK_SCL_SUNDERFLOW, "def");
+			return false;
+		}
+
+		SCLToken name = curStack.pop();
+		if (name.type != SYMBOL) {
+			Errors.inst.printError(EK_SCL_INVARG, name.type.toString());
+			return false;
+		}
+		String nam = ((SymbolSCLToken) name).stringVal;
+
+		SCLToken def = curStack.pop();
+		if (name.type != WORDS) {
+			Errors.inst.printError(EK_SCL_INVARG, def.type.toString());
+			return false;
+		}
+
+		words.put(nam, def);
+		return false;
+	}
+
+	/* Handle nipping a specified number of items. */
+	private boolean handleNNip() {
+		final SCLToken num = curStack.pop();
+
+		if (num.type != ILIT) {
+			Errors.inst.printError(EK_SCL_INVARG, num.type.toString());
+			return false;
+		}
+
+		final int n = (int) ((IntSCLToken) num).intVal;
+
+		if (curStack.size() < n) {
+			Errors.inst.printError(EK_SCL_SUNDERFLOW, NNIP.toString());
+			return false;
+		}
+
+		curStack.nip(n);
+		return true;
+	}
+
+	/* Handle dropping a specified number of items. */
+	private boolean handleNDrop() {
+		final SCLToken num = curStack.pop();
+
+		if (num.type != ILIT) {
+			Errors.inst.printError(EK_SCL_INVARG, num.type.toString());
+			return false;
+		}
+
+		final int n = (int) ((IntSCLToken) num).intVal;
+
+		if (curStack.size() < n) {
+			Errors.inst.printError(EK_SCL_SUNDERFLOW, NDROP.toString());
+			return false;
+		}
+
+		curStack.drop(n);
+		return true;
+	}
+
+	/* Handle toggling the executable flag on an array. */
+	private boolean toggleExec(final boolean exec) {
+		final SCLToken top = curStack.top();
+
+		if (exec) {
+			if (top.type != ARRAY) {
+				Errors.inst.printError(EK_SCL_INVARG, top.toString());
+				return false;
+			}
+
+			top.type = WORDS;
+		} else {
+			if (top.type != WORDS) {
+				Errors.inst.printError(EK_SCL_INVARG, top.toString());
+				return false;
+			}
+
+			top.type = ARRAY;
+		}
+
+		return true;
+	}
+
+	/* Handle creating an array. */
+	private boolean makeArray() {
+		final SCLToken num = curStack.pop();
+
+		if (num.type != ILIT) {
+			Errors.inst.printError(EK_SCL_INVARG, num.type.toString());
+		}
+
+		final IList<SCLToken> arr = new FunctionalList<>();
+
+		for (int i = 0; i < ((IntSCLToken) num).intVal; i++) {
+			arr.add(curStack.pop());
+		}
+
+		curStack.push(new ArraySCLToken(arr));
+
+		return true;
+	}
+
+	/* Handle a delimited series of tokens. */
+	private int handleDelim(final int i, final String[] tokens, final String delim) {
+		final IList<SCLToken> toks = new FunctionalList<>();
+
+		int n = i + 1;
+
+		if (n >= tokens.length) {
+			Errors.inst.printError(EK_SCL_MMQUOTE);
+			return -1;
+		}
+
+		String tok = tokens[n];
+
+		while (!tok.equals(delim)) {
+			final SCLToken ntok = SCLToken.tokenizeString(tok);
+
+			switch (ntok.type) {
+			case SQUOTE:
+				n = handleSingleQuote(n, tokens);
+				if (n == -1) {
+					return -1;
+				}
+				toks.add(curStack.pop());
+				break;
+			case OBRACKET:
+				n = handleDelim(n, tokens, "]");
+				if (n == -1) {
+					return -1;
+				}
+				toks.add(curStack.pop());
+				break;
+			case OBRACE:
+				n = handleDelim(i, tokens, "}");
+				if (n == -1) {
+					return -1;
+				}
+				final SCLToken brak = curStack.pop();
+				toks.add(new ArraySCLToken(((WordListSCLToken) brak).tokenVals));
+				break;
+			default:
+				toks.add(ntok);
+			}
+
+			/* Move to the next token */
+			n += 1;
+
+			if (n >= tokens.length) {
+				Errors.inst.printError(EK_SCL_MMQUOTE);
+				return -1;
+			}
+
+			tok = tokens[n];
+		}
+
+		/* Skip the closing bracket */
+		n += 1;
+
+		/*
+		 * @NOTE Instead of being hardcoded, this should be a parameter.
+		 */
+		curStack.push(new WordsSCLToken(toks));
+
+		return n;
+	}
+
+	/* Handle a single-quoted string. */
+	private int handleSingleQuote(final int i, final String[] tokens) {
+		final StringBuilder sb = new StringBuilder();
+
+		int n = i + 1;
+
+		if (n >= tokens.length) {
+			Errors.inst.printError(EK_SCL_MMQUOTE);
+			return -1;
+		}
+
+		String tok = tokens[n];
+
+		while (!tok.equals("'")) {
+			if (tok.matches("\\\\+'")) {
+				/* Handle escaped quotes. */
+				sb.append(tok.substring(1));
+			} else {
+				sb.append(tok);
+			}
+
+			/* Move to the next token */
+			n += 1;
+
+			if (n >= tokens.length) {
+				Errors.inst.printError(EK_SCL_MMQUOTE);
+				return -1;
+			}
+
+			tok = tokens[n];
+		}
+
+		/*
+		 * Skip the single quote
+		 */
+		n += 1;
+
+		String strang = TokenUtils.descapeString(sb.toString());
+
+		curStack.push(new StringLitSCLToken(strang));
+
+		return n;
+	}
+}
diff --git a/src/main/java/bjc/dicelang/scl/StreamEngine.java b/src/main/java/bjc/dicelang/scl/StreamEngine.java
new file mode 100644
index 0000000..59c2121
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/StreamEngine.java
@@ -0,0 +1,242 @@
+package bjc.dicelang.scl;
+
+import bjc.utils.esodata.SingleTape;
+import bjc.utils.esodata.Tape;
+import bjc.utils.esodata.TapeLibrary;
+import bjc.utils.funcdata.FunctionalList;
+import bjc.utils.funcdata.FunctionalMap;
+import bjc.utils.funcdata.IList;
+import bjc.utils.funcdata.IMap;
+import bjc.utils.funcutils.ListUtils;
+
+import java.util.Arrays;
+import java.util.function.Predicate;
+
+import static bjc.dicelang.scl.Errors.ErrorKey.*;
+
+/**
+ * Implements multiple interleaved parse streams, as well as a command language
+ * for the streams.
+ *
+ * The idea for the interleaved streams came from the language Oozylbub &amp;
+ * Murphy, but the command language was my own idea.
+ *
+ * @author Ben Culkin
+ */
+public class StreamEngine {
+	/**
+	 * Whether or not we're doing debugging.
+	 */
+	public final boolean debug = true;
+
+	/* Our streams. */
+	Tape<IList<String>> streams;
+	IList<String> currStream;
+
+	/* Saved streams */
+	TapeLibrary<IList<String>> savedStreams;
+
+	/* Handler for SCL programs */
+	private final StreamControlEngine scleng;
+
+	private static IMap<Character, Predicate<StreamEngine>> commands;
+
+	static {
+		commands = new FunctionalMap<>();
+
+		commands.put('+', (eng) -> {
+			eng.newStream();
+			return true;
+		});
+
+		commands.put('>', (eng) -> eng.rightStream());
+		commands.put('<', (eng) -> eng.leftStream());
+		commands.put('-', (eng) -> eng.deleteStream());
+		commands.put('M', (eng) -> eng.mergeStream());
+		commands.put('L', (eng) -> {
+			String[] arr = eng.currStream.toArray(new String[0]);
+
+			boolean succ = eng.scleng.runProgram(arr);
+
+			return succ;
+		});
+	}
+
+	/**
+	 * Create a new stream engine.
+	 *
+	 */
+	public StreamEngine() {
+		savedStreams = new TapeLibrary<>();
+		scleng = new StreamControlEngine(this);
+	}
+
+	/* Do pre-run (re)initialization. */
+	private void init() {
+		/* Reinitialize our list of streams. */
+		streams = new SingleTape<>();
+
+		/* Create an initial stream. */
+		currStream = new FunctionalList<>();
+		streams.insertBefore(currStream);
+	}
+
+	/**
+	 * Process a possibly interleaved set of streams.
+	 *
+	 * @param toks
+	 *        The raw token to read streams from.
+	 *
+	 * @param dest
+	 *        The list to write the final stream to.
+	 *
+	 * @return Whether or not the streams were successfully processed.
+	 */
+	public boolean doStreams(final String[] toks, final IList<String> dest) {
+		return doStreams(Arrays.asList(toks), dest);
+	}
+
+	/**
+	 * Process a possibly interleaved set of streams.
+	 *
+	 * @param toks
+	 *        The raw token to read streams from.
+	 *
+	 * @param dest
+	 *        The list to write the final stream to.
+	 *
+	 * @return Whether or not the streams were successfully processed.
+	 */
+	public boolean doStreams(final Iterable<String> toks, final IList<String> dest) {
+		/* Initialize per-run state. */
+		init();
+
+		/* Are we currently quoting things? */
+		boolean quoteMode = false;
+
+		/* Process each token. */
+		for(final String tk : toks) {
+			/* Process stream commands. */
+			if(tk.startsWith("{@S") && !quoteMode) {
+				if(tk.equals("{@SQ}")) {
+					/* Start quoting. */
+					quoteMode = true;
+				} else if(!processCommand(tk)) {
+					return false;
+				}
+			} else {
+				if(tk.equals("{@SU}")) {
+					/* Stop quoting. */
+					quoteMode = false;
+				} else if(tk.startsWith("\\") && tk.endsWith("{@SU}")) {
+					/* Unquote quoted end. */
+					currStream.add(tk.substring(1));
+				} else {
+					currStream.add(tk);
+				}
+			}
+		}
+
+		for(final String tk : currStream) {
+			/* Collect tokens from the current stream. */
+			dest.add(tk);
+		}
+
+		return true;
+	}
+
+	/** Create a new stream. */
+	public void newStream() {
+		streams.insertAfter(new FunctionalList<>());
+	}
+
+	/**
+	 * Move to a stream to the right.
+	 *
+	 * @return Whether or not the move was successful.
+	 */
+	public boolean rightStream() {
+		if(!streams.right()) {
+			Errors.inst.printError(EK_STRM_NONEX);
+			return false;
+		}
+
+		currStream = streams.item();
+		return true;
+	}
+
+	/**
+	 * Move to a stream to the left.
+	 *
+	 * @return Whether or not the move was successful.
+	 */
+	public boolean leftStream() {
+		if(!streams.left()) {
+			Errors.inst.printError(EK_STRM_NONEX);
+			return false;
+		}
+
+		currStream = streams.item();
+		return true;
+	}
+
+	/**
+	 * Delete the current stream.
+	 *
+	 * @return Whether or not the delete succeeded.
+	 */
+	public boolean deleteStream() {
+		if(streams.size() == 1) {
+			Errors.inst.printError(EK_STRM_LAST);
+			return false;
+		}
+
+		streams.remove();
+		currStream = streams.item();
+
+		return true;
+	}
+
+	/**
+	 * Merge the current stream into the previous stream.
+	 *
+	 * @return Whether or not the merge succeded.
+	 */
+	public boolean mergeStream() {
+		if(streams.size() == 1) {
+			Errors.inst.printError(EK_STRM_LAST);
+			return false;
+		}
+
+		final IList<String> stringLit = streams.remove();
+		currStream = streams.item();
+		currStream.add(ListUtils.collapseTokens(stringLit, " "));
+
+		return true;
+	}
+
+	private boolean processCommand(final String tk) {
+		char[] comms = null;
+
+		if(tk.length() > 5) {
+			/* Pull off {@S and closing } */
+			comms = tk.substring(3, tk.length() - 1).toCharArray();
+		} else {
+			/* Its a single char. command. */
+			comms = new char[1];
+			comms[0] = tk.charAt(3);
+		}
+
+		/* Process each command. */
+		for(final char comm : comms) {
+			boolean succ = commands.getOrDefault(comm, (eng) -> {
+				Errors.inst.printError(EK_STRM_INVCOM, tk);
+				return false;
+			}).test(this);
+
+			if(!succ) return false;
+		}
+
+		return true;
+	}
+}
\ No newline at end of file
diff --git a/src/main/java/bjc/dicelang/scl/tokens/ArraySCLToken.java b/src/main/java/bjc/dicelang/scl/tokens/ArraySCLToken.java
new file mode 100644
index 0000000..a06a0fd
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/ArraySCLToken.java
@@ -0,0 +1,27 @@
+package bjc.dicelang.scl.tokens;
+
+import bjc.utils.funcdata.IList;
+
+/**
+ * Represents an array token.
+ * 
+ * @author student
+ *
+ */
+public class ArraySCLToken extends WordListSCLToken {
+
+	/**
+	 * Create a new array token.
+	 * 
+	 * @param tokens
+	 *            The tokens in the array.
+	 */
+	public ArraySCLToken(IList<SCLToken> tokens) {
+		super(true, tokens);
+	}
+
+	@Override
+	public String toString() {
+		return "ArraySCLToken [tokenVals=" + tokenVals + "]";
+	}
+}
diff --git a/src/main/java/bjc/dicelang/scl/tokens/BooleanSCLToken.java b/src/main/java/bjc/dicelang/scl/tokens/BooleanSCLToken.java
new file mode 100644
index 0000000..bccffe0
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/BooleanSCLToken.java
@@ -0,0 +1,53 @@
+package bjc.dicelang.scl.tokens;
+
+/**
+ * Represents a boolean token.
+ * 
+ * @author student
+ *
+ */
+public class BooleanSCLToken extends SCLToken {
+	/**
+	 * The value of the token.
+	 */
+	public boolean boolVal;
+
+	/**
+	 * Create a new token.
+	 * 
+	 * @param val
+	 *            The value of the token.
+	 */
+	public BooleanSCLToken(boolean val) {
+		super(TokenType.BLIT);
+
+		boolVal = val;
+	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = super.hashCode();
+		result = prime * result + (boolVal ? 1231 : 1237);
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (!super.equals(obj))
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		BooleanSCLToken other = (BooleanSCLToken) obj;
+		if (boolVal != other.boolVal)
+			return false;
+		return true;
+	}
+
+	@Override
+	public String toString() {
+		return "BooleanSCLToken [boolVal=" + boolVal + "]";
+	}
+}
diff --git a/src/main/java/bjc/dicelang/scl/tokens/FloatSCLToken.java b/src/main/java/bjc/dicelang/scl/tokens/FloatSCLToken.java
new file mode 100644
index 0000000..82e44e2
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/FloatSCLToken.java
@@ -0,0 +1,55 @@
+package bjc.dicelang.scl.tokens;
+
+/**
+ * Represents a floating-point token.
+ * 
+ * @author student
+ *
+ */
+public class FloatSCLToken extends SCLToken {
+	/**
+	 * The value of the token.
+	 */
+	public double floatVal;
+
+	/**
+	 * Create a new floating-point token.
+	 * 
+	 * @param val
+	 *            The value of the token.
+	 */
+	public FloatSCLToken(double val) {
+		super(TokenType.FLIT);
+
+		floatVal = val;
+	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = super.hashCode();
+		long temp;
+		temp = Double.doubleToLongBits(floatVal);
+		result = prime * result + (int) (temp ^ (temp >>> 32));
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (!super.equals(obj))
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		FloatSCLToken other = (FloatSCLToken) obj;
+		if (Double.doubleToLongBits(floatVal) != Double.doubleToLongBits(other.floatVal))
+			return false;
+		return true;
+	}
+
+	@Override
+	public String toString() {
+		return "FloatSCLToken [floatVal=" + floatVal + "]";
+	}
+}
diff --git a/src/main/java/bjc/dicelang/scl/tokens/IntSCLToken.java b/src/main/java/bjc/dicelang/scl/tokens/IntSCLToken.java
new file mode 100644
index 0000000..3c77eee
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/IntSCLToken.java
@@ -0,0 +1,26 @@
+package bjc.dicelang.scl.tokens;
+
+/**
+ * Represents an integer token.
+ * 
+ * @author student
+ *
+ */
+public class IntSCLToken extends SCLToken {
+	/**
+	 * The integer value of the token.
+	 */
+	public long intVal;
+
+	/**
+	 * Create a new integer token.
+	 * 
+	 * @param iVal
+	 *            The value of the token.
+	 */
+	public IntSCLToken(final long iVal) {
+		super(TokenType.ILIT);
+
+		intVal = iVal;
+	}
+}
\ No newline at end of file
diff --git a/src/main/java/bjc/dicelang/scl/tokens/SCLToken.java b/src/main/java/bjc/dicelang/scl/tokens/SCLToken.java
new file mode 100644
index 0000000..4d4987f
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/SCLToken.java
@@ -0,0 +1,98 @@
+package bjc.dicelang.scl.tokens;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import bjc.dicelang.scl.Errors;
+import bjc.utils.parserutils.TokenUtils;
+
+import static bjc.dicelang.scl.Errors.ErrorKey.*;
+import static bjc.dicelang.scl.tokens.TokenType.*;
+
+/**
+ * Base class for SCL tokens.
+ * 
+ * @author student
+ *
+ */
+public class SCLToken {
+	/**
+	 * The type of the token.
+	 */
+	public TokenType type;
+
+	/**
+	 * Convert a string into a token.
+	 * 
+	 * @param token
+	 *            The string to convert into a token.
+	 * @return The token.
+	 */
+	public static SCLToken tokenizeString(final String token) {
+		if (litTokens.containsKey(token)) {
+			return new SCLToken(litTokens.get(token));
+		} else if (token.startsWith("\\")) {
+			return new SymbolSCLToken(token.substring(1));
+		} else if (WordSCLToken.isBuiltinWord(token)) {
+			return new WordSCLToken(token);
+		} else if (token.equals("true")) {
+			return new BooleanSCLToken(true);
+		} else if (token.equals("false")) {
+			return new BooleanSCLToken(false);
+		} else if (TokenUtils.isInt(token)) {
+			return new IntSCLToken(Long.parseLong(token));
+		} else if (TokenUtils.isDouble(token)) {
+			return new FloatSCLToken(Double.parseDouble(token));
+		} else {
+			Errors.inst.printError(EK_SCL_INVTOKEN, token);
+			return null;
+		}
+	}
+
+	protected static final Map<String, TokenType> litTokens;
+
+	protected SCLToken() {
+
+	}
+
+	protected SCLToken(TokenType typ) {
+		type = typ;
+	}
+
+	static {
+		/* Init literal tokens. */
+		litTokens = new HashMap<>();
+
+		litTokens.put("'", SQUOTE);
+		litTokens.put("\"", DQUOTE);
+		litTokens.put("[", OBRACKET);
+		litTokens.put("{", OBRACE);
+	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + ((type == null) ? 0 : type.hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		SCLToken other = (SCLToken) obj;
+		if (type != other.type)
+			return false;
+		return true;
+	}
+
+	@Override
+	public String toString() {
+		return "SCLToken [type=" + type + "]";
+	}
+}
\ No newline at end of file
diff --git a/src/main/java/bjc/dicelang/scl/tokens/StringLitSCLToken.java b/src/main/java/bjc/dicelang/scl/tokens/StringLitSCLToken.java
new file mode 100644
index 0000000..d2a10f9
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/StringLitSCLToken.java
@@ -0,0 +1,25 @@
+package bjc.dicelang.scl.tokens;
+
+/**
+ * Represents a literal string token.
+ * 
+ * @author student
+ *
+ */
+public class StringLitSCLToken extends StringSCLToken {
+
+	/**
+	 * Create a new literal string token.
+	 * 
+	 * @param val
+	 *            The string value of the token.
+	 */
+	public StringLitSCLToken(String val) {
+		super(false, val);
+	}
+
+	@Override
+	public String toString() {
+		return "StringLitSCLToken [stringVal=" + stringVal + "]";
+	}
+}
diff --git a/src/main/java/bjc/dicelang/scl/tokens/StringSCLToken.java b/src/main/java/bjc/dicelang/scl/tokens/StringSCLToken.java
new file mode 100644
index 0000000..40e5c27
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/StringSCLToken.java
@@ -0,0 +1,54 @@
+package bjc.dicelang.scl.tokens;
+
+/**
+ * Base class for tokens containing strings.
+ * 
+ * @author student
+ *
+ */
+public abstract class StringSCLToken extends SCLToken {
+	/**
+	 * String value of the token.
+	 */
+	public String stringVal;
+
+	protected StringSCLToken(boolean isSymbol, String val) {
+		if (isSymbol) {
+			type = TokenType.SYMBOL;
+		} else {
+			type = TokenType.SLIT;
+		}
+
+		stringVal = val;
+	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = super.hashCode();
+		result = prime * result + ((stringVal == null) ? 0 : stringVal.hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (!super.equals(obj))
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		StringSCLToken other = (StringSCLToken) obj;
+		if (stringVal == null) {
+			if (other.stringVal != null)
+				return false;
+		} else if (!stringVal.equals(other.stringVal))
+			return false;
+		return true;
+	}
+
+	@Override
+	public String toString() {
+		return "StringSCLToken [stringVal=" + stringVal + "]";
+	}
+}
diff --git a/src/main/java/bjc/dicelang/scl/tokens/SymbolSCLToken.java b/src/main/java/bjc/dicelang/scl/tokens/SymbolSCLToken.java
new file mode 100644
index 0000000..d67be78
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/SymbolSCLToken.java
@@ -0,0 +1,25 @@
+package bjc.dicelang.scl.tokens;
+
+/**
+ * Represents a symbol literal.
+ * 
+ * @author student
+ *
+ */
+public class SymbolSCLToken extends StringSCLToken {
+
+	/**
+	 * Create a symbol literal
+	 * 
+	 * @param val
+	 *            The value of the symbol.
+	 */
+	public SymbolSCLToken(String val) {
+		super(true, val);
+	}
+
+	@Override
+	public String toString() {
+		return "SymbolSCLToken [stringVal=" + stringVal + "]";
+	}
+}
diff --git a/src/main/java/bjc/dicelang/scl/tokens/TokenType.java b/src/main/java/bjc/dicelang/scl/tokens/TokenType.java
new file mode 100644
index 0000000..519331c
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/TokenType.java
@@ -0,0 +1,61 @@
+package bjc.dicelang.scl.tokens;
+
+/**
+ * Represent all the types of a token.
+ * 
+ * @author student
+ *
+ */
+public enum TokenType {
+	/* Natural tokens. These come directly from strings */
+	/**
+	 * Integer literal.
+	 */
+	ILIT,
+	/**
+	 * Floating-point literal.
+	 */
+	FLIT,
+	/**
+	 * Boolean literal.
+	 */
+	BLIT,
+	/**
+	 * Single-quote.
+	 */
+	SQUOTE,
+	/**
+	 * Double-quote.
+	 */
+	DQUOTE,
+	/**
+	 * Open-bracket.
+	 */
+	OBRACKET,
+	/**
+	 * Open-brace.
+	 */
+	OBRACE,
+	/**
+	 * Symbol.
+	 */
+	SYMBOL,
+	/**
+	 * Word.
+	 */
+	WORD,
+
+	/* Synthetic tokens. These are produced from special tokens. */
+	/**
+	 * String literal.
+	 */
+	SLIT,
+	/**
+	 * List of words.
+	 */
+	WORDS,
+	/**
+	 * List of data.
+	 */
+	ARRAY,
+}
\ No newline at end of file
diff --git a/src/main/java/bjc/dicelang/scl/tokens/WordListSCLToken.java b/src/main/java/bjc/dicelang/scl/tokens/WordListSCLToken.java
new file mode 100644
index 0000000..1d870db
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/WordListSCLToken.java
@@ -0,0 +1,56 @@
+package bjc.dicelang.scl.tokens;
+
+import bjc.utils.funcdata.IList;
+
+/**
+ * Represents a list of words.
+ * 
+ * @author student
+ *
+ */
+public abstract class WordListSCLToken extends SCLToken {
+	/**
+	 * The list of words.
+	 */
+	public IList<SCLToken> tokenVals;
+
+	protected WordListSCLToken(boolean isArray, IList<SCLToken> tokens) {
+		if (isArray) {
+			type = TokenType.ARRAY;
+		} else {
+			type = TokenType.WORDS;
+		}
+
+		tokenVals = tokens;
+	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = super.hashCode();
+		result = prime * result + ((tokenVals == null) ? 0 : tokenVals.hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (!super.equals(obj))
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		WordListSCLToken other = (WordListSCLToken) obj;
+		if (tokenVals == null) {
+			if (other.tokenVals != null)
+				return false;
+		} else if (!tokenVals.equals(other.tokenVals))
+			return false;
+		return true;
+	}
+
+	@Override
+	public String toString() {
+		return "WordsSCLToken [tokenVals=" + tokenVals + "]";
+	}
+}
diff --git a/src/main/java/bjc/dicelang/scl/tokens/WordSCLToken.java b/src/main/java/bjc/dicelang/scl/tokens/WordSCLToken.java
new file mode 100644
index 0000000..6fd444d
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/WordSCLToken.java
@@ -0,0 +1,106 @@
+package bjc.dicelang.scl.tokens;
+
+import static bjc.dicelang.scl.tokens.WordType.*;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Represents a single word.
+ * 
+ * @author student
+ *
+ */
+public class WordSCLToken extends SCLToken {
+	/**
+	 * The value of the word.
+	 */
+	public WordType wordVal;
+
+	/**
+	 * Create a new word token.
+	 * 
+	 * @param wrd
+	 *            The value of the word.
+	 */
+	public WordSCLToken(String wrd) {
+		this(builtinWords.get(wrd));
+	}
+
+	/**
+	 * Create a new word token.
+	 * 
+	 * @param wrd
+	 *            The value of the word.
+	 */
+	public WordSCLToken(WordType wrd) {
+		super(TokenType.WORD);
+
+		wordVal = wrd;
+	}
+
+	@Override
+	public String toString() {
+		return "WordSCLToken [wordVal=" + wordVal + "]";
+	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = super.hashCode();
+		result = prime * result + ((wordVal == null) ? 0 : wordVal.hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (!super.equals(obj))
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		WordSCLToken other = (WordSCLToken) obj;
+		if (wordVal != other.wordVal)
+			return false;
+		return true;
+	}
+
+	/**
+	 * Check if a word is built-in.
+	 * 
+	 * @param wrd
+	 *            The word to check.
+	 * 
+	 * @return Whether or not the word is builtin.
+	 */
+	public static boolean isBuiltinWord(String wrd) {
+		return builtinWords.containsKey(wrd);
+	}
+
+	private static final Map<String, WordType> builtinWords;
+
+	static {
+		/* Init builtin words. */
+		builtinWords = new HashMap<>();
+
+		builtinWords.put("makearray", MAKEARRAY);
+		builtinWords.put("cvx", MAKEEXEC);
+		builtinWords.put("cvux", MAKEUNEXEC);
+
+		builtinWords.put("+stream", NEWSTREAM);
+		builtinWords.put(">stream", LEFTSTREAM);
+		builtinWords.put("<stream", RIGHTSTREAM);
+		builtinWords.put("-stream", DELETESTREAM);
+		builtinWords.put("<-stream", MERGESTREAM);
+
+		builtinWords.put("#", STACKCOUNT);
+		builtinWords.put("empty?", STACKEMPTY);
+		builtinWords.put("drop", DROP);
+		builtinWords.put("ndrop", NDROP);
+		builtinWords.put("nip", NIP);
+		builtinWords.put("nnip", NNIP);
+		
+		builtinWords.put("def", DEFINE);
+	}
+}
diff --git a/src/main/java/bjc/dicelang/scl/tokens/WordType.java b/src/main/java/bjc/dicelang/scl/tokens/WordType.java
new file mode 100644
index 0000000..5a8eb85
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/WordType.java
@@ -0,0 +1,77 @@
+package bjc.dicelang.scl.tokens;
+
+/**
+ * Represents the word type.
+ * 
+ * @author student
+ *
+ */
+public enum WordType {
+	/* Array manipulation */
+	/**
+	 * Create an array
+	 */
+	MAKEARRAY,
+	/**
+	 * Make a token executable.
+	 */
+	MAKEEXEC,
+	/**
+	 * Make a token unexecutable.
+	 */
+	MAKEUNEXEC,
+
+	/* Stream manipulation */
+	/**
+	 * Create a new stream.
+	 */
+	NEWSTREAM,
+	/**
+	 * Swap to the left stream.
+	 */
+	LEFTSTREAM,
+	/**
+	 * Swap to the right stream.
+	 */
+	RIGHTSTREAM,
+	/**
+	 * Delete the current stream.
+	 */
+	DELETESTREAM,
+	/**
+	 * Merge the streams.
+	 */
+	MERGESTREAM,
+
+	/* Stack manipulation */
+	/**
+	 * Get the count of items on the stack.
+	 */
+	STACKCOUNT,
+	/**
+	 * Check if the stack is empty.
+	 */
+	STACKEMPTY,
+	/**
+	 * Drop an item from the top of the stack.
+	 */
+	DROP,
+	/**
+	 * Drop a number of items from the top of the stack.
+	 */
+	NDROP,
+	/**
+	 * Drop an item, leaving the top of the stack alone.
+	 */
+	NIP,
+	/**
+	 * Drop a number of items, leaving the top of the stack alone.
+	 */
+	NNIP,
+	
+	/* Definition manipulation. */
+	/**
+	 * Define a word.
+	 */
+	DEFINE,
+}
\ No newline at end of file
diff --git a/src/main/java/bjc/dicelang/scl/tokens/WordsSCLToken.java b/src/main/java/bjc/dicelang/scl/tokens/WordsSCLToken.java
new file mode 100644
index 0000000..40c4cd4
--- /dev/null
+++ b/src/main/java/bjc/dicelang/scl/tokens/WordsSCLToken.java
@@ -0,0 +1,27 @@
+package bjc.dicelang.scl.tokens;
+
+import bjc.utils.funcdata.IList;
+
+/**
+ * A token representing an executable bunch of words.
+ * 
+ * @author student
+ *
+ */
+public class WordsSCLToken extends WordListSCLToken {
+
+	/**
+	 * Create a new executable words token.
+	 * 
+	 * @param tokens
+	 *            The tokens to use.
+	 */
+	public WordsSCLToken(IList<SCLToken> tokens) {
+		super(false, tokens);
+	}
+
+	@Override
+	public String toString() {
+		return "WordsSCLToken [tokenVals=" + tokenVals + "]";
+	}
+}