4 files changed, 268 insertions, 0 deletions
diff --git a/base/src/bjc/dicelang/tokens/DiceToken.java b/base/src/bjc/dicelang/tokens/DiceToken.java
new file mode 100644
index 0000000..4bf2068
--- /dev/null
+++ b/base/src/bjc/dicelang/tokens/DiceToken.java
@@ -0,0 +1,38 @@
+package bjc.dicelang.tokens;
+
+import bjc.dicelang.dice.DiceExpression;
+
+public class DiceToken extends Token {
+	public DiceExpression diceValue;
+
+	public DiceToken(DiceExpression val) {
+		super(Type.DICE_LIT);
+
+		diceValue = val;
+	}
+
+	@Override
+	public String toString() {
+		return super.toString() + "(" + diceValue + ")";
+	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + ((diceValue == null) ? 0 : diceValue.hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(this == obj) return true;
+		if(!super.equals(obj)) return false;
+		if(getClass() != obj.getClass()) return false;
+		DiceToken other = (DiceToken) obj;
+		if(diceValue == null) {
+			if(other.diceValue != null) return false;
+		} else if(!diceValue.equals(other.diceValue)) return false;
+		return true;
+	}
+}
diff --git a/base/src/bjc/dicelang/tokens/FloatToken.java b/base/src/bjc/dicelang/tokens/FloatToken.java
new file mode 100644
index 0000000..f56bef7
--- /dev/null
+++ b/base/src/bjc/dicelang/tokens/FloatToken.java
@@ -0,0 +1,36 @@
+package bjc.dicelang.tokens;
+
+public class FloatToken extends Token {
+	public double floatValue;
+
+	public FloatToken(double val) {
+		super(Type.FLOAT_LIT);
+
+		floatValue = val;
+	}
+
+	@Override
+	public String toString() {
+		return super.toString() + "(" + floatValue + ")";
+	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		long temp;
+		temp = Double.doubleToLongBits(floatValue);
+		result = prime * result + (int) (temp ^ (temp >>> 32));
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if(this == obj) return true;
+		if(!super.equals(obj)) return false;
+		if(getClass() != obj.getClass()) return false;
+		FloatToken other = (FloatToken) obj;
+		if(Double.doubleToLongBits(floatValue) != Double.doubleToLongBits(other.floatValue)) return false;
+		return true;
+	}
+}
diff --git a/base/src/bjc/dicelang/tokens/IntToken.java b/base/src/bjc/dicelang/tokens/IntToken.java
new file mode 100644
index 0000000..5ff4b7c
--- /dev/null
+++ b/base/src/bjc/dicelang/tokens/IntToken.java
@@ -0,0 +1,26 @@
+package bjc.dicelang.tokens;
+
+/**
+ * Represents an integer literal.
+ * 
+ * @author EVE
+ *
+ */
+public class IntToken extends Token {
+	/**
+	 * The literal value.
+	 */
+	public final long value;
+
+	/**
+	 * Create a new integer literal.
+	 * 
+	 * @param val
+	 *        The integer to use.
+	 */
+	public IntToken(long val) {
+		super(Type.INT_LIT);
+
+		value = val;
+	}
+}
diff --git a/base/src/bjc/dicelang/tokens/Token.java b/base/src/bjc/dicelang/tokens/Token.java
new file mode 100644
index 0000000..1a11ccd
--- /dev/null
+++ b/base/src/bjc/dicelang/tokens/Token.java
@@ -0,0 +1,168 @@
+package bjc.dicelang.tokens;
+
+import bjc.utils.funcdata.IList;
+
+/*
+ * @TODO 10/09/17 Ben Culkin :TokenReorg
+ * 
+ * Split the class into subclasses based off of type.
+ */
+/**
+ * Lexer token.
+ */
+@SuppressWarnings("javadoc")
+public class Token {
+	public final static Token NIL_TOKEN = new Token(Type.NIL);
+
+	/**
+	 * Possible token types
+	 */
+	public static enum Type {
+		/*
+		 * Natural tokens
+		 * 
+		 * These are produced from lexemes.
+		 */
+		/*
+		 * Math tokens.
+		 */
+		ADD, SUBTRACT, MULTIPLY, DIVIDE, IDIVIDE,
+
+		/*
+		 * Literal tokens.
+		 */
+		INT_LIT, FLOAT_LIT, STRING_LIT, VREF, DICE_LIT,
+
+		/*
+		 * Dice operators.
+		 */
+		DICESCALAR, DICEFUDGE, DICEGROUP, DICECONCAT, DICELIST,
+
+		/*
+		 * Expression operators.
+		 */
+		LET, BIND, COERCE,
+
+		/*
+		 * String operators.
+		 */
+		STRCAT, STRREP,
+
+		/*
+		 * Grouping operators.
+		 */
+		OPAREN, CPAREN, OBRACKET, CBRACKET, OBRACE, CBRACE,
+
+		/*
+		 * Synthetic tokens
+		 * 
+		 * These are produced when needed.
+		 */
+		NIL, GROUPSEP, TOKGROUP, TAGOP, TAGOPR
+	}
+
+	public final Type type;
+
+	/*
+	 * This is used for the following token types
+	 *
+	 * - INT_LIT (int value)
+	 * 
+	 * - STRING_LIT (index into string table)
+	 * 
+	 * - VREF (index into sym table)
+	 * 
+	 * - O* and C* (sym-count of current token)
+	 * 
+	 */
+	public long intValue;
+
+	/*
+	 * This is used for the following token types
+	 *
+	 * - TOKGROUP (the tokens in the group)
+	 * 
+	 * - TAG* (the tagged construct)
+	 * 
+	 */
+	public IList<Token> tokenValues;
+
+	public Token(final Type typ) {
+		type = typ;
+	}
+
+	public Token(final Type typ, final long val) {
+		this(typ);
+
+		intValue = val;
+	}
+
+	public Token(final Type typ, final IList<Token> tkVals) {
+		this(typ);
+
+		tokenValues = tkVals;
+	}
+
+	@Override
+	public String toString() {
+		switch(type) {
+		case INT_LIT:
+		case STRING_LIT:
+		case VREF:
+		case OPAREN:
+		case CPAREN:
+		case OBRACKET:
+		case CBRACKET:
+		case OBRACE:
+		case CBRACE:
+			return type.toString() + "(" + intValue + ")";
+
+		case TAGOP:
+		case TAGOPR:
+		case TOKGROUP:
+			return type.toString() + "(" + tokenValues + ")";
+
+		default:
+			return type.toString();
+		}
+	}
+
+	@Override
+	public boolean equals(final Object other) {
+		if(!(other instanceof Token)) {
+			return false;
+		}
+
+		final Token otk = (Token) other;
+
+		if(otk.type != type) {
+			return false;
+		}
+
+		switch(type) {
+		case OBRACE:
+		case OBRACKET:
+			return intValue == otk.intValue;
+		default:
+			return true;
+		}
+	}
+
+	public boolean isGrouper() {
+		switch(type) {
+		case OPAREN:
+		case OBRACE:
+		case OBRACKET:
+			return true;
+
+		default:
+			return false;
+		}
+	}
+
+	@Override
+	public int hashCode() {
+		// TODO Auto-generated method stub
+		return super.hashCode();
+	}
+}