From 36e0911c6ec27707a74f0b90b1052a16374243ea Mon Sep 17 00:00:00 2001
From: bjculkin <bjculkin@WIT-136XG42.wvu-ad.wvu.edu>
Date: Wed, 1 Mar 2017 10:13:41 -0500
Subject: Package reorganization

---
 dice-lang/src/bjc/dicelang/v2/Tokenizer.java | 154 ---------------------------
 1 file changed, 154 deletions(-)
 delete mode 100644 dice-lang/src/bjc/dicelang/v2/Tokenizer.java

(limited to 'dice-lang/src/bjc/dicelang/v2/Tokenizer.java')

diff --git a/dice-lang/src/bjc/dicelang/v2/Tokenizer.java b/dice-lang/src/bjc/dicelang/v2/Tokenizer.java
deleted file mode 100644
index 40b1285..0000000
--- a/dice-lang/src/bjc/dicelang/v2/Tokenizer.java
+++ /dev/null
@@ -1,154 +0,0 @@
-package bjc.dicelang.v2;
-
-import static bjc.dicelang.v2.Errors.ErrorKey.*;
-import static bjc.dicelang.v2.Token.Type.*;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import bjc.dicelang.v2.dice.DiceBox;
-import bjc.utils.funcdata.FunctionalMap;
-import bjc.utils.funcdata.IMap;
-import bjc.utils.funcutils.StringUtils;
-
-public class Tokenizer {
-	// Literal tokens for tokenization
-	private IMap<String, Token.Type> litTokens;
-	
-	private DiceLangEngine eng;
-
-	private int nextSym = 0;
-	
-	public Tokenizer(DiceLangEngine engine) {
-		eng = engine;
-		
-		litTokens = new FunctionalMap<>();
-
-		litTokens.put("+",   ADD);
-		litTokens.put("-",   SUBTRACT);
-		litTokens.put("*",   MULTIPLY);
-		litTokens.put("/",   DIVIDE);
-		litTokens.put("//",  IDIVIDE);
-		litTokens.put("dg",  DICEGROUP);
-		litTokens.put("dc",  DICECONCAT);
-		litTokens.put("dl",  DICELIST);
-		litTokens.put("=>",  LET);
-		litTokens.put(":=",  BIND);
-		litTokens.put(",",   GROUPSEP);
-		litTokens.put("crc", COERCE);
-	}
-	
-	public Token lexToken(String token, IMap<String, String> stringLts) {
-		if(token.equals("")) return null;
-
-		Token tk = Token.NIL_TOKEN;
-
-		if(litTokens.containsKey(token)) {
-			tk = new Token(litTokens.get(token));
-		} else {
-			switch(token.charAt(0)) {
-				case '(':
-				case ')':
-				case '[':
-				case ']':
-				case '{':
-				case '}':
-					   tk = tokenizeGrouping(token);
-					   break;
-				default:
-					   tk = tokenizeLiteral(token, stringLts);
-			}
-		}
-
-		return tk;
-	}
-
-	private Token tokenizeGrouping(String token) {
-		Token tk = Token.NIL_TOKEN;
-
-		if(StringUtils.containsOnly(token, "\\" + token.charAt(0))) {
-			switch(token.charAt(0)) {
-				case '(':
-					tk = new Token(OPAREN, token.length());
-					break;
-				case ')':
-					tk = new Token(CPAREN, token.length());
-					break;
-				case '[':
-					tk = new Token(OBRACKET, token.length());
-					break;
-				case ']':
-					tk = new Token(CBRACKET, token.length());
-					break;
-				case '{':
-					tk = new Token(OBRACE, token.length());
-					break;
-				case '}':
-					tk = new Token(CBRACE, token.length());
-					break;
-				default:
-					Errors.inst.printError(EK_TOK_UNGROUP, token);
-					break;
-			}
-		}
-
-		return tk;
-	}
-
-	private Pattern intMatcher          = Pattern.compile("\\A[\\-\\+]?\\d+\\Z");
-	private Pattern hexadecimalMatcher  = Pattern.compile("\\A[\\-\\+]?0x[0-9A-Fa-f]+\\Z");
-	private Pattern flexadecimalMatcher = Pattern.compile("\\A[\\-\\+]?[0-9][0-9A-Za-z]+B\\d{1,2}\\Z");
-	private Pattern stringLitMatcher    = Pattern.compile("\\AstringLiteral(\\d+)\\Z");
-
-	private Token tokenizeLiteral(String token, IMap<String, String> stringLts) {
-		Token tk = Token.NIL_TOKEN;
-
-		if(intMatcher.matcher(token).matches()) {
-			tk = new Token(INT_LIT, Long.parseLong(token));
-		} else if(hexadecimalMatcher.matcher(token).matches()) {
-			String newToken = token.substring(0, 1) + token.substring(token.indexOf('x'));
-
-			tk = new Token(INT_LIT, Long.parseLong(newToken.substring(2).toUpperCase(), 16));
-		} else if(flexadecimalMatcher.matcher(token).matches()) {
-			int parseBase = Integer.parseInt(token.substring(token.lastIndexOf('B') + 1));
-
-			if(parseBase < Character.MIN_RADIX || parseBase > Character.MAX_RADIX) {
-				Errors.inst.printError(EK_TOK_INVBASE, Integer.toString(parseBase));
-				return Token.NIL_TOKEN;
-			}
-
-			String flexNum = token.substring(0, token.lastIndexOf('B'));
-
-			try {
-				tk = new Token(INT_LIT, Long.parseLong(flexNum, parseBase));
-			} catch (NumberFormatException nfex) {
-				Errors.inst.printError(EK_TOK_INVFLEX, flexNum, Integer.toString(parseBase));
-				return Token.NIL_TOKEN;
-			}
-		} else if(DoubleMatcher.floatingLiteral.matcher(token).matches()) {
-			tk = new Token(FLOAT_LIT, Double.parseDouble(token));
-		} else if(DiceBox.isValidExpression(token)) {
-			tk = new Token(DICE_LIT, DiceBox.parseExpression(token));
-		} else {
-			Matcher stringLit = stringLitMatcher.matcher(token);
-
-			if(stringLit.matches()) {
-				int litNum = Integer.parseInt(stringLit.group(1));
-
-				eng.stringLits.put(litNum, stringLts.get(token));
-				tk = new Token(STRING_LIT, litNum);
-			} else {
-				// @TODO define what a valid identifier is
-				eng.symTable.put(nextSym++, token);
-
-				tk = new Token(VREF, nextSym - 1);
-			}
-
-			// @TODO uncomment when we have a defn. for var names
-			// System.out.printf("\tERROR: Unrecognized token:"
-			// 		+ "%s\n", token);
-		}
-
-		return tk;
-	}
-}
-- 
cgit v1.2.3