diff options
| author | bjculkin <bjculkin@WIT-136XG42.wvu-ad.wvu.edu> | 2017-02-27 10:48:18 -0500 |
|---|---|---|
| committer | bjculkin <bjculkin@WIT-136XG42.wvu-ad.wvu.edu> | 2017-02-27 10:48:18 -0500 |
| commit | e36e3b56b9b736d7a5caa889f280f90688eb9ad7 (patch) | |
| tree | 5c20e9f79b44e97cb30d3c324ebe60311aea230c /dice-lang/src/bjc/dicelang/v2/Tokenizer.java | |
| parent | 96e868388c7617136e916c27ddd90694c0bb8356 (diff) | |
Seperate tokenizer
Diffstat (limited to 'dice-lang/src/bjc/dicelang/v2/Tokenizer.java')
| -rw-r--r-- | dice-lang/src/bjc/dicelang/v2/Tokenizer.java | 154 |
1 files changed, 154 insertions, 0 deletions
diff --git a/dice-lang/src/bjc/dicelang/v2/Tokenizer.java b/dice-lang/src/bjc/dicelang/v2/Tokenizer.java new file mode 100644 index 0000000..40b1285 --- /dev/null +++ b/dice-lang/src/bjc/dicelang/v2/Tokenizer.java @@ -0,0 +1,154 @@ +package bjc.dicelang.v2;
+
+import static bjc.dicelang.v2.Errors.ErrorKey.*;
+import static bjc.dicelang.v2.Token.Type.*;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import bjc.dicelang.v2.dice.DiceBox;
+import bjc.utils.funcdata.FunctionalMap;
+import bjc.utils.funcdata.IMap;
+import bjc.utils.funcutils.StringUtils;
+
+public class Tokenizer {
+ // Literal tokens for tokenization
+ private IMap<String, Token.Type> litTokens;
+
+ private DiceLangEngine eng;
+
+ private int nextSym = 0;
+
+ public Tokenizer(DiceLangEngine engine) {
+ eng = engine;
+
+ litTokens = new FunctionalMap<>();
+
+ litTokens.put("+", ADD);
+ litTokens.put("-", SUBTRACT);
+ litTokens.put("*", MULTIPLY);
+ litTokens.put("/", DIVIDE);
+ litTokens.put("//", IDIVIDE);
+ litTokens.put("dg", DICEGROUP);
+ litTokens.put("dc", DICECONCAT);
+ litTokens.put("dl", DICELIST);
+ litTokens.put("=>", LET);
+ litTokens.put(":=", BIND);
+ litTokens.put(",", GROUPSEP);
+ litTokens.put("crc", COERCE);
+ }
+
+ public Token lexToken(String token, IMap<String, String> stringLts) {
+ if(token.equals("")) return null;
+
+ Token tk = Token.NIL_TOKEN;
+
+ if(litTokens.containsKey(token)) {
+ tk = new Token(litTokens.get(token));
+ } else {
+ switch(token.charAt(0)) {
+ case '(':
+ case ')':
+ case '[':
+ case ']':
+ case '{':
+ case '}':
+ tk = tokenizeGrouping(token);
+ break;
+ default:
+ tk = tokenizeLiteral(token, stringLts);
+ }
+ }
+
+ return tk;
+ }
+
+ private Token tokenizeGrouping(String token) {
+ Token tk = Token.NIL_TOKEN;
+
+ if(StringUtils.containsOnly(token, "\\" + token.charAt(0))) {
+ switch(token.charAt(0)) {
+ case '(':
+ tk = new Token(OPAREN, token.length());
+ break;
+ case ')':
+ tk = new Token(CPAREN, token.length());
+ break;
+ case '[':
+ tk = new Token(OBRACKET, token.length());
+ break;
+ case ']':
+ tk = new Token(CBRACKET, token.length());
+ break;
+ case '{':
+ tk = new Token(OBRACE, token.length());
+ break;
+ case '}':
+ tk = new Token(CBRACE, token.length());
+ break;
+ default:
+ Errors.inst.printError(EK_TOK_UNGROUP, token);
+ break;
+ }
+ }
+
+ return tk;
+ }
+
+ private Pattern intMatcher = Pattern.compile("\\A[\\-\\+]?\\d+\\Z");
+ private Pattern hexadecimalMatcher = Pattern.compile("\\A[\\-\\+]?0x[0-9A-Fa-f]+\\Z");
+ private Pattern flexadecimalMatcher = Pattern.compile("\\A[\\-\\+]?[0-9][0-9A-Za-z]+B\\d{1,2}\\Z");
+ private Pattern stringLitMatcher = Pattern.compile("\\AstringLiteral(\\d+)\\Z");
+
+ private Token tokenizeLiteral(String token, IMap<String, String> stringLts) {
+ Token tk = Token.NIL_TOKEN;
+
+ if(intMatcher.matcher(token).matches()) {
+ tk = new Token(INT_LIT, Long.parseLong(token));
+ } else if(hexadecimalMatcher.matcher(token).matches()) {
+ String newToken = token.substring(0, 1) + token.substring(token.indexOf('x'));
+
+ tk = new Token(INT_LIT, Long.parseLong(newToken.substring(2).toUpperCase(), 16));
+ } else if(flexadecimalMatcher.matcher(token).matches()) {
+ int parseBase = Integer.parseInt(token.substring(token.lastIndexOf('B') + 1));
+
+ if(parseBase < Character.MIN_RADIX || parseBase > Character.MAX_RADIX) {
+ Errors.inst.printError(EK_TOK_INVBASE, Integer.toString(parseBase));
+ return Token.NIL_TOKEN;
+ }
+
+ String flexNum = token.substring(0, token.lastIndexOf('B'));
+
+ try {
+ tk = new Token(INT_LIT, Long.parseLong(flexNum, parseBase));
+ } catch (NumberFormatException nfex) {
+ Errors.inst.printError(EK_TOK_INVFLEX, flexNum, Integer.toString(parseBase));
+ return Token.NIL_TOKEN;
+ }
+ } else if(DoubleMatcher.floatingLiteral.matcher(token).matches()) {
+ tk = new Token(FLOAT_LIT, Double.parseDouble(token));
+ } else if(DiceBox.isValidExpression(token)) {
+ tk = new Token(DICE_LIT, DiceBox.parseExpression(token));
+ } else {
+ Matcher stringLit = stringLitMatcher.matcher(token);
+
+ if(stringLit.matches()) {
+ int litNum = Integer.parseInt(stringLit.group(1));
+
+ eng.stringLits.put(litNum, stringLts.get(token));
+ tk = new Token(STRING_LIT, litNum);
+ } else {
+ // @TODO define what a valid identifier is
+ eng.symTable.put(nextSym++, token);
+
+ tk = new Token(VREF, nextSym - 1);
+ }
+
+ // @TODO uncomment when we have a defn. for var names
+ // System.out.printf("\tERROR: Unrecognized token:"
+ // + "%s\n", token);
+ }
+
+ return tk;
+ }
+}
|