diff options
| author | Benjamin J. Culkin <bjculkin@mix.wvu.edu> | 2017-07-23 22:58:58 -0300 |
|---|---|---|
| committer | Benjamin J. Culkin <bjculkin@mix.wvu.edu> | 2017-07-23 22:58:58 -0300 |
| commit | c74cb8eaf454d9243b75e3770001d8d128356b28 (patch) | |
| tree | 899197afc0b7c382aa37c778053357e0e9b56c26 /dice-lang/src/bjc/dicelang/Tokenizer.java | |
| parent | 57f9a3bfdad20bead5b35ee540e8790e80a6b9a4 (diff) | |
Format
Diffstat (limited to 'dice-lang/src/bjc/dicelang/Tokenizer.java')
| -rw-r--r-- | dice-lang/src/bjc/dicelang/Tokenizer.java | 16 |
1 files changed, 13 insertions, 3 deletions
diff --git a/dice-lang/src/bjc/dicelang/Tokenizer.java b/dice-lang/src/bjc/dicelang/Tokenizer.java index 3ca9a94..259661d 100644 --- a/dice-lang/src/bjc/dicelang/Tokenizer.java +++ b/dice-lang/src/bjc/dicelang/Tokenizer.java @@ -89,6 +89,7 @@ public class Tokenizer { case '}': tk = tokenizeGrouping(token); break; + default: tk = tokenizeLiteral(token, stringLts); } @@ -105,21 +106,27 @@ public class Tokenizer { case '(': tk = new Token(OPAREN, token.length()); break; + case ')': tk = new Token(CPAREN, token.length()); break; + case '[': tk = new Token(OBRACKET, token.length()); break; + case ']': tk = new Token(CBRACKET, token.length()); break; + case '{': tk = new Token(OBRACE, token.length()); break; + case '}': tk = new Token(CBRACE, token.length()); break; + default: Errors.inst.printError(EK_TOK_UNGROUP, token); break; @@ -129,9 +136,12 @@ public class Tokenizer { return tk; } - private final Pattern hexadecimalMatcher = Pattern.compile("\\A[\\-\\+]?0x[0-9A-Fa-f]+\\Z"); - private final Pattern flexadecimalMatcher = Pattern.compile("\\A[\\-\\+]?[0-9][0-9A-Za-z]+B\\d{1,2}\\Z"); - private final Pattern stringLitMatcher = Pattern.compile("\\AstringLiteral(\\d+)\\Z"); + private final Pattern hexadecimalMatcher = + Pattern.compile("\\A[\\-\\+]?0x[0-9A-Fa-f]+\\Z"); + private final Pattern flexadecimalMatcher = + Pattern.compile("\\A[\\-\\+]?[0-9][0-9A-Za-z]+B\\d{1,2}\\Z"); + private final Pattern stringLitMatcher = + Pattern.compile("\\AstringLiteral(\\d+)\\Z"); private Token tokenizeLiteral(final String token, final IMap<String, String> stringLts) { Token tk = Token.NIL_TOKEN; |
