From c74cb8eaf454d9243b75e3770001d8d128356b28 Mon Sep 17 00:00:00 2001 From: "Benjamin J. Culkin" Date: Sun, 23 Jul 2017 22:58:58 -0300 Subject: Format --- dice-lang/src/bjc/dicelang/Tokenizer.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'dice-lang/src/bjc/dicelang/Tokenizer.java') diff --git a/dice-lang/src/bjc/dicelang/Tokenizer.java b/dice-lang/src/bjc/dicelang/Tokenizer.java index 3ca9a94..259661d 100644 --- a/dice-lang/src/bjc/dicelang/Tokenizer.java +++ b/dice-lang/src/bjc/dicelang/Tokenizer.java @@ -89,6 +89,7 @@ public class Tokenizer { case '}': tk = tokenizeGrouping(token); break; + default: tk = tokenizeLiteral(token, stringLts); } @@ -105,21 +106,27 @@ public class Tokenizer { case '(': tk = new Token(OPAREN, token.length()); break; + case ')': tk = new Token(CPAREN, token.length()); break; + case '[': tk = new Token(OBRACKET, token.length()); break; + case ']': tk = new Token(CBRACKET, token.length()); break; + case '{': tk = new Token(OBRACE, token.length()); break; + case '}': tk = new Token(CBRACE, token.length()); break; + default: Errors.inst.printError(EK_TOK_UNGROUP, token); break; @@ -129,9 +136,12 @@ public class Tokenizer { return tk; } - private final Pattern hexadecimalMatcher = Pattern.compile("\\A[\\-\\+]?0x[0-9A-Fa-f]+\\Z"); - private final Pattern flexadecimalMatcher = Pattern.compile("\\A[\\-\\+]?[0-9][0-9A-Za-z]+B\\d{1,2}\\Z"); - private final Pattern stringLitMatcher = Pattern.compile("\\AstringLiteral(\\d+)\\Z"); + private final Pattern hexadecimalMatcher = + Pattern.compile("\\A[\\-\\+]?0x[0-9A-Fa-f]+\\Z"); + private final Pattern flexadecimalMatcher = + Pattern.compile("\\A[\\-\\+]?[0-9][0-9A-Za-z]+B\\d{1,2}\\Z"); + private final Pattern stringLitMatcher = + Pattern.compile("\\AstringLiteral(\\d+)\\Z"); private Token tokenizeLiteral(final String token, final IMap stringLts) { Token tk = Token.NIL_TOKEN; -- cgit v1.2.3