diff options
| author | Benjamin J. Culkin <bjculkin@mix.wvu.edu> | 2017-10-09 16:02:10 -0300 |
|---|---|---|
| committer | Benjamin J. Culkin <bjculkin@mix.wvu.edu> | 2017-10-09 16:02:10 -0300 |
| commit | f028ea6dc555fc5192a96b00b8e96e90dbf6de55 (patch) | |
| tree | 4b2a28ecbeb30095b50e6e9e8ac8b98fa8ddc79e /dice-lang/src/bjc/dicelang/Tokenizer.java | |
| parent | be4675f9512060aa85b1e0a4f223208b51b55812 (diff) | |
TODO tagging
Diffstat (limited to 'dice-lang/src/bjc/dicelang/Tokenizer.java')
| -rw-r--r-- | dice-lang/src/bjc/dicelang/Tokenizer.java | 10 |
1 files changed, 7 insertions, 3 deletions
diff --git a/dice-lang/src/bjc/dicelang/Tokenizer.java b/dice-lang/src/bjc/dicelang/Tokenizer.java index 4bf0d5d..3e4a490 100644 --- a/dice-lang/src/bjc/dicelang/Tokenizer.java +++ b/dice-lang/src/bjc/dicelang/Tokenizer.java @@ -14,9 +14,11 @@ import bjc.utils.funcdata.IMap; import bjc.utils.funcutils.StringUtils; import bjc.utils.parserutils.TokenUtils; -@SuppressWarnings("javadoc") +/** + * Converts strings into tokens. + */ public class Tokenizer { - // Literal tokens for tokenization + /* Literal tokens for tokenization */ private final IMap<String, Token.Type> litTokens; private final DiceLangEngine eng; @@ -65,7 +67,6 @@ public class Tokenizer { case '}': tk = tokenizeGrouping(token); break; - default: tk = tokenizeLiteral(token, stringLts); } @@ -78,6 +79,7 @@ public class Tokenizer { Token tk = Token.NIL_TOKEN; if (StringUtils.containsOnly(token, "\\" + token.charAt(0))) { + /* Handle multiple-grouped delimiters. */ switch (token.charAt(0)) { case '(': tk = new Token(OPAREN, token.length()); @@ -112,6 +114,7 @@ public class Tokenizer { return tk; } + /* Patterns for matching. */ private final Pattern hexadecimalMatcher = Pattern.compile("\\A[\\-\\+]?0x[0-9A-Fa-f]+\\Z"); private final Pattern flexadecimalMatcher = @@ -119,6 +122,7 @@ public class Tokenizer { private final Pattern stringLitMatcher = Pattern.compile("\\AstringLiteral(\\d+)\\Z"); + /* Tokenize a literal value. */ private Token tokenizeLiteral(final String rtoken, final IMap<String, String> stringLts) { Token tk = Token.NIL_TOKEN; |