From f028ea6dc555fc5192a96b00b8e96e90dbf6de55 Mon Sep 17 00:00:00 2001 From: "Benjamin J. Culkin" Date: Mon, 9 Oct 2017 16:02:10 -0300 Subject: TODO tagging --- dice-lang/src/bjc/dicelang/Tokenizer.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'dice-lang/src/bjc/dicelang/Tokenizer.java') diff --git a/dice-lang/src/bjc/dicelang/Tokenizer.java b/dice-lang/src/bjc/dicelang/Tokenizer.java index 4bf0d5d..3e4a490 100644 --- a/dice-lang/src/bjc/dicelang/Tokenizer.java +++ b/dice-lang/src/bjc/dicelang/Tokenizer.java @@ -14,9 +14,11 @@ import bjc.utils.funcdata.IMap; import bjc.utils.funcutils.StringUtils; import bjc.utils.parserutils.TokenUtils; -@SuppressWarnings("javadoc") +/** + * Converts strings into tokens. + */ public class Tokenizer { - // Literal tokens for tokenization + /* Literal tokens for tokenization */ private final IMap litTokens; private final DiceLangEngine eng; @@ -65,7 +67,6 @@ public class Tokenizer { case '}': tk = tokenizeGrouping(token); break; - default: tk = tokenizeLiteral(token, stringLts); } @@ -78,6 +79,7 @@ public class Tokenizer { Token tk = Token.NIL_TOKEN; if (StringUtils.containsOnly(token, "\\" + token.charAt(0))) { + /* Handle multiple-grouped delimiters. */ switch (token.charAt(0)) { case '(': tk = new Token(OPAREN, token.length()); @@ -112,6 +114,7 @@ public class Tokenizer { return tk; } + /* Patterns for matching. */ private final Pattern hexadecimalMatcher = Pattern.compile("\\A[\\-\\+]?0x[0-9A-Fa-f]+\\Z"); private final Pattern flexadecimalMatcher = @@ -119,6 +122,7 @@ public class Tokenizer { private final Pattern stringLitMatcher = Pattern.compile("\\AstringLiteral(\\d+)\\Z"); + /* Tokenize a literal value. */ private Token tokenizeLiteral(final String rtoken, final IMap stringLts) { Token tk = Token.NIL_TOKEN; -- cgit v1.2.3