diff options
Diffstat (limited to 'JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java')
| -rw-r--r-- | JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java | 32 |
1 files changed, 12 insertions, 20 deletions
diff --git a/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java b/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java index 9b3fa74..4f8da12 100644 --- a/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java +++ b/JPratt/src/examples/java/bjc/pratt/examples/Tokenizer.java @@ -11,7 +11,9 @@ import bjc.pratt.tokens.StringToken; final class Tokenizer implements Function<String, Token<String, String>> { private final Set<String> ops; private final Set<String> reserved; - private final TestContext ctx; + + @SuppressWarnings("unused") + private final TestContext ctx; public Tokenizer(final Set<String> operators, final Set<String> reservedWords, final TestContext context) { ops = operators; @@ -21,26 +23,16 @@ final class Tokenizer implements Function<String, Token<String, String>> { @Override public Token<String, String> apply(final String strang) { - if (ops.contains(strang) || reserved.contains(strang)) + if (ops.contains(strang) || reserved.contains(strang)) { return litToken(strang); - else if (ctx.scopes.top().containsKey(strang)) - return new StringToken("(vref)", strang); - else if (strang.matches("(?:[\\u00B2\\u00B3\\u00B9\\u2070]|[\\u2074-\\u2079])+")) /* - * This - * regular - * expression - * matches - * series - * of - * unicode - * super - * - - * scripts - * 1 - * - - * 9. - */ + } else if (strang.matches("(?:[\\u00B2\\u00B3\\u00B9\\u2070]|[\\u2074-\\u2079])+")) { + /* + * This regular expression matches series of unicode + * super - scripts 1 - 9. + */ return new StringToken("(superexp)", strang); - else return new StringToken("(literal)", strang); + } else { + return new StringToken("(literal)", strang); + } } } |