diff options
Diffstat (limited to 'dice-lang/src/bjc/dicelang/expr/Lexer.java')
| -rw-r--r-- | dice-lang/src/bjc/dicelang/expr/Lexer.java | 25 |
1 files changed, 16 insertions, 9 deletions
diff --git a/dice-lang/src/bjc/dicelang/expr/Lexer.java b/dice-lang/src/bjc/dicelang/expr/Lexer.java index bac866b..dfa0f76 100644 --- a/dice-lang/src/bjc/dicelang/expr/Lexer.java +++ b/dice-lang/src/bjc/dicelang/expr/Lexer.java @@ -6,20 +6,22 @@ import java.util.List; import bjc.utils.funcdata.IList; import bjc.utils.parserutils.splitter.ConfigurableTokenSplitter; +/* + * @TODO 10/08/18 :IntExpressions + * Add support for integer constants, and maybe floating-point ones as well + * if you feel like. Heck, you could even go for ratio constants and things + * as well. + */ /** * Implements the lexer for simple expression operations. * * @author Ben Culkin */ public class Lexer { - /* - * Splitter we use. - */ + /* Splitter we use. */ private final ConfigurableTokenSplitter split; - /** - * Create a new expression lexer. - */ + /** Create a new expression lexer. */ public Lexer() { split = new ConfigurableTokenSplitter(true); @@ -39,14 +41,19 @@ public class Lexer { * @return A series of infix tokens representing the command. */ public Token[] lexString(final String inp, final Tokens tks) { + /* Split tokens on whitespace. */ final String[] spacedTokens = inp.split("[ \t]"); + /* Tokens to return. */ + final List<Token> tokens = new LinkedList<>(); - final List<Token> tokens = new LinkedList<>(); - + /* Process each token. */ for (final String spacedToken : spacedTokens) { + /* Split on operators. */ final IList<String> splitTokens = split.split(spacedToken); - final IList<Token> rawTokens = splitTokens.map(tok -> tks.lexToken(tok, spacedToken)); + /* Convert strings to tokens. */ + final IList<Token> rawTokens = splitTokens.map(tok -> tks.lexToken(tok, spacedToken)); + /* Add tokens to results. */ rawTokens.forEach(tokens::add); } |