From f028ea6dc555fc5192a96b00b8e96e90dbf6de55 Mon Sep 17 00:00:00 2001 From: "Benjamin J. Culkin" Date: Mon, 9 Oct 2017 16:02:10 -0300 Subject: TODO tagging --- dice-lang/src/bjc/dicelang/expr/Lexer.java | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'dice-lang/src/bjc/dicelang/expr/Lexer.java') diff --git a/dice-lang/src/bjc/dicelang/expr/Lexer.java b/dice-lang/src/bjc/dicelang/expr/Lexer.java index bac866b..dfa0f76 100644 --- a/dice-lang/src/bjc/dicelang/expr/Lexer.java +++ b/dice-lang/src/bjc/dicelang/expr/Lexer.java @@ -6,20 +6,22 @@ import java.util.List; import bjc.utils.funcdata.IList; import bjc.utils.parserutils.splitter.ConfigurableTokenSplitter; +/* + * @TODO 10/08/18 :IntExpressions + * Add support for integer constants, and maybe floating-point ones as well + * if you feel like. Heck, you could even go for ratio constants and things + * as well. + */ /** * Implements the lexer for simple expression operations. * * @author Ben Culkin */ public class Lexer { - /* - * Splitter we use. - */ + /* Splitter we use. */ private final ConfigurableTokenSplitter split; - /** - * Create a new expression lexer. - */ + /** Create a new expression lexer. */ public Lexer() { split = new ConfigurableTokenSplitter(true); @@ -39,14 +41,19 @@ public class Lexer { * @return A series of infix tokens representing the command. */ public Token[] lexString(final String inp, final Tokens tks) { + /* Split tokens on whitespace. */ final String[] spacedTokens = inp.split("[ \t]"); + /* Tokens to return. */ + final List tokens = new LinkedList<>(); - final List tokens = new LinkedList<>(); - + /* Process each token. */ for (final String spacedToken : spacedTokens) { + /* Split on operators. */ final IList splitTokens = split.split(spacedToken); - final IList rawTokens = splitTokens.map(tok -> tks.lexToken(tok, spacedToken)); + /* Convert strings to tokens. */ + final IList rawTokens = splitTokens.map(tok -> tks.lexToken(tok, spacedToken)); + /* Add tokens to results. */ rawTokens.forEach(tokens::add); } -- cgit v1.2.3