blob: 3147463ada8b88919c771bcf9fce8ebb0b0b2c8b (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
package bjc.dicelang.expr;
import java.util.LinkedList;
import java.util.List;
import bjc.utils.funcdata.IList;
import bjc.utils.parserutils.splitter.ConfigurableTokenSplitter;
/**
* Implements the lexer for simple expression operations.
*
* @author Ben Culkin
*/
public class Lexer {
/*
* Splitter we use.
*/
private ConfigurableTokenSplitter split;
/**
* Create a new expression lexer.
*/
public Lexer() {
split = new ConfigurableTokenSplitter(true);
split.addSimpleDelimiters("(", ")");
split.addSimpleDelimiters("+", "-", "*", "/");
}
/**
* Convert a string from a input command to a series of infix tokens.
*
* @param inp
* The input command.
*
* @param tks
* The token state.
*
* @return A series of infix tokens representing the command.
*/
public Token[] lexString(String inp, Tokens tks) {
String[] spacedTokens = inp.split("[ \t]");
List<Token> tokens = new LinkedList<>();
for(String spacedToken : spacedTokens) {
IList<String> splitTokens = split.split(spacedToken);
IList<Token> rawTokens = splitTokens.map(tok -> tks.lexToken(tok, spacedToken));
rawTokens.forEach(tokens::add);
}
return tokens.toArray(new Token[0]);
}
}
|
