1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
|
package bjc.dicelang.expr;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
/**
* Contains per-instance state for token parsing.
*
* @author EVE
*
*/
public class Tokens {
/* Contains mappings from variable references to string names. */
private final Map<Integer, String> symTab;
/* Reverse index into the symbol table. */
private final Map<String, Integer> revSymTab;
/** Read-only view on the symbol table. */
public final Map<Integer, String> symbolTable;
/* Next index into the symbol table. */
private int nextSym;
/* Mapping from literal tokens to token types. */
private static final Map<String, TokenType> litTokens;
static {
/*
* Setup literal mappings.
*
* @NOTE Should this be a static member?
*/
litTokens = new HashMap<>();
litTokens.put("+", TokenType.ADD);
litTokens.put("-", TokenType.SUBTRACT);
litTokens.put("*", TokenType.MULTIPLY);
litTokens.put("/", TokenType.DIVIDE);
litTokens.put("(", TokenType.OPAREN);
litTokens.put(")", TokenType.CPAREN);
}
/** Create a new set of tokens. */
public Tokens() {
/* Create tables. */
symTab = new HashMap<>();
revSymTab = new HashMap<>();
/* Init public view. */
symbolTable = Collections.unmodifiableMap(symTab);
/* Set sym ID. */
nextSym = 0;
}
/**
* Convert the string representation of a token into a token.
*
* @param tok
* The string representation of the token.
* @param raw
* The original string the token came from.
*
* @return The token the string represents.
*/
public Token lexToken(final String tok, final String raw) {
if (litTokens.containsKey(tok)) {
/* Return matching literal token. */
return new Token(litTokens.get(tok), raw, this);
}
/* Its a variable reference. */
return parseVRef(tok, raw);
}
/* Parse a variable reference. */
private Token parseVRef(final String tok, final String raw) {
final Token tk = new Token(TokenType.VREF, raw, this);
if (revSymTab.containsKey(tok)) {
/* Reuse the entry if it exists. */
tk.intValue = revSymTab.get(tok);
} else {
/* Create a new entry. */
tk.intValue = nextSym;
/* Record it. */
symTab.put(nextSym, tok);
revSymTab.put(tok, nextSym);
/* Next ID. */
nextSym += 1;
}
return tk;
}
}
|
