1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
|
package bjc.dicelang.expr;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
/**
* Contains per-instance state for token parsing.
*
* @author EVE
*
*/
public class Tokens {
/*
* Contains mappings from variable references to string names.
*/
private final Map<Integer, String> symTab;
/*
* Reverse index into the symbol table.
*/
private final Map<String, Integer> revSymTab;
/**
* Read-only view on the symbol table.
*/
public final Map<Integer, String> symbolTable;
/*
* Next index into the symbol table.
*/
private int nextSym;
/*
* Mapping from literal tokens to token types.
*/
private final Map<String, TokenType> litTokens;
/**
* Create a new set of tokens.
*/
public Tokens() {
symTab = new HashMap<>();
revSymTab = new HashMap<>();
symbolTable = Collections.unmodifiableMap(symTab);
nextSym = 0;
litTokens = new HashMap<>();
litTokens.put("+", TokenType.ADD);
litTokens.put("-", TokenType.SUBTRACT);
litTokens.put("*", TokenType.MULTIPLY);
litTokens.put("/", TokenType.DIVIDE);
litTokens.put("(", TokenType.OPAREN);
litTokens.put(")", TokenType.CPAREN);
}
/**
* Convert the string representation of a token into a token.
*
* @param tok
* The string representation of the token.
* @param raw
* The original string the token came from.
*
* @return The token the string represents.
*/
public Token lexToken(String tok, String raw) {
if(litTokens.containsKey(tok))
return new Token(litTokens.get(tok), raw, this);
else
return parseVRef(tok, raw);
}
/*
* Parse a variable reference.
*/
private Token parseVRef(String tok, String raw) {
Token tk = new Token(TokenType.VREF, raw, this);
if(revSymTab.containsKey(tok)) {
/*
* Reuse the entry if it exists.
*/
tk.intValue = revSymTab.get(tok);
} else {
/*
* Create a new entry.
*/
tk.intValue = nextSym;
symTab.put(nextSym, tok);
revSymTab.put(tok, nextSym);
nextSym += 1;
}
return tk;
}
}
|
