diff options
8 files changed, 278 insertions, 42 deletions
diff --git a/base/src/bjc/dicelang/Tokenizer.java b/base/src/bjc/dicelang/Tokenizer.java index cd732ee..66a1505 100644 --- a/base/src/bjc/dicelang/Tokenizer.java +++ b/base/src/bjc/dicelang/Tokenizer.java @@ -47,21 +47,37 @@ public class Tokenizer { litTokens.put("crc", COERCE); } + /** + * Create a new tokenizer. + * + * @param engine + * The engine to use. + */ public Tokenizer(final DiceLangEngine engine) { eng = engine; } + /** + * Lex a token. + * + * @param token + * The string to lex. + * @param stringLts + * The set of string literals. + * + * @return A lexed token. + */ public Token lexToken(final String token, final IMap<String, String> stringLts) { - if(token.equals("")) { + if (token.equals("")) { return null; } Token tk = Token.NIL_TOKEN; - if(litTokens.containsKey(token)) { + if (litTokens.containsKey(token)) { tk = new Token(litTokens.get(token)); } else { - switch(token.charAt(0)) { + switch (token.charAt(0)) { case '(': case ')': case '[': @@ -81,9 +97,9 @@ public class Tokenizer { private static Token tokenizeGrouping(final String token) { Token tk = Token.NIL_TOKEN; - if(StringUtils.containsOnly(token, "\\" + token.charAt(0))) { + if (StringUtils.containsOnly(token, "\\" + token.charAt(0))) { /* Handle multiple-grouped delimiters. */ - switch(token.charAt(0)) { + switch (token.charAt(0)) { case '(': tk = new Token(OPAREN, token.length()); break; @@ -128,15 +144,15 @@ public class Tokenizer { String token = rtoken.trim(); - if(TokenUtils.isInt(token)) { + if (TokenUtils.isInt(token)) { tk = new Token(INT_LIT, Long.parseLong(token)); - } else if(hexadecimalMatcher.matcher(token).matches()) { + } else if (hexadecimalMatcher.matcher(token).matches()) { final String newToken = token.substring(0, 1) + token.substring(token.indexOf('x')); tk = new Token(INT_LIT, Long.parseLong(newToken.substring(2).toUpperCase(), 16)); - } else if(flexadecimalMatcher.matcher(token).matches()) { + } else if (flexadecimalMatcher.matcher(token).matches()) { final int parseBase = Integer.parseInt(token.substring(token.lastIndexOf('B') + 1)); - if(parseBase < Character.MIN_RADIX || parseBase > Character.MAX_RADIX) { + if (parseBase < Character.MIN_RADIX || parseBase > Character.MAX_RADIX) { Errors.inst.printError(EK_TOK_INVBASE, Integer.toString(parseBase)); return Token.NIL_TOKEN; } @@ -145,18 +161,18 @@ public class Tokenizer { try { tk = new Token(INT_LIT, Long.parseLong(flexNum, parseBase)); - } catch(final NumberFormatException nfex) { + } catch (final NumberFormatException nfex) { Errors.inst.printError(EK_TOK_INVFLEX, flexNum, Integer.toString(parseBase)); return Token.NIL_TOKEN; } - } else if(TokenUtils.isDouble(token)) { + } else if (TokenUtils.isDouble(token)) { tk = new FloatToken(Double.parseDouble(token)); - } else if(DiceBox.isValidExpression(token)) { + } else if (DiceBox.isValidExpression(token)) { tk = new DiceToken(DiceBox.parseExpression(token)); } else { final Matcher stringLit = stringLitMatcher.matcher(token); - if(stringLit.matches()) { + if (stringLit.matches()) { final int litNum = Integer.parseInt(stringLit.group(1)); eng.addStringLiteral(litNum, stringLts.get(token)); diff --git a/scl/src/main/java/bjc/dicelang/scl/tokens/SCLToken.java b/scl/src/main/java/bjc/dicelang/scl/tokens/SCLToken.java index 127756b..659f545 100644 --- a/scl/src/main/java/bjc/dicelang/scl/tokens/SCLToken.java +++ b/scl/src/main/java/bjc/dicelang/scl/tokens/SCLToken.java @@ -17,16 +17,78 @@ import static bjc.dicelang.scl.tokens.SCLToken.Type.*; */ public class SCLToken { + /** + * Represent all the types of a token. + * + * @author student + * + */ public static enum Type { /* Natural tokens. These come directly from strings */ - ILIT, FLIT, BLIT, SQUOTE, DQUOTE, OBRACKET, OBRACE, SYMBOL, WORD, + /** + * Integer literal. + */ + ILIT, + /** + * Floating-point literal. + */ + FLIT, + /** + * Boolean literal. + */ + BLIT, + /** + * Single-quote. + */ + SQUOTE, + /** + * Double-quote. + */ + DQUOTE, + /** + * Open-bracket. + */ + OBRACKET, + /** + * Open-brace. + */ + OBRACE, + /** + * Symbol. + */ + SYMBOL, + /** + * Word. + */ + WORD, /* Synthetic tokens. These are produced from special tokens. */ - SLIT, WORDS, ARRAY, + /** + * String literal. + */ + SLIT, + /** + * List of words. + */ + WORDS, + /** + * List of data. + */ + ARRAY, } + /** + * The type of the token. + */ public SCLToken.Type type; + /** + * Convert a string into a token. + * + * @param token + * The string to convert into a token. + * @return The token. + */ public static SCLToken tokenizeString(final String token) { if (litTokens.containsKey(token)) { return new SCLToken(litTokens.get(token)); diff --git a/scl/src/main/java/bjc/dicelang/scl/tokens/StringLitSCLToken.java b/scl/src/main/java/bjc/dicelang/scl/tokens/StringLitSCLToken.java index 85c6cb0..d2a10f9 100644 --- a/scl/src/main/java/bjc/dicelang/scl/tokens/StringLitSCLToken.java +++ b/scl/src/main/java/bjc/dicelang/scl/tokens/StringLitSCLToken.java @@ -1,7 +1,19 @@ package bjc.dicelang.scl.tokens; +/** + * Represents a literal string token. + * + * @author student + * + */ public class StringLitSCLToken extends StringSCLToken { + /** + * Create a new literal string token. + * + * @param val + * The string value of the token. + */ public StringLitSCLToken(String val) { super(false, val); } diff --git a/scl/src/main/java/bjc/dicelang/scl/tokens/StringSCLToken.java b/scl/src/main/java/bjc/dicelang/scl/tokens/StringSCLToken.java index d4ed8ec..d026bdb 100644 --- a/scl/src/main/java/bjc/dicelang/scl/tokens/StringSCLToken.java +++ b/scl/src/main/java/bjc/dicelang/scl/tokens/StringSCLToken.java @@ -1,11 +1,19 @@ package bjc.dicelang.scl.tokens; -public class StringSCLToken extends SCLToken { - /* Used for SYMBOL & SLIT */ +/** + * Base class for tokens containing strings. + * + * @author student + * + */ +public abstract class StringSCLToken extends SCLToken { + /** + * String value of the token. + */ public String stringVal; protected StringSCLToken(boolean isSymbol, String val) { - if(isSymbol) { + if (isSymbol) { type = Type.SYMBOL; } else { type = Type.SLIT; @@ -24,13 +32,18 @@ public class StringSCLToken extends SCLToken { @Override public boolean equals(Object obj) { - if(this == obj) return true; - if(!super.equals(obj)) return false; - if(getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (!super.equals(obj)) + return false; + if (getClass() != obj.getClass()) + return false; StringSCLToken other = (StringSCLToken) obj; - if(stringVal == null) { - if(other.stringVal != null) return false; - } else if(!stringVal.equals(other.stringVal)) return false; + if (stringVal == null) { + if (other.stringVal != null) + return false; + } else if (!stringVal.equals(other.stringVal)) + return false; return true; } diff --git a/scl/src/main/java/bjc/dicelang/scl/tokens/SymbolSCLToken.java b/scl/src/main/java/bjc/dicelang/scl/tokens/SymbolSCLToken.java index c9ba43a..d67be78 100644 --- a/scl/src/main/java/bjc/dicelang/scl/tokens/SymbolSCLToken.java +++ b/scl/src/main/java/bjc/dicelang/scl/tokens/SymbolSCLToken.java @@ -1,7 +1,19 @@ package bjc.dicelang.scl.tokens; +/** + * Represents a symbol literal. + * + * @author student + * + */ public class SymbolSCLToken extends StringSCLToken { + /** + * Create a symbol literal + * + * @param val + * The value of the symbol. + */ public SymbolSCLToken(String val) { super(true, val); } diff --git a/scl/src/main/java/bjc/dicelang/scl/tokens/WordListSCLToken.java b/scl/src/main/java/bjc/dicelang/scl/tokens/WordListSCLToken.java index 8463b66..d5bf537 100644 --- a/scl/src/main/java/bjc/dicelang/scl/tokens/WordListSCLToken.java +++ b/scl/src/main/java/bjc/dicelang/scl/tokens/WordListSCLToken.java @@ -2,12 +2,20 @@ package bjc.dicelang.scl.tokens; import bjc.utils.funcdata.IList; -public class WordListSCLToken extends SCLToken { - /* Used for WORDS & ARRAY */ +/** + * Represents a list of words. + * + * @author student + * + */ +public abstract class WordListSCLToken extends SCLToken { + /** + * The list of words. + */ public IList<SCLToken> tokenVals; protected WordListSCLToken(boolean isArray, IList<SCLToken> tokens) { - if(isArray) { + if (isArray) { type = Type.ARRAY; } else { type = Type.WORDS; @@ -26,13 +34,18 @@ public class WordListSCLToken extends SCLToken { @Override public boolean equals(Object obj) { - if(this == obj) return true; - if(!super.equals(obj)) return false; - if(getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (!super.equals(obj)) + return false; + if (getClass() != obj.getClass()) + return false; WordListSCLToken other = (WordListSCLToken) obj; - if(tokenVals == null) { - if(other.tokenVals != null) return false; - } else if(!tokenVals.equals(other.tokenVals)) return false; + if (tokenVals == null) { + if (other.tokenVals != null) + return false; + } else if (!tokenVals.equals(other.tokenVals)) + return false; return true; } diff --git a/scl/src/main/java/bjc/dicelang/scl/tokens/WordSCLToken.java b/scl/src/main/java/bjc/dicelang/scl/tokens/WordSCLToken.java index 5c3c8cb..ae185de 100644 --- a/scl/src/main/java/bjc/dicelang/scl/tokens/WordSCLToken.java +++ b/scl/src/main/java/bjc/dicelang/scl/tokens/WordSCLToken.java @@ -5,22 +5,104 @@ import static bjc.dicelang.scl.tokens.WordSCLToken.Word.*; import java.util.HashMap; import java.util.Map; +/** + * Represents a single word. + * + * @author student + * + */ public class WordSCLToken extends SCLToken { + /** + * Represents the word type. + * + * @author student + * + */ public static enum Word { /* Array manipulation */ - MAKEARRAY, MAKEEXEC, MAKEUNEXEC, + /** + * Create an array + */ + MAKEARRAY, + /** + * Make a token executable. + */ + MAKEEXEC, + /** + * Make a token unexecutable. + */ + MAKEUNEXEC, + /* Stream manipulation */ - NEWSTREAM, LEFTSTREAM, RIGHTSTREAM, DELETESTREAM, MERGESTREAM, + /** + * Create a new stream. + */ + NEWSTREAM, + /** + * Swap to the left stream. + */ + LEFTSTREAM, + /** + * Swap to the right stream. + */ + RIGHTSTREAM, + /** + * Delete the current stream. + */ + DELETESTREAM, + /** + * Merge the streams. + */ + MERGESTREAM, + /* Stack manipulation */ - STACKCOUNT, STACKEMPTY, DROP, NDROP, NIP, NNIP, + /** + * Get the count of items on the stack. + */ + STACKCOUNT, + /** + * Check if the stack is empty. + */ + STACKEMPTY, + /** + * Drop an item from the top of the stack. + */ + DROP, + /** + * Drop a number of items from the top of the stack. + */ + NDROP, + /** + * Drop an item, leaving the top of the stack alone. + */ + NIP, + /** + * Drop a number of items, leaving the top of the stack alone. + */ + NNIP, } + /** + * The value of the word. + */ public Word wordVal; + /** + * Create a new word token. + * + * @param wrd + * The value of the word. + */ public WordSCLToken(String wrd) { this(builtinWords.get(wrd)); } + /** + * Create a new word token. + * + * @param wrd + * The value of the word. + */ public WordSCLToken(Word wrd) { super(Type.WORD); @@ -42,14 +124,26 @@ public class WordSCLToken extends SCLToken { @Override public boolean equals(Object obj) { - if(this == obj) return true; - if(!super.equals(obj)) return false; - if(getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (!super.equals(obj)) + return false; + if (getClass() != obj.getClass()) + return false; WordSCLToken other = (WordSCLToken) obj; - if(wordVal != other.wordVal) return false; + if (wordVal != other.wordVal) + return false; return true; } + /** + * Check if a word is built-in. + * + * @param wrd + * The word to check. + * + * @return Whether or not the word is builtin. + */ public static boolean isBuiltinWord(String wrd) { return builtinWords.containsKey(wrd); } @@ -61,13 +155,15 @@ public class WordSCLToken extends SCLToken { builtinWords = new HashMap<>(); builtinWords.put("makearray", MAKEARRAY); + builtinWords.put("cvx", MAKEEXEC); + builtinWords.put("cvux", MAKEUNEXEC); + builtinWords.put("+stream", NEWSTREAM); builtinWords.put(">stream", LEFTSTREAM); builtinWords.put("<stream", RIGHTSTREAM); builtinWords.put("-stream", DELETESTREAM); builtinWords.put("<-stream", MERGESTREAM); - builtinWords.put("cvx", MAKEEXEC); - builtinWords.put("cvux", MAKEUNEXEC); + builtinWords.put("#", STACKCOUNT); builtinWords.put("empty?", STACKEMPTY); builtinWords.put("drop", DROP); diff --git a/scl/src/main/java/bjc/dicelang/scl/tokens/WordsSCLToken.java b/scl/src/main/java/bjc/dicelang/scl/tokens/WordsSCLToken.java index 02d720c..40c4cd4 100644 --- a/scl/src/main/java/bjc/dicelang/scl/tokens/WordsSCLToken.java +++ b/scl/src/main/java/bjc/dicelang/scl/tokens/WordsSCLToken.java @@ -2,8 +2,20 @@ package bjc.dicelang.scl.tokens; import bjc.utils.funcdata.IList; +/** + * A token representing an executable bunch of words. + * + * @author student + * + */ public class WordsSCLToken extends WordListSCLToken { + /** + * Create a new executable words token. + * + * @param tokens + * The tokens to use. + */ public WordsSCLToken(IList<SCLToken> tokens) { super(false, tokens); } |