diff options
Diffstat (limited to 'JPratt/src/main/java/com/ashardalon/pratt/tokens')
7 files changed, 468 insertions, 0 deletions
diff --git a/JPratt/src/main/java/com/ashardalon/pratt/tokens/ExpectionNotMet.java b/JPratt/src/main/java/com/ashardalon/pratt/tokens/ExpectionNotMet.java new file mode 100644 index 0000000..b700203 --- /dev/null +++ b/JPratt/src/main/java/com/ashardalon/pratt/tokens/ExpectionNotMet.java @@ -0,0 +1,23 @@ +package com.ashardalon.pratt.tokens; + +import bjc.utils.parserutils.ParserException; + +/** + * The exception thrown when an expectation fails. + * + * @author EVE + * + */ +public class ExpectionNotMet extends ParserException { + private static final long serialVersionUID = 4299299480127680805L; + + /** + * Create a new exception with the specified message. + * + * @param msg + * The message of the exception. + */ + public ExpectionNotMet(final String msg) { + super(msg); + } +}
\ No newline at end of file diff --git a/JPratt/src/main/java/com/ashardalon/pratt/tokens/SimpleToken.java b/JPratt/src/main/java/com/ashardalon/pratt/tokens/SimpleToken.java new file mode 100644 index 0000000..ee68835 --- /dev/null +++ b/JPratt/src/main/java/com/ashardalon/pratt/tokens/SimpleToken.java @@ -0,0 +1,60 @@ +package com.ashardalon.pratt.tokens; + +import java.util.Objects; + +/** + * Simple token implementation + * + * @author bjcul + * + * @param <K> The key type + * @param <V> The value type + * + */ +public class SimpleToken<K, V> implements Token<K, V> { + private K key; + private V value; + + /** + * Create a new token + * @param key The key + * @param value The value + */ + public SimpleToken(K key, V value) { + super(); + this.key = key; + this.value = value; + } + + @Override + public K getKey() { + return key; + } + + @Override + public V getValue() { + return value; + } + + @Override + public int hashCode() { + return Objects.hash(key, value); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + SimpleToken<?, ?> other = (SimpleToken<?, ?>) obj; + return Objects.equals(key, other.key) && Objects.equals(value, other.value); + } + + @Override + public String toString() { + return "SimpleToken [key=" + key + ", value=" + value + "]"; + } +} diff --git a/JPratt/src/main/java/com/ashardalon/pratt/tokens/SimpleTokenStream.java b/JPratt/src/main/java/com/ashardalon/pratt/tokens/SimpleTokenStream.java new file mode 100644 index 0000000..cc751d1 --- /dev/null +++ b/JPratt/src/main/java/com/ashardalon/pratt/tokens/SimpleTokenStream.java @@ -0,0 +1,82 @@ +package com.ashardalon.pratt.tokens; + +import java.util.Iterator; + +import bjc.data.MarkListIterator; + +/** + * Simple token stream implementation + * @author bjcul + * + * @param <K> The key type + * @param <V> The value type + */ +public class SimpleTokenStream<K, V> extends TokenStream<K, V> { + private final MarkListIterator<Token<K, V>> iter; + + private Token<K, V> curr; + + private Token<K, V> terminal; + /** + * Create a new token stream from a iterator. + * + * @param itr The iterator to use. + * @param terminal The terminal token to use for end-of-stream + * + */ + public SimpleTokenStream(final Iterator<Token<K, V>> itr, Token<K, V> terminal) { + this.iter = new MarkListIterator<>(itr); + this.terminal = terminal; + } + + @Override + public Token<K, V> current() { + // Prime stream if necessary + if (curr == null) + return next(); + return curr; + } + + @Override + public Token<K, V> next() { + if (iter.hasNext()) { + curr = iter.next(); + } else { + curr = terminal; + } + + return curr; + } + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public void mark() { + iter.mark(); + } + + @Override + public void commit() { + iter.commit(); + + if (!iter.hasMark()) { + // No marks outstanding; we can release the previous state + iter.reset(); + } + } + + @Override + public void rollback() { + iter.rollback(); + + curr = iter.current(); + } + + @Override + public boolean hasMark() { + return iter.hasMark(); + } +} diff --git a/JPratt/src/main/java/com/ashardalon/pratt/tokens/StringToken.java b/JPratt/src/main/java/com/ashardalon/pratt/tokens/StringToken.java new file mode 100644 index 0000000..e9b17dc --- /dev/null +++ b/JPratt/src/main/java/com/ashardalon/pratt/tokens/StringToken.java @@ -0,0 +1,83 @@ +package com.ashardalon.pratt.tokens; + +/** + * Simple token implementation for strings. + * + * @author EVE + * + */ +public class StringToken implements Token<String, String> { + private final String key; + private final String val; + + /** + * Create a new string token. + * + * @param ky + * The key for the token. + * + * @param vl + * The value for the token. + */ + public StringToken(final String ky, final String vl) { + key = ky; + val = vl; + } + + @Override + public String getKey() { + return key; + } + + @Override + public String getValue() { + return val; + } + + @Override + public int hashCode() { + final int prime = 31; + + int result = 1; + result = prime * result + (key == null ? 0 : key.hashCode()); + result = prime * result + (val == null ? 0 : val.hashCode()); + + return result; + } + + @Override + public boolean equals(final Object obj) { + if(this == obj) return true; + if(obj == null) return false; + if(!(obj instanceof StringToken)) return false; + + final StringToken other = (StringToken) obj; + + if(key == null) { + if(other.key != null) return false; + } else if(!key.equals(other.key)) return false; + + if(val == null) { + if(other.val != null) return false; + } else if(!val.equals(other.val)) return false; + + return true; + } + + @Override + public String toString() { + return String.format("StringToken [key='%s', val='%s']", key, val); + } + + /** + * Create a new literal token (has same key/value). + * + * @param val + * The value for the literal token. + * + * @return A literal token with that key. + */ + public static StringToken litToken(final String val) { + return new StringToken(val, val); + } +} diff --git a/JPratt/src/main/java/com/ashardalon/pratt/tokens/StringTokenStream.java b/JPratt/src/main/java/com/ashardalon/pratt/tokens/StringTokenStream.java new file mode 100644 index 0000000..fb25016 --- /dev/null +++ b/JPratt/src/main/java/com/ashardalon/pratt/tokens/StringTokenStream.java @@ -0,0 +1,84 @@ +package com.ashardalon.pratt.tokens; + +import static com.ashardalon.pratt.tokens.StringToken.litToken; + +import java.util.Iterator; + +import bjc.data.MarkListIterator; + +/** + * Simple implementation of token stream for strings. + * + * The terminal token here is represented by a token with type and value + * '(end)'. + * + * @author EVE + * + */ +public class StringTokenStream extends TokenStream<String, String> { + private final MarkListIterator<Token<String, String>> iter; + + private Token<String, String> curr; + + /** + * Create a new token stream from a iterator. + * + * @param itr + * The iterator to use. + * + */ + public StringTokenStream(final Iterator<Token<String, String>> itr) { + iter = new MarkListIterator<>(itr); + } + + @Override + public Token<String, String> current() { + // Prime stream if necessary + if (curr == null) + return next(); + return curr; + } + + @Override + public Token<String, String> next() { + if(iter.hasNext()) { + curr = iter.next(); + } else { + curr = litToken("(end)"); + } + + return curr; + } + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public void mark() { + iter.mark(); + } + + @Override + public void commit() { + iter.commit(); + + if (!iter.hasMark()) { + // No marks outstanding; we can release the previous state + iter.reset(); + } + } + + @Override + public void rollback() { + iter.rollback(); + + curr = iter.current(); + } + + @Override + public boolean hasMark() { + return iter.hasMark(); + } +} diff --git a/JPratt/src/main/java/com/ashardalon/pratt/tokens/Token.java b/JPratt/src/main/java/com/ashardalon/pratt/tokens/Token.java new file mode 100644 index 0000000..86f0d05 --- /dev/null +++ b/JPratt/src/main/java/com/ashardalon/pratt/tokens/Token.java @@ -0,0 +1,30 @@ +package com.ashardalon.pratt.tokens; + +/** + * Represents a simple parsing token. + * + * @author EVE + * + * @param <K> + * The key type of this token. Represents the type of the token. + * + * @param <V> + * The value type of this token. Represents any additional data for the + * token. + * + */ +public interface Token<K, V> { + /** + * Get the key for this token. + * + * @return The key for this token + */ + K getKey(); + + /** + * Get the value for this token. + * + * @return The value for this token. + */ + V getValue(); +} diff --git a/JPratt/src/main/java/com/ashardalon/pratt/tokens/TokenStream.java b/JPratt/src/main/java/com/ashardalon/pratt/tokens/TokenStream.java new file mode 100644 index 0000000..3f2681b --- /dev/null +++ b/JPratt/src/main/java/com/ashardalon/pratt/tokens/TokenStream.java @@ -0,0 +1,106 @@ +package com.ashardalon.pratt.tokens; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import bjc.utils.funcutils.StringUtils; + +/** + * A stream of tokens. + * + * @author EVE + * + * @param <K> + * The key type of the token. + * + * @param <V> + * The value type of the token. + */ +public abstract class TokenStream<K, V> implements Iterator<Token<K, V>> { + /** + * Get the current token. + * + * @return The current token. + */ + public abstract Token<K, V> current(); + + @Override + public abstract Token<K, V> next(); + + @Override + public abstract boolean hasNext(); + + /** + * Place a mark in the current stream, which can be either returned to or abandoned later on. + */ + public abstract void mark(); + + /** + * Reset the stream to the state it was in when the last mark was taken. + */ + public abstract void rollback(); + + /** + * Check if the stream has at least one mark. + * + * @return Whether or not at least one mark exists. + */ + public abstract boolean hasMark(); + + /** + * Remove the last mark placed into the stream. This prevents returning to it later on. + */ + public abstract void commit(); + + /** + * Utility method for checking that the next token is one of a specific + * set of types, and then consuming it. + * + * @param expectedKeys + * The expected values + * + * @throws ExpectionNotMet + * If the token is not one of the expected types. + */ + public void expect(final Set<K> expectedKeys) throws ExpectionNotMet { + final K curKey = current().getKey(); + + if(!expectedKeys.contains(curKey)) { + final String expectedList = StringUtils.toEnglishList(expectedKeys.toArray(), false); + + throw new ExpectionNotMet("One of '" + expectedList + "' was expected, not " + curKey); + } + + next(); + } + + /** + * Utility method for checking that the next token is one of a specific + * set of types, and then consuming it. + * + * @param expectedKeys + * The expected values + * + * @throws ExpectionNotMet + * If the token is not one of the expected types. + */ + @SafeVarargs + public final void expect(final K... expectedKeys) throws ExpectionNotMet { + HashSet<K> keys = new HashSet<>(Arrays.asList(expectedKeys)); + expect(keys); + } + + /** + * Check whether the head token is a certain type. + * + * @param val + * The type to check for. + * + * @return Whether or not the head token is of that type. + */ + public boolean headIs(final K val) { + return current().getKey().equals(val); + } +} |