From 848dc739becfa41193aff9a07c918aed91e5ef79 Mon Sep 17 00:00:00 2001 From: bjculkin Date: Fri, 7 Apr 2017 08:56:27 -0400 Subject: Cleanup --- BJC-Utils2/src/main/java/bjc/utils/data/IPair.java | 17 +- .../java/bjc/utils/funcdata/theory/Bifunctor.java | 236 ++++++---- .../java/bjc/utils/parserutils/DoubleMatcher.java | 28 +- .../bjc/utils/parserutils/ParserException.java | 2 +- .../java/bjc/utils/parserutils/ShuntingYard.java | 80 ++-- .../bjc/utils/parserutils/TokenTransformer.java | 44 +- .../java/bjc/utils/parserutils/TokenUtils.java | 522 +++++++++++---------- .../bjc/utils/parserutils/TreeConstructor.java | 57 ++- .../parserutils/splitter/SimpleTokenSplitter.java | 3 + 9 files changed, 540 insertions(+), 449 deletions(-) (limited to 'BJC-Utils2/src/main/java') diff --git a/BJC-Utils2/src/main/java/bjc/utils/data/IPair.java b/BJC-Utils2/src/main/java/bjc/utils/data/IPair.java index 24e0381..2dbd141 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/data/IPair.java +++ b/BJC-Utils2/src/main/java/bjc/utils/data/IPair.java @@ -68,8 +68,7 @@ public interface IPair extends Bifunctor IPair, IPair> combine( IPair otherPair) { - return combine(otherPair, (left, otherLeft) -> new Pair<>(left, otherLeft), - (right, otherRight) -> new Pair<>(right, otherRight)); + return combine(otherPair, Pair::new, Pair::new); } /** @@ -113,8 +112,11 @@ public interface IPair extends Bifunctor Function, Bifunctor> fmapLeft( Function func) { return (argumentPair) -> { - if(!(argumentPair instanceof IPair)) throw new IllegalArgumentException( - "This function can only be applied to instances of IPair"); + if(!(argumentPair instanceof IPair)) { + String msg = "This function can only be applied to instances of IPair"; + + throw new IllegalArgumentException(msg); + } IPair argPair = (IPair) argumentPair; @@ -127,8 +129,11 @@ public interface IPair extends Bifunctor func) { return (argumentPair) -> { - if(!(argumentPair instanceof IPair)) throw new IllegalArgumentException( - "This function can only be applied to instances of IPair"); + if(!(argumentPair instanceof IPair)) { + String msg = "This function can only be applied to instances of IPair"; + + throw new IllegalArgumentException(msg); + } IPair argPair = (IPair) argumentPair; diff --git a/BJC-Utils2/src/main/java/bjc/utils/funcdata/theory/Bifunctor.java b/BJC-Utils2/src/main/java/bjc/utils/funcdata/theory/Bifunctor.java index 8e9bbd2..5380b24 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/funcdata/theory/Bifunctor.java +++ b/BJC-Utils2/src/main/java/bjc/utils/funcdata/theory/Bifunctor.java @@ -1,97 +1,139 @@ -package bjc.utils.funcdata.theory; - -import java.util.function.Function; - -/** - * A functor over a pair of heterogeneous types - * - * @author ben - * @param - * The type stored on the 'left' of the pair - * @param - * The type stored on the 'right' of the pair - * - */ -public interface Bifunctor { - /** - * Lift a pair of functions to a single function that maps over both - * parts of a pair - * - * @param - * The old left type of the pair - * @param - * The old right type of the pair - * @param - * The new left type of the pair - * @param - * The new right type of the pair - * @param leftFunc - * The function that maps over the left of the pair - * @param rightFunc - * The function that maps over the right of the pair - * @return A function that maps over both parts of the pair - */ - public default Function, Bifunctor> bimap( - Function leftFunc, Function rightFunc) { - Function, Bifunctor> bimappedFunc = (argPair) -> { - Function, Bifunctor> leftMapper = argPair - .fmapLeft(leftFunc); - - Bifunctor leftMappedFunctor = leftMapper.apply(argPair); - Function, Bifunctor> rightMapper = leftMappedFunctor - .fmapRight(rightFunc); - - return rightMapper.apply(leftMappedFunctor); - }; - - return bimappedFunc; - } - - /** - * Lift a function to operate over the left part of this pair - * - * @param - * The old left type of the pair - * @param - * The old right type of the pair - * @param - * The new left type of the pair - * @param func - * The function to lift to work over the left side of the - * pair - * @return The function lifted to work over the left side of bifunctors - */ - public Function, Bifunctor> fmapLeft( - Function func); - - /** - * Lift a function to operate over the right part of this pair - * - * @param - * The old left type of the pair - * @param - * The old right type of the pair - * @param - * The new right type of the pair - * @param func - * The function to lift to work over the right side of - * the pair - * @return The function lifted to work over the right side of bifunctors - */ - public Function, Bifunctor> fmapRight( - Function func); - - /** - * Get the value contained on the left of this bifunctor - * - * @return The value on the left side of this bifunctor - */ - public LeftType getLeft(); - - /** - * Get the value contained on the right of this bifunctor - * - * @return The value on the right of this bifunctor - */ - public RightType getRight(); -} +package bjc.utils.funcdata.theory; + +import java.util.function.Function; + +/** + * A functor over a pair of heterogeneous types + * + * @author ben + * @param + * The type stored on the 'left' of the pair + * @param + * The type stored on the 'right' of the pair + * + */ +public interface Bifunctor { + /** + * Alias for functor mapping. + * + * @author EVE + * + * @param + * @param + * @param + * @param + */ + public interface BifunctorMap + extends Function, Bifunctor> { + + } + + /** + * Alias for left functor mapping. + * + * @author EVE + * + * @param + * @param + * @param + */ + public interface LeftBifunctorMap + extends BifunctorMap { + + } + + /** + * Alias for right functor mapping. + * + * @author EVE + * + * @param + * @param + * @param + */ + public interface RightBifunctorMap + extends BifunctorMap { + + } + + /** + * Lift a pair of functions to a single function that maps over both + * parts of a pair + * + * @param + * The old left type of the pair + * @param + * The old right type of the pair + * @param + * The new left type of the pair + * @param + * The new right type of the pair + * @param leftFunc + * The function that maps over the left of the pair + * @param rightFunc + * The function that maps over the right of the pair + * @return A function that maps over both parts of the pair + */ + public default BifunctorMap bimap( + Function leftFunc, Function rightFunc) { + BifunctorMap bimappedFunc = (argPair) -> { + LeftBifunctorMap leftMapper = argPair.fmapLeft(leftFunc); + + Bifunctor leftMappedFunctor = leftMapper.apply(argPair); + RightBifunctorMap rightMapper = leftMappedFunctor + .fmapRight(rightFunc); + + return rightMapper.apply(leftMappedFunctor); + }; + + return bimappedFunc; + } + + /** + * Lift a function to operate over the left part of this pair + * + * @param + * The old left type of the pair + * @param + * The old right type of the pair + * @param + * The new left type of the pair + * @param func + * The function to lift to work over the left side of the + * pair + * @return The function lifted to work over the left side of bifunctors + */ + public LeftBifunctorMap fmapLeft( + Function func); + + /** + * Lift a function to operate over the right part of this pair + * + * @param + * The old left type of the pair + * @param + * The old right type of the pair + * @param + * The new right type of the pair + * @param func + * The function to lift to work over the right side of + * the pair + * @return The function lifted to work over the right side of bifunctors + */ + public RightBifunctorMap fmapRight( + Function func); + + /** + * Get the value contained on the left of this bifunctor + * + * @return The value on the left side of this bifunctor + */ + public LeftType getLeft(); + + /** + * Get the value contained on the right of this bifunctor + * + * @return The value on the right of this bifunctor + */ + public RightType getRight(); +} diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java index 83d2d2a..a91bf2a 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/DoubleMatcher.java @@ -13,31 +13,31 @@ class DoubleMatcher { /* * Unit pieces. */ - private static final String DecDigits = getRegex("fpDigits"); - private static final String HexDigits = getRegex("fpHexDigits"); - private static final String Exponent = applyFormat("fpExponent", getRegex("fpExponent"), DecDigits); + private static final String rDecDigits = getRegex("fpDigits"); + private static final String rHexDigits = getRegex("fpHexDigits"); + private static final String rExponent = applyFormat("fpExponent", getRegex("fpExponent"), rDecDigits); /* * Decimal floating point numbers. */ - private static final String SIMPLE_DEC = applyFormat("fpDecimalDecimal", DecDigits, Exponent); - private static final String SIMPLE_INTDEC = applyFormat("fpDecimalInteger", DecDigits, Exponent); + private static final String rSimpleDec = applyFormat("fpDecimalDecimal", rDecDigits, rExponent); + private static final String rSimpleIntDec = applyFormat("fpDecimalInteger", rDecDigits, rExponent); /* * Hex floating point numbers. */ - private static final String HEX_INT = applyFormat("fpHexInteger", HexDigits); - private static final String HEX_DEC = applyFormat("fpHexDecimal", HexDigits); - private static final String HEX_LEAD = applyFormat("fpHexLeader", HEX_INT, HEX_DEC); - private static final String HEX_STRING = applyFormat("fpHexString", HEX_LEAD, DecDigits); + private static final String rHexInt = applyFormat("fpHexInteger", rHexDigits); + private static final String rHexDec = applyFormat("fpHexDecimal", rHexDigits); + private static final String rHexLead = applyFormat("fpHexLeader", rHexInt, rHexDec); + private static final String rHexString = applyFormat("fpHexString", rHexLead, rDecDigits); /* * Floating point components. */ - private static final String FP_LEADER = getRegex("fpLeader"); - private static final String FP_NUM = applyFormat("fpNumber", SIMPLE_INTDEC, SIMPLE_DEC, - HEX_STRING); + private static final String rFPLeader = getRegex("fpLeader"); + private static final String rFPNum = applyFormat("fpNumber", rSimpleIntDec, rSimpleDec, + rHexString); - private static final String fpRegex = applyFormat("fpDouble", FP_LEADER, FP_NUM); - public static final Pattern floatingLiteral = Pattern.compile("\\A" + fpRegex + "\\Z"); + private static final String rDouble = applyFormat("fpDouble", rFPLeader, rFPNum); + public static final Pattern doubleLiteral = Pattern.compile("\\A" + rDouble + "\\Z"); } \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/ParserException.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/ParserException.java index 67812de..2a41009 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/ParserException.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/ParserException.java @@ -25,4 +25,4 @@ public class ParserException extends Exception { public ParserException(String msg, Exception cause) { super(msg, cause); } -} +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java index 7fc3688..b30a69c 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/ShuntingYard.java @@ -12,36 +12,36 @@ import java.util.function.Consumer; import java.util.function.Function; /** - * Utility to run the shunting yard algorithm on a bunch of tokens + * Utility to run the shunting yard algorithm on a bunch of tokens. * * @author ben * * @param - * The type of tokens being shunted + * The type of tokens being shunted. */ public class ShuntingYard { /** - * A enum representing the fundamental operator types + * A enum representing the fundamental operator types. * * @author ben * */ public static enum Operator implements IPrecedent { /** - * Represents addition + * Represents addition. */ ADD(1), /** - * Represents subtraction + * Represents subtraction. */ SUBTRACT(2), /** - * Represents multiplication + * Represents multiplication. */ MULTIPLY(3), /** - * Represents division + * Represents division. */ DIVIDE(4); @@ -72,26 +72,26 @@ public class ShuntingYard { @Override public void accept(String token) { // Handle operators - if (operators.containsKey(token)) { + if(operators.containsKey(token)) { // Pop operators while there isn't a higher // precedence one - while (!stack.isEmpty() && isHigherPrec(token, stack.peek())) { + while(!stack.isEmpty() && isHigherPrec(token, stack.peek())) { output.add(transformer.apply(stack.pop())); } // Put this operator onto the stack stack.push(token); - } else if (StringUtils.containsOnly(token, "\\(")) { + } else if(StringUtils.containsOnly(token, "\\(")) { // Handle groups of parenthesis for multiple // nesting levels stack.push(token); - } else if (StringUtils.containsOnly(token, "\\)")) { + } else if(StringUtils.containsOnly(token, "\\)")) { // Handle groups of parenthesis for multiple // nesting levels String swappedToken = token.replace(')', '('); // Remove tokens up to a matching parenthesis - while (!stack.peek().equals(swappedToken)) { + while(!stack.peek().equals(swappedToken)) { output.add(transformer.apply(stack.pop())); } @@ -105,21 +105,22 @@ public class ShuntingYard { } /* - * Holds all the shuntable operations + * Holds all the shuntable operations. */ private IMap operators; /** - * Create a new shunting yard with a default set of operators + * Create a new shunting yard with a default set of operators. * * @param configureBasics - * Whether or not basic math operators should be provided + * Whether or not basic math operators should be + * provided. */ public ShuntingYard(boolean configureBasics) { operators = new FunctionalMap<>(); // Add basic operators if we're configured to do so - if (configureBasics) { + if(configureBasics) { operators.put("+", Operator.ADD); operators.put("-", Operator.SUBTRACT); operators.put("*", Operator.MULTIPLY); @@ -128,12 +129,13 @@ public class ShuntingYard { } /** - * Add an operator to the list of shuntable operators + * Add an operator to the list of shuntable operators. * * @param operator - * The token representing the operator + * The token representing the operator. + * * @param precedence - * The precedence of the operator to add + * The precedence of the operator to add. */ public void addOp(String operator, int precedence) { /* @@ -145,24 +147,24 @@ public class ShuntingYard { } /** - * Add an operator to the list of shuntable operators + * Add an operator to the list of shuntable operators. * * @param operator - * The token representing the operator + * The token representing the operator. * * @param precedence - * The precedence of the operator + * The precedence of the operator. */ public void addOp(String operator, IPrecedent precedence) { /* - * Complain about trying to add an incorrect operator + * Complain about trying to add an incorrect operator */ - if (operator == null) + if(operator == null) throw new NullPointerException("Operator must not be null"); - else if (precedence == null) throw new NullPointerException("Precedence must not be null"); + else if(precedence == null) throw new NullPointerException("Precedence must not be null"); /* - * Add the operator to the ones we handle + * Add the operator to the ones we handle */ operators.put(operator, precedence); } @@ -172,7 +174,7 @@ public class ShuntingYard { boolean exists = operators.containsKey(right); // If it doesn't, the left is higher precedence. - if (!exists) return false; + if(!exists) return false; // Get the precedence of operators int rightPrecedence = operators.get(right).getPrecedence(); @@ -183,19 +185,21 @@ public class ShuntingYard { } /** - * Transform a string of tokens from infix notation to postfix + * Transform a string of tokens from infix notation to postfix. * * @param input - * The string to transform + * The string to transform. + * * @param transformer - * The function to use to transform strings to tokens - * @return A list of tokens in postfix notation + * The function to use to transform strings to tokens. + * + * @return A list of tokens in postfix notation. */ public IList postfix(IList input, Function transformer) { // Check our input - if (input == null) + if(input == null) throw new NullPointerException("Input must not be null"); - else if (transformer == null) throw new NullPointerException("Transformer must not be null"); + else if(transformer == null) throw new NullPointerException("Transformer must not be null"); // Here's what we're handing back IList output = new FunctionalList<>(); @@ -207,7 +211,7 @@ public class ShuntingYard { input.forEach(new TokenShunter(output, stack, transformer)); // Transform any resulting tokens - stack.forEach((token) -> { + stack.forEach(token -> { output.add(transformer.apply(token)); }); @@ -215,18 +219,18 @@ public class ShuntingYard { } /** - * Remove an operator from the list of shuntable operators + * Remove an operator from the list of shuntable operators. * * @param operator * The token representing the operator. If null, remove - * all operators + * all operators. */ public void removeOp(String operator) { // Check if we want to remove all operators - if (operator == null) { + if(operator == null) { operators = new FunctionalMap<>(); } else { operators.remove(operator); } } -} +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java index 5f843a2..c441dff 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenTransformer.java @@ -1,10 +1,11 @@ package bjc.utils.parserutils; import bjc.utils.data.IHolder; -import bjc.utils.data.IPair; import bjc.utils.data.ITree; import bjc.utils.data.Pair; import bjc.utils.data.Tree; +import bjc.utils.parserutils.TreeConstructor.ConstructorState; +import bjc.utils.parserutils.TreeConstructor.QueueFlattener; import java.util.Deque; import java.util.function.Consumer; @@ -14,7 +15,7 @@ import java.util.function.UnaryOperator; final class TokenTransformer implements Consumer { // Handle operators - private final class OperatorHandler implements UnaryOperator>, ITree>> { + private final class OperatorHandler implements UnaryOperator> { private TokenType element; public OperatorHandler(TokenType element) { @@ -22,17 +23,15 @@ final class TokenTransformer implements Consumer { } @Override - public IPair>, ITree> apply( - IPair>, ITree> pair) { + public ConstructorState apply(ConstructorState pair) { // Replace the current AST with the result of handling // an operator - return pair.bindLeft((queuedASTs) -> { + return new ConstructorState<>(pair.bindLeft(queuedASTs -> { return handleOperator(queuedASTs); - }); + })); } - private IPair>, ITree> handleOperator( - Deque> queuedASTs) { + private ConstructorState handleOperator(Deque> queuedASTs) { // The AST we're going to hand back ITree newAST; @@ -42,10 +41,13 @@ final class TokenTransformer implements Consumer { } else { // Error if we don't have enough for a binary // operator - if(queuedASTs.size() < 2) throw new IllegalStateException( - "Attempted to parse binary operator without enough operands.\n" - + "Problem operator is " + element - + "\nPossible operand is: \n\t" + queuedASTs.peek()); + if(queuedASTs.size() < 2) { + String msg = String.format( + "Attempted to parse binary operator without enough operands\n\tProblem operator is: %s\n\tPossible operand is: %s", + element.toString(), queuedASTs.peek().toString()); + + throw new IllegalStateException(msg); + } // Grab the two operands ITree right = queuedASTs.pop(); @@ -59,21 +61,21 @@ final class TokenTransformer implements Consumer { queuedASTs.push(newAST); // Hand back the state - return new Pair<>(queuedASTs, newAST); + return new ConstructorState<>(queuedASTs, newAST); } } - private IHolder>, ITree>> initialState; + private IHolder> initialState; private Predicate operatorPredicate; - private Predicate isSpecialOperator; - private Function>, ITree>> handleSpecialOperator; + private Predicate isSpecialOperator; + private Function> handleSpecialOperator; // Create a new transformer - public TokenTransformer(IHolder>, ITree>> initialState, + public TokenTransformer(IHolder> initialState, Predicate operatorPredicate, Predicate isSpecialOperator, - Function>, ITree>> handleSpecialOperator) { + Function> handleSpecialOperator) { this.initialState = initialState; this.operatorPredicate = operatorPredicate; this.isSpecialOperator = isSpecialOperator; @@ -89,15 +91,15 @@ final class TokenTransformer implements Consumer { ITree newAST = new Tree<>(element); // Insert the new tree into the AST - initialState.transform((pair) -> { + initialState.transform(pair -> { // Transform the pair, ignoring the current AST // in favor of the // one consisting of the current element - return pair.bindLeft((queue) -> { + return new ConstructorState<>(pair.bindLeft(queue -> { queue.push(newAST); return new Pair<>(queue, newAST); - }); + })); }); } } diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java index 0ec00ee..52eba1d 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TokenUtils.java @@ -1,257 +1,267 @@ -package bjc.utils.parserutils; - -import java.util.LinkedList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static bjc.utils.PropertyDB.getRegex; -import static bjc.utils.PropertyDB.getCompiledRegex; -import static bjc.utils.PropertyDB.applyFormat; - -/** - * Utilities useful for operating on PL tokens. - * - * @author EVE - * - */ -public class TokenUtils { - private static String possibleEscapeString = getRegex("possibleStringEscape"); - - private static Pattern possibleEscapePatt = Pattern.compile(possibleEscapeString); - - private static String shortEscape = getRegex("shortFormStringEscape"); - private static String octalEscape = getRegex("octalStringEscape"); - private static String unicodeEscape = getRegex("unicodeStringEscape"); - - private static String escapeString = applyFormat("stringEscape", shortEscape, octalEscape, unicodeEscape); - - private static Pattern escapePatt = Pattern.compile(escapeString); - - private static String doubleQuoteString = applyFormat("doubleQuotes", getRegex("nonEscape"), - possibleEscapeString); - - private static Pattern doubleQuotePatt = Pattern.compile(doubleQuoteString); - - private static Pattern quotePatt = getCompiledRegex("unescapedQuote"); - - /** - * Remove double quoted strings from a string. - * - * Splits a string around instances of java-style double-quoted strings. - * - * @param inp - * The string to split. - * - * @return An list containing alternating bits of the string and the - * embedded double-quoted strings that separated them. - */ - public static List removeDQuotedStrings(String inp) { - if(inp == null) { - throw new NullPointerException("inp must not be null"); - } - - /* - * What we need for piece-by-piece string building - */ - StringBuffer work = new StringBuffer(); - List res = new LinkedList<>(); - - /* - * Matcher for proper strings and single quotes. - */ - Matcher mt = doubleQuotePatt.matcher(inp); - Matcher corr = quotePatt.matcher(inp); - - if(corr.find() && !corr.find()) { - /* - * There's a unmatched opening quote with no strings. - */ - throw new IllegalArgumentException( - String.format("Unclosed string literal '%s'. Opening quote was at position %d", - inp, inp.indexOf("\""))); - } - - while(mt.find()) { - /* - * Remove the string until the quoted string. - */ - mt.appendReplacement(work, ""); - - /* - * Add the string preceding the double-quoted string and - * the double-quoted string to the list. - */ - res.add(work.toString()); - res.add(mt.group(1)); - - /* - * Renew the buffer. - */ - work = new StringBuffer(); - } - - /* - * Grab the remainder of the string. - */ - mt.appendTail(work); - String tail = work.toString(); - - if(tail.contains("\"")) { - /* - * There's a unmatched opening quote with at least one - * string. - */ - throw new IllegalArgumentException( - String.format("Unclosed string literal '%s'. Opening quote was at position %d", - inp, inp.lastIndexOf("\""))); - } - - /* - * Only add an empty tail if the string was empty. - */ - if(!tail.equals("") || res.isEmpty()) { - res.add(tail); - } - - return res; - } - - /** - * Replace escape characters with their actual equivalents. - * - * @param inp - * The string to replace escape sequences in. - * - * @return The string with escape sequences replaced by their equivalent - * characters. - */ - public static String descapeString(String inp) { - if(inp == null) { - throw new NullPointerException("inp must not be null"); - } - - StringBuffer work = new StringBuffer(); - - Matcher possibleEscapeFinder = possibleEscapePatt.matcher(inp); - Matcher escapeFinder = escapePatt.matcher(inp); - - while(possibleEscapeFinder.find()) { - if(!escapeFinder.find()) { - throw new IllegalArgumentException(String.format( - "Illegal escape sequence '%s' at position %d", - possibleEscapeFinder.group(), possibleEscapeFinder.start())); - } - - String escapeSeq = escapeFinder.group(); - - String escapeRep = ""; - switch(escapeSeq) { - case "\\b": - escapeRep = "\b"; - break; - case "\\t": - escapeRep = "\t"; - break; - case "\\n": - escapeRep = "\n"; - break; - case "\\f": - escapeRep = "\f"; - break; - case "\\r": - escapeRep = "\r"; - break; - case "\\\"": - escapeRep = "\""; - break; - case "\\'": - escapeRep = "'"; - break; - case "\\\\": - /* - * Skip past the second slash. - */ - possibleEscapeFinder.find(); - escapeRep = "\\"; - break; - default: - if(escapeSeq.startsWith("u")) { - escapeRep = handleUnicodeEscape(escapeSeq.substring(1)); - } else { - escapeRep = handleOctalEscape(escapeSeq); - } - } - - escapeFinder.appendReplacement(work, escapeRep); - } - - escapeFinder.appendTail(work); - - return work.toString(); - } - - private static String handleUnicodeEscape(String seq) { - try { - int codepoint = Integer.parseInt(seq, 16); - - return new String(Character.toChars(codepoint)); - } catch(IllegalArgumentException iaex) { - IllegalArgumentException reiaex = new IllegalArgumentException( - String.format("'%s' is not a valid Unicode escape sequence'", seq)); - - reiaex.initCause(iaex); - - throw reiaex; - } - } - - private static String handleOctalEscape(String seq) { - try { - int codepoint = Integer.parseInt(seq, 8); - - if(codepoint > 255) { - throw new IllegalArgumentException(String - .format("'%d' is outside the range of octal escapes', codepoint")); - } - - return new String(Character.toChars(codepoint)); - } catch(IllegalArgumentException iaex) { - IllegalArgumentException reiaex = new IllegalArgumentException( - String.format("'%s' is not a valid octal escape sequence'", seq)); - - reiaex.initCause(iaex); - - throw reiaex; - } - } - - /** - * Check if a given string would be successfully converted to a double - * by {@link Double#parseDouble(String)}. - * - * @param inp - * The string to check. - * @return Whether the string is a valid double or not. - */ - public static boolean isDouble(String inp) { - return DoubleMatcher.floatingLiteral.matcher(inp).matches(); - } - - private static Pattern intLitPattern = getCompiledRegex("intLiteral"); - - /** - * Check if a given string would be successfully converted to a integer - * by {@link Integer#parseInt(String)}. - * - * NOTE: This only checks syntax. Using values out of the range of - * integers will still cause errors. - * - * @param inp - * The input to check. - * @return Whether the string is a valid double or not. - */ - public static boolean isInt(String inp) { - return intLitPattern.matcher(inp).matches(); - } +package bjc.utils.parserutils; + +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static bjc.utils.PropertyDB.getRegex; +import static bjc.utils.PropertyDB.getCompiledRegex; +import static bjc.utils.PropertyDB.applyFormat; + +/** + * Utilities useful for operating on PL tokens. + * + * @author EVE + * + * TODO add support for user defined escapes. + */ +public class TokenUtils { + /* + * Patterns and pattern parts. + */ + private static String rPossibleEscapeString = getRegex("possibleStringEscape"); + + private static Pattern possibleEscapePatt = Pattern.compile(rPossibleEscapeString); + + private static String rShortEscape = getRegex("shortFormStringEscape"); + private static String rOctalEscape = getRegex("octalStringEscape"); + private static String rUnicodeEscape = getRegex("unicodeStringEscape"); + + private static String rEscapeString = applyFormat("stringEscape", rShortEscape, rOctalEscape, rUnicodeEscape); + + private static Pattern escapePatt = Pattern.compile(rEscapeString); + + private static String rDoubleQuoteString = applyFormat("doubleQuotes", getRegex("nonEscape"), + rPossibleEscapeString); + + private static Pattern doubleQuotePatt = Pattern.compile(rDoubleQuoteString); + + private static Pattern quotePatt = getCompiledRegex("unescapedQuote"); + + private static Pattern intLitPattern = getCompiledRegex("intLiteral"); + + /** + * Remove double quoted strings from a string. + * + * Splits a string around instances of java-style double-quoted strings. + * + * @param inp + * The string to split. + * + * @return An list containing alternating bits of the string and the + * embedded double-quoted strings that separated them. + */ + public static List removeDQuotedStrings(String inp) { + if(inp == null) { + throw new NullPointerException("inp must not be null"); + } + + /* + * What we need for piece-by-piece string building + */ + StringBuffer work = new StringBuffer(); + List res = new LinkedList<>(); + + /* + * Matcher for proper strings and single quotes. + */ + Matcher mt = doubleQuotePatt.matcher(inp); + Matcher corr = quotePatt.matcher(inp); + + if(corr.find() && !corr.find()) { + /* + * There's a unmatched opening quote with no strings. + */ + String msg = String.format("Unclosed string literal '%s'. Opening quote was at position %d", + inp, inp.indexOf("\"")); + + throw new IllegalArgumentException(msg); + } + + while(mt.find()) { + /* + * Remove the string until the quoted string. + */ + mt.appendReplacement(work, ""); + + /* + * Add the string preceding the double-quoted string and + * the double-quoted string to the list. + */ + res.add(work.toString()); + res.add(mt.group(1)); + + /* + * Renew the buffer. + */ + work = new StringBuffer(); + } + + /* + * Grab the remainder of the string. + */ + mt.appendTail(work); + String tail = work.toString(); + + if(tail.contains("\"")) { + /* + * There's a unmatched opening quote with at least one + * string. + */ + String msg = String.format("Unclosed string literal '%s'. Opening quote was at position %d", + inp, inp.lastIndexOf("\"")); + + throw new IllegalArgumentException(msg); + } + + /* + * Only add an empty tail if the string was empty. + */ + if(!tail.equals("") || res.isEmpty()) { + res.add(tail); + } + + return res; + } + + /** + * Replace escape characters with their actual equivalents. + * + * @param inp + * The string to replace escape sequences in. + * + * @return The string with escape sequences replaced by their equivalent + * characters. + */ + public static String descapeString(String inp) { + if(inp == null) { + throw new NullPointerException("inp must not be null"); + } + + StringBuffer work = new StringBuffer(); + + Matcher possibleEscapeFinder = possibleEscapePatt.matcher(inp); + Matcher escapeFinder = escapePatt.matcher(inp); + + while(possibleEscapeFinder.find()) { + if(!escapeFinder.find()) { + String msg = String.format("Illegal escape sequence '%s' at position %d", + possibleEscapeFinder.group(), possibleEscapeFinder.start()); + + throw new IllegalArgumentException(msg); + } + + String escapeSeq = escapeFinder.group(); + + String escapeRep = ""; + switch(escapeSeq) { + case "\\b": + escapeRep = "\b"; + break; + case "\\t": + escapeRep = "\t"; + break; + case "\\n": + escapeRep = "\n"; + break; + case "\\f": + escapeRep = "\f"; + break; + case "\\r": + escapeRep = "\r"; + break; + case "\\\"": + escapeRep = "\""; + break; + case "\\'": + escapeRep = "'"; + break; + case "\\\\": + /* + * Skip past the second slash. + */ + possibleEscapeFinder.find(); + escapeRep = "\\"; + break; + default: + if(escapeSeq.startsWith("u")) { + escapeRep = handleUnicodeEscape(escapeSeq.substring(1)); + } else { + escapeRep = handleOctalEscape(escapeSeq); + } + } + + escapeFinder.appendReplacement(work, escapeRep); + } + + escapeFinder.appendTail(work); + + return work.toString(); + } + + private static String handleUnicodeEscape(String seq) { + try { + int codepoint = Integer.parseInt(seq, 16); + + return new String(Character.toChars(codepoint)); + } catch(IllegalArgumentException iaex) { + String msg = String.format("'%s' is not a valid Unicode escape sequence'", seq); + + IllegalArgumentException reiaex = new IllegalArgumentException(msg); + + reiaex.initCause(iaex); + + throw reiaex; + } + } + + private static String handleOctalEscape(String seq) { + try { + int codepoint = Integer.parseInt(seq, 8); + + if(codepoint > 255) { + String msg = String.format("'%d' is outside the range of octal escapes', codepoint"); + + throw new IllegalArgumentException(msg); + } + + return new String(Character.toChars(codepoint)); + } catch(IllegalArgumentException iaex) { + String msg = String.format("'%s' is not a valid octal escape sequence'", seq); + + IllegalArgumentException reiaex = new IllegalArgumentException(msg); + + reiaex.initCause(iaex); + + throw reiaex; + } + } + + /** + * Check if a given string would be successfully converted to a double + * by {@link Double#parseDouble(String)}. + * + * @param inp + * The string to check. + * @return Whether the string is a valid double or not. + */ + public static boolean isDouble(String inp) { + return DoubleMatcher.doubleLiteral.matcher(inp).matches(); + } + + /** + * Check if a given string would be successfully converted to a integer + * by {@link Integer#parseInt(String)}. + * + * NOTE: This only checks syntax. Using values out of the range of + * integers will still cause errors. + * + * @param inp + * The input to check. + * @return Whether the string is a valid double or not. + */ + public static boolean isInt(String inp) { + return intLitPattern.matcher(inp).matches(); + } } \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java index 82ded42..bd0ab97 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/TreeConstructor.java @@ -19,6 +19,29 @@ import java.util.function.Predicate; * */ public class TreeConstructor { + /** + * Alias interface for special operator types. + * + * @param + * The token type of the tree. + */ + public interface QueueFlattener extends Function>, ITree> { + + } + + /* + * Alias for constructor state. + */ + static final class ConstructorState extends Pair>, ITree> { + public ConstructorState(Deque> left, ITree right) { + super(left, right); + } + + public ConstructorState(IPair>, ITree> par) { + super(par.getLeft(), par.getRight()); + } + } + /** * Construct a tree from a list of tokens in postfix notation * @@ -36,36 +59,38 @@ public class TreeConstructor { public static ITree constructTree(IList tokens, Predicate isOperator) { // Construct a tree with no special operators - return constructTree(tokens, isOperator, (op) -> false, null); + return constructTree(tokens, isOperator, op -> false, null); } /** - * Construct a tree from a list of tokens in postfix notation + * Construct a tree from a list of tokens in postfix notation. * * Only binary operators are accepted by default. Use the last two - * parameters to handle non-binary operators + * parameters to handle non-binary operators. * * @param - * The elements of the parse tree + * The elements of the parse tree. + * * @param tokens - * The list of tokens to build a tree from + * The list of tokens to build a tree from. + * * @param isOperator * The predicate to use to determine if something is a - * operator + * operator. + * * @param isSpecialOperator * The predicate to use to determine if an operator needs - * special handling + * special handling. + * * @param handleSpecialOperator - * The function to use to handle special case operators + * The function to use to handle special case operators. + * * @return A AST from the expression * - * FIXME The handleSpecialOp function seems like an ugly - * interface. Maybe there's a better way to express how that - * works. */ public static ITree constructTree(IList tokens, Predicate isOperator, Predicate isSpecialOperator, - Function>, ITree>> handleSpecialOperator) { + Function> handleSpecialOperator) { // Make sure our parameters are valid if(tokens == null) throw new NullPointerException("Tokens must not be null"); @@ -75,16 +100,16 @@ public class TreeConstructor { throw new NullPointerException("Special operator determiner must not be null"); // Here is the state for the tree construction - IHolder>, ITree>> initialState = new Identity<>( - new Pair<>(new LinkedList<>(), null)); + IHolder> initialState = new Identity<>( + new ConstructorState<>(new LinkedList<>(), null)); // Transform each of the tokens tokens.forEach(new TokenTransformer<>(initialState, isOperator, isSpecialOperator, handleSpecialOperator)); // Grab the tree from the state - return initialState.unwrap((pair) -> { + return initialState.unwrap(pair -> { return pair.getRight(); }); } -} +} \ No newline at end of file diff --git a/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java index 8b078a9..170c619 100644 --- a/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java +++ b/BJC-Utils2/src/main/java/bjc/utils/parserutils/splitter/SimpleTokenSplitter.java @@ -1,5 +1,7 @@ package bjc.utils.parserutils.splitter; +import bjc.utils.ioutils.RegexStringEditor; + import java.util.HashSet; import java.util.Set; import java.util.regex.Pattern; @@ -9,6 +11,7 @@ import java.util.regex.Pattern; * * @author EVE * + * TODO rewrite using {@link RegexStringEditor} */ public class SimpleTokenSplitter implements TokenSplitter { /* -- cgit v1.2.3