diff options
Diffstat (limited to 'RGens/src/main/java/bjc')
3 files changed, 540 insertions, 62 deletions
diff --git a/RGens/src/main/java/bjc/RGens/server/CLIArgsParser.java b/RGens/src/main/java/bjc/RGens/server/CLIArgsParser.java new file mode 100644 index 0000000..a39ce1d --- /dev/null +++ b/RGens/src/main/java/bjc/RGens/server/CLIArgsParser.java @@ -0,0 +1,141 @@ +package bjc.RGens.server; + +import java.io.File; + +public class CLIArgsParser { + public static void parseArgs(String[] args, GrammarServerEngine eng) { + boolean didTerminalOp = false; + boolean forceInteractive = false; + + // @TODO report error status + boolean didError = false; + + if(args.length < 0) return; + + if(args.length == 1 && args[0].equals("--help")) { + // @TODO show help + } else { + for(int i = 0; i < args.length; i++) { + String arg = args[i]; + + switch(arg) { + case "-lc": + case "--load-config-file": + String configFileName = args[++i]; + + eng.doLoadConfig(configFileName); + break; + case "-lg": + case "--load-grammar-file": + String grammarFileName = args[++i]; + + File grammarFile = new File(grammarFileName); + + String ruleName = grammarFile.getName(); + ruleName = ruleName.substring(0, ruleName.lastIndexOf('.')); + + if(!args[i+1].startsWith("-")) { + ruleName = args[++i]; + } + + eng.doLoadGrammar(ruleName, grammarFileName); + + break; + case "-ll": + case "--load-long-file": + String longRuleFileName = args[++i]; + + File longRuleFile = new File(longRuleFileName); + + String longRuleName = longRuleFile.getName(); + longRuleName = longRuleName.substring(0, longRuleName.lastIndexOf('.')); + + if(!args[i+1].startsWith("-")) { + longRuleName = args[++i]; + } + + eng.doLoadLongRule(longRuleName, longRuleFileName); + + break; + case "-ge": + case "--generate-exported-rule": + String exportedRuleName = args[++i]; + + if(eng.hasExportedRule(exportedRuleName)) { + eng.doGenerateExportedRule(exportedRuleName); + didTerminalOp = true; + } else { + System.out.printf("Error: No exported rule named %s\n", exportedRuleName); + } + break; + case "-gg": + case "--generate-grammar-rule": + String grammarName = args[++i]; + + if(!eng.hasLoadedGrammar(grammarName)) { + System.out.printf("Error: No grammar named %s\n", grammarName); + } else { + String ruleToGenerate = ""; + + if(!args[i+1].startsWith("-")) { + ruleToGenerate = args[++i]; + } else if(eng.hasInitialRule(grammarName)) { + ruleToGenerate = eng.getInitialRule(grammarName); + } else { + System.out.printf("Error: Grammar %s has no initial rule. A " + + "rule must be provided.",grammarName); + } + + eng.doGenerateGrammar(grammarName, ruleToGenerate); + didTerminalOp = true; + } + break; + case "-st": + case "--stress-test": + String thingToTest = args[++i]; + + // @TODO support testing rules from grammars + // as well as a specified number of times + if(thingToTest.equals("*")) { + eng.doStressTest(10000); + } else { + eng.doStressTest(thingToTest, 1000); + } + + didTerminalOp = true; + + break; + case "-d": + case "--debug": + if(eng.debugMode) { + System.out.println("Warning: debug mode is already on. Use -nd or --no-debug" + + " to turn it off"); + } else { + eng.debugMode = true; + } + break; + case "-nd": + case "--no-debug": + if(!eng.debugMode) { + System.out.println("Warning: debug mode is already off. Use -d or --debug" + + " to turn it on"); + } else { + eng.debugMode = false; + } + break; + case "-i": + case "--interactive": + forceInteractive = true; + break; + default: + System.out.println("Error: Unrecognized argument " + arg); + break; + } + } + } + + if(!forceInteractive && didTerminalOp) { + System.exit(didError ? 1 : 0); + } + } +} diff --git a/RGens/src/main/java/bjc/RGens/server/GrammarServerEngine.java b/RGens/src/main/java/bjc/RGens/server/GrammarServerEngine.java new file mode 100644 index 0000000..0fe96d9 --- /dev/null +++ b/RGens/src/main/java/bjc/RGens/server/GrammarServerEngine.java @@ -0,0 +1,300 @@ +package bjc.RGens.server; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.LinkedList; +import java.util.Queue; +import java.util.Scanner; + +import bjc.utils.funcdata.FunctionalList; +import bjc.utils.funcdata.IList; +import bjc.utils.funcdata.IMap; +import bjc.utils.funcutils.ListUtils; +import bjc.utils.gen.WeightedGrammar; + +public class GrammarServerEngine { + private IMap<String, WeightedGrammar<String>> loadedGrammars; + private IMap<String, WeightedGrammar<String>> exportedRules; + + public static boolean debugMode = false; + + public GrammarServerEngine(IMap<String, WeightedGrammar<String>> loadedGrammars, + IMap<String, WeightedGrammar<String>> exportedRules) { + this.loadedGrammars = loadedGrammars; + this.exportedRules = exportedRules; + } + + public String getInitialRule(String grammarName) { + return loadedGrammars.get(grammarName).getInitialRule(); + } + + public boolean hasInitialRule(String grammarName) { + return loadedGrammars.get(grammarName).hasInitialRule(); + } + + public boolean hasExportedRule(String ruleName) { + return exportedRules.containsKey(ruleName); + } + + public boolean hasLoadedGrammar(String grammarName) { + return loadedGrammars.containsKey(grammarName); + } + + public void doLoadConfig(String fileName) { + File inputFile = new File(fileName); + + try(FileInputStream inputStream = new FileInputStream(inputFile)) { + try(Scanner fle = new Scanner(inputStream)) { + while(fle.hasNextLine()) { + String line = fle.nextLine().trim(); + + // Handle comments + if(line.equals("") || + line.startsWith("#") || + line.startsWith("//")) { + continue; + } + + // Handle mixed whitespace in input + line = line.replaceAll("\\s+", " "); + + String path; + String name; + + if(line.lastIndexOf(' ') != -1) { + path = line.substring(0, line.lastIndexOf(' ')); + name = line.substring(line.lastIndexOf(' ') + 1, line.length()); + } else { + path = line; + + File pathFile = new File(path); + String pathName = pathFile.getName(); + + if(pathFile.isDirectory()) { + // Load all the files in the directory recursively + Queue<File> entries = new LinkedList<>(); + + for (File entry : pathFile.listFiles()) { + entries.add(entry); + } + + while(!entries.isEmpty()) { + File entry = entries.poll(); + + String entryPath = entry.getName(); + + if(entry.isHidden()) continue; + if(entry.isDirectory()) { + for (File newEntry : entry.listFiles()) { + entries.add(newEntry); + } + + continue; + } + + name = entryPath.substring(0, entryPath.lastIndexOf('.')); + + doLoadGrammarEntry(entry.toString(), name); + } + + continue; + } + + name = pathName.substring(0, pathName.lastIndexOf('.')); + } + + doLoadGrammarEntry(path, name); + } + } + } catch(IOException ioex) { + System.out.printf("? Error reading configuration from file" + + " (reason: %s)\n", ioex.getMessage()); + } + } + + private void doLoadGrammarEntry(String path, String name) { + if(path.endsWith(".gram")) { + doLoadGrammar(name, path); + } else if(path.endsWith(".template")) { + System.out.println("Error: Templates are not supported yet"); + } else if(path.endsWith(".long")) { + doLoadLongRule(name, path); + } else { + System.out.println("Error: Unknown filetype " + + path.substring(path.lastIndexOf("."), path.length())); + } + } + + public void doLoadLongRule(String ruleName, String ruleFile) { + ruleName = "[" + ruleName + "]"; + + if(debugMode) { + System.out.printf("Loading long rule (named %s) from path %s\n", + ruleName, ruleFile); + } + + try (FileInputStream inputStream = new FileInputStream(ruleFile)) { + try (Scanner fle = new Scanner(inputStream)) { + IList<IList<String>> ruleParts = new FunctionalList<>(); + + while(fle.hasNextLine()) { + ruleParts.add(new FunctionalList<>(fle.nextLine().trim().split(" "))); + } + + WeightedGrammar<String> longGram = new WeightedGrammar<>(); + + longGram.addSpecialRule(ruleName, () -> ruleParts.randItem()); + longGram.setInitialRule(ruleName); + + exportedRules.put(ruleName, longGram); + + if(debugMode) { + System.out.printf("Loaded long rule (named %s) from path %s\n", + ruleName, ruleFile); + } + } + } catch (IOException ioex) { + System.out.printf("Error reading long rule (%s)\n", ioex.getMessage()); + } + } + + public void doLoadGrammar(String grammarName, String grammarPath) { + if(debugMode) { + System.out.printf("Loading grammar (named %s) from path %s\n", + grammarName, grammarPath); + } + + try (FileInputStream inputStream = new FileInputStream(grammarPath)) { + WeightedGrammar<String> newGram = + ServerGrammarReader.fromStream(inputStream).merge((gram, exports) -> { + for(String export : exports.toIterable()) { + if(debugMode) { + System.out.printf("\tLoaded exported rule %s from grammar %s\n", + export, grammarName); + + if(exportedRules.containsKey(export)) { + System.out.printf("\tWarning: Exported rule %s from grammar %s" + + " shadows a pre-existing rule\n", export, grammarName); + } + } + + exportedRules.put(export, gram); + } + + return gram; + }); + + loadedGrammars.put(grammarName, newGram); + } catch (IOException ioex) { + System.out.printf("? Error reading grammar from file" + + " (reason: %s)\n", ioex.getMessage()); + } + + if(debugMode) { + System.out.printf("Loaded grammar (named %s) from path %s\n", + grammarName, grammarPath); + } + } + + public void doGenerateExportedRule(String ruleName) { + String ruleResult = ListUtils.collapseTokens( + exportedRules.get(ruleName) + .generateListValues(ruleName, " ")); + + System.out.println("Generated Result: "); + System.out.println("\t" + ruleResult.replaceAll("\\s+", " ")); + } + + public void doGenerateGrammar(String currentGram, String ruleName) { + doGenerateGrammar(loadedGrammars.get(currentGram), ruleName); + } + + public void doGenerateGrammar(WeightedGrammar<String> currentGram, String ruleName) { + String ruleResult = ListUtils.collapseTokens( + currentGram.generateListValues(ruleName, " ")); + + System.out.println("Generated Result: "); + System.out.println("\t" + ruleResult.replaceAll("\\s+", " ")); + } + + public void doShowExportedRules() { + System.out.printf("Currently exported rules (%d total):\n", + exportedRules.getSize()); + + exportedRules.forEachKey(key -> { + System.out.println("\t" + key); + }); + } + + public void doShowGrammarRules(String gramName) { + WeightedGrammar<String> gram = loadedGrammars.get(gramName); + + IList<String> ruleNames = gram.getRuleNames(); + + System.out.printf("Rules for grammar %s (%d total)\n", + gramName, ruleNames.getSize()); + + ruleNames.forEach(rule -> { + System.out.println("\t" + rule); + }); + } + + public void doShowLoadedGrammars() { + System.out.printf("Currently loaded grammars (%d total):\n", + loadedGrammars.getSize()); + + loadedGrammars.forEachKey(key -> { + System.out.println("\t" + key); + }); + } + + public void doStressTest(int count) { + exportedRules.forEachKey(key -> { + doStressTest(key, count); + }); + } + + public void doStressTest(String ruleName, int count) { + doStressTest(exportedRules.get(ruleName), ruleName, count); + } + + public void doStressTest(WeightedGrammar<String> gram, String ruleName, int count) { + if(debugMode) System.out.println("Stress-testing rule " + ruleName); + + IList<String> res = new FunctionalList<>(); + IList<String> foundTags = new FunctionalList<>(); + + boolean foundBroken = false; + + for(int i = 0; i < count; i++) { + res = gram.generateListValues(ruleName, " "); + + for(String tok : res.toIterable()) { + if(tok.matches("\\[\\S+\\]") && !foundTags.contains(tok)) { + System.out.println("\tWarning: Possible un-expanded rule " + tok + " found" + + " in expansion of " + ruleName); + + doFindRule(tok); + + foundBroken = true; + + foundTags.add(tok); + } + } + } + + if(debugMode) { + if(!foundBroken) System.out.printf("Rule %s succesfully passed stress-testing\n", ruleName); + else System.out.printf("Rule %s failed stress-testing\n", ruleName); + } + } + + private void doFindRule(String ruleName) { + loadedGrammars.forEach((gramName, gram) -> { + if(gram.hasRule(ruleName)) { + System.out.printf("\t\tFound rule %s in grammar %s\n", ruleName, gramName); + } + }); + } +} diff --git a/RGens/src/main/java/bjc/RGens/server/ServerGrammarReader.java b/RGens/src/main/java/bjc/RGens/server/ServerGrammarReader.java index 62c59ff..da42b36 100644 --- a/RGens/src/main/java/bjc/RGens/server/ServerGrammarReader.java +++ b/RGens/src/main/java/bjc/RGens/server/ServerGrammarReader.java @@ -139,80 +139,40 @@ public class ServerGrammarReader { System.out.println(); } - private static void doCase(FunctionalStringTokenizer tokenizer, ReaderState state) { - int ruleProbability = readOptionalProbability(tokenizer, state); - - state.addCase(ruleProbability, tokenizer.toList()); - } - - private static void initialRule(FunctionalStringTokenizer tokenizer, ReaderState state) { - String initialRuleName = tokenizer.nextToken(); - - state.setInitialRule(initialRuleName); - } - private static IList<String> handleSpecialRule(IMap<String, IList<String>> vars, String strang, WeightedGrammar<String> gramm) { IList<String> retList = new FunctionalList<>(); if(strang.matches("\\[\\[\\S+\\]\\]")) { - if(strang.matches("\\[\\S+:=\\S+\\]")) { - String[] varParts = strang.split(":="); - - String varName = varParts[0].substring(1); - String ruleName = varParts[1].substring(0, varParts[1].length()); - - IList<String> varValue = gramm.generateGenericValues( - ruleName, (s) -> s, " "); - - vars.put(varName, varValue); + if(strang.matches("\\[\\[\\S+:=\\S+\\]\\]")) { + doDefineExpandedVariable(vars, strang, gramm); } else if(strang.matches("\\[\\[\\S+=\\S+\\]\\]")) { - String[] varParts = strang.split("="); - - String varName = varParts[0].substring(1); - String varValue = varParts[1].substring(0, varParts[1].length()); - - vars.put(varName, new FunctionalList<>(varValue)); + doDefineVariable(vars, strang); } else if(strang.matches("\\[\\[\\S+\\]\\]")) { - String ruleName = strang.substring(1, strang.length() - 1); - - IList<String> ruleValue = gramm.generateGenericValues( - ruleName, (s) -> s, ""); + if(GrammarServerEngine.debugMode) { + if(strang.contains("+")) + System.out.println("Double-triggering no-space rule " + strang); + else + System.out.println("Triggered alternate no-space rule " + strang); + } + + doNoSpaceRule(strang, gramm, retList); + } else if (strang.contains("+")) { + if(GrammarServerEngine.debugMode) + System.out.println("Triggered alternate no-space rule " + strang); - retList.add(ListUtils.collapseTokens(ruleValue)); + doNoSpaceRule(strang, gramm, retList); } else { // @FIXME notify the user they did something wrong retList.add(strang); } } else { - if(strang.matches("\\[\\$\\S+\\]")) { + if(strang.matches("\\[\\$\\S+\\-\\S+\\]")) { + retList = doExpandVariableReference(vars, strang, gramm); + } else if(strang.matches("\\[\\$\\S+\\]")) { String varName = strang.substring(2, strang.length()); retList = vars.get(varName); - } else if(strang.matches("\\[\\$\\S+\\-\\S+\\]")) { - String[] varParts = strang.substring(1, strang.length()).split("-"); - - StringBuilder actualName = new StringBuilder("["); - - for(String varPart : varParts) { - if(varPart.startsWith("$")) { - IList<String> varName = vars.get(varPart.substring(1)); - - if(varName.getSize() != 1) { - // @FIXME notify the user they did something wrong - } - - actualName.append(varName.first() + "-"); - } else { - actualName.append(varPart + "-"); - } - } - - // Trim trailing - - actualName.deleteCharAt(actualName.length() - 1); - actualName.append("]"); - - retList = gramm.generateGenericValues(actualName.toString(), (s) -> s, " "); } else if (exportedRules.containsKey(strang) && exportedRules.get(strang) != gramm && !gramm.hasRule(strang)) { @@ -222,6 +182,8 @@ public class ServerGrammarReader { WeightedGrammar<String> exportGram = exportedRules.get(strang); retList = exportGram.generateGenericValues(strang, (s) -> s, " "); + } else if (strang.contains("+")) { + doNoSpaceRule(strang, gramm, retList); } else { // @FIXME notify the user they did something wrong retList.add(strang); @@ -263,10 +225,10 @@ public class ServerGrammarReader { return Integer.parseInt(tokenizer.nextToken()); } - private static void removeRule(FunctionalStringTokenizer tokenizer, ReaderState state) { - String ruleName = tokenizer.nextToken(); - - state.deleteRule(ruleName); + private static void initialRule(FunctionalStringTokenizer tokenizer, ReaderState state) { + String initialRuleName = tokenizer.nextToken(); + + state.setInitialRule(initialRuleName); } private static void prefixRule(FunctionalStringTokenizer tokenizer, ReaderState state) { @@ -278,6 +240,12 @@ public class ServerGrammarReader { state.prefixRule(ruleName, prefixToken, additionalProbability); } + private static void removeRule(FunctionalStringTokenizer tokenizer, ReaderState state) { + String ruleName = tokenizer.nextToken(); + + state.deleteRule(ruleName); + } + private static void suffixRule(FunctionalStringTokenizer tokenizer, ReaderState state) { String ruleName = tokenizer.nextToken(); String suffixToken = tokenizer.nextToken(); @@ -286,4 +254,73 @@ public class ServerGrammarReader { state.suffixRule(ruleName, suffixToken, additionalProbability); } + + private static void doCase(FunctionalStringTokenizer tokenizer, ReaderState state) { + int ruleProbability = readOptionalProbability(tokenizer, state); + + state.addCase(ruleProbability, tokenizer.toList()); + } + + private static void doDefineExpandedVariable(IMap<String, IList<String>> vars, String strang, WeightedGrammar<String> gramm) { + String[] varParts = strang.split(":="); + + String varName = varParts[0].substring(2); + String ruleName = varParts[1].substring(0, varParts[1].length() - 2); + + IList<String> varValue = gramm.generateGenericValues( + ruleName, (s) -> s, " "); + + vars.put(varName, varValue); + } + + private static void doDefineVariable(IMap<String, IList<String>> vars, String strang) { + String[] varParts = strang.split("="); + + String varName = varParts[0].substring(2); + String varValue = varParts[1].substring(0, varParts[1].length() - 2); + + vars.put(varName, new FunctionalList<>(varValue)); + } + + private static IList<String> doExpandVariableReference(IMap<String, IList<String>> vars, String strang, + WeightedGrammar<String> gramm) { + IList<String> retList; + String[] varParts = strang.substring(1, strang.length()).split("-"); + + StringBuilder actualName = new StringBuilder("["); + + for(String varPart : varParts) { + if(varPart.startsWith("$")) { + IList<String> varName = vars.get(varPart.substring(1)); + + if(varName.getSize() != 1) { + // @FIXME notify the user they did something wrong + } + + actualName.append(varName.first() + "-"); + } else { + actualName.append(varPart + "-"); + } + } + + // Trim trailing - + actualName.deleteCharAt(actualName.length() - 1); + + retList = gramm.generateGenericValues(actualName.toString(), (s) -> s, " "); + return retList; + } + + private static void doNoSpaceRule(String strang, WeightedGrammar<String> gramm, IList<String> retList) { + if(!GrammarServerEngine.debugMode) { + IList<String> ruleValue = gramm.generateGenericValues( + strang, (s) -> s.trim(), ""); + + retList.add(ListUtils.collapseTokens(ruleValue)); + } else { + // if(!gramm.hasRule(strang)) + // System.out.println("Warning: Possible unexpanded rule " + strang); + + retList.add(strang); + } + } } |