From f25d1062a56a81b17348b799e6d4d7e1dc12a1cc Mon Sep 17 00:00:00 2001 From: "Benjamin J. Culkin" Date: Tue, 5 Jun 2018 15:52:44 -0300 Subject: Templates pt.2 More work that leads towards getting templates working --- src/main/java/bjc/rgens/parser/ConfigLoader.java | 73 +++++++++++-- src/main/java/bjc/rgens/parser/ConfigSet.java | 3 + .../java/bjc/rgens/parser/GenerationState.java | 40 ++++++-- .../java/bjc/rgens/parser/GrammarTemplate.java | 18 ++++ src/main/java/bjc/rgens/parser/RGrammar.java | 24 +++-- src/main/java/bjc/rgens/parser/RGrammarSet.java | 5 + src/main/java/bjc/rgens/parser/RGrammarTest.java | 113 +++++++++++---------- src/main/java/bjc/rgens/parser/RGrammars.java | 27 +++-- src/main/java/bjc/rgens/parser/Rule.java | 2 +- .../parser/elements/ExpVariableCaseElement.java | 21 +--- .../rgens/parser/elements/RangeCaseElement.java | 4 +- .../bjc/rgens/parser/elements/RuleCaseElement.java | 46 ++++----- .../parser/elements/RuleVariableCaseElement.java | 31 ++---- 13 files changed, 248 insertions(+), 159 deletions(-) (limited to 'src/main/java/bjc') diff --git a/src/main/java/bjc/rgens/parser/ConfigLoader.java b/src/main/java/bjc/rgens/parser/ConfigLoader.java index 26c5f66..f3028fe 100644 --- a/src/main/java/bjc/rgens/parser/ConfigLoader.java +++ b/src/main/java/bjc/rgens/parser/ConfigLoader.java @@ -20,9 +20,14 @@ public class ConfigLoader { * @throws IOException * If something goes wrong during configuration loading. */ - public static RGrammarSet fromConfigFile(Path cfgFile) throws IOException { - /* The grammar set to hand back. */ + public static ConfigSet fromConfigFile(Path cfgFile) throws IOException { + ConfigSet cfgSet = new ConfigSet(); + + /* The grammar set we're parsing into. */ RGrammarSet set = new RGrammarSet(); + cfgSet.grammars.put("default", set); + set.belongsTo = cfgSet; + set.name = "default"; long startCFGTime = System.nanoTime(); @@ -40,15 +45,15 @@ public class ConfigLoader { try { /* Ignore blank/comment lines. */ - if (ln.equals("")) return; + if (ln.equals("")) continue; - if (ln.startsWith("#")) return; + if (ln.startsWith("#")) continue; /* Handle mixed whitespace. */ ln = ln.replaceAll("\\s+", " "); /* Get line type */ - int typeIdx = ln.indexOf(" '); + int typeIdx = ln.indexOf(" "); if(typeIdx == -1) { throw new GrammarException("Must specify config line type"); } @@ -83,7 +88,7 @@ public class ConfigLoader { System.err.printf("\n\nPERF: Read config file %s in %d ns (%f s)\n", cfgFile, cfgDur, cfgDur / 1000000000.0); - return set; + return cfgSet; } private static void loadConfigLine(String ln, RGrammarSet set, Path cfgParent) throws IOException { @@ -112,8 +117,60 @@ public class ConfigLoader { switch(tag) { case "template": { - throw new GrammarException("Templates aren't implemented yet"); + Path path = Paths.get(ln); + + /* + * Convert from configuration relative path to + * absolute path. + */ + Path convPath = cfgParent.resolve(path.toString()); + + if(Files.isDirectory(convPath)) { + throw new GrammarException("Can't load grammar from directory" + convPath.toString()); + } else { + /* Load template file. */ + try { + long startFileTime = System.nanoTime(); + + BufferedReader fis = Files.newBufferedReader(convPath); + GrammarTemplate template = GrammarTemplate.readTemplate(fis); + if(template.name == null) { + System.err.printf("\tINFO: Naming unnamed template loaded from %s off config name '%s'\n", + convPath, name); + + template.name = name; + } + + fis.close(); + + long endFileTime = System.nanoTime(); + + long fileTime = endFileTime - startFileTime; + + System.err.printf("\tPERF: Read template %s (from %s) in %d ns (%f s)\n", + gram.name, convPath, fileTime, fileTime / 1000000000.0); + + /* Add grammar to the set. */ + cfgSet.templates.put(name, gram); + + /* + * @NOTE + * + * Do we need to do this + * for templates? + * + * Mark where the + * template came + * from. + */ + //set.loadedFrom.put(name, path.toString()); + } catch (GrammarException gex) { + String msg = String.format("Error loading file '%s'", path); + throw new GrammarException(msg, gex); + } + } } + break; case "subset": { /* @TODO implement subset grammars */ @@ -140,7 +197,7 @@ public class ConfigLoader { BufferedReader fis = Files.newBufferedReader(convPath); RGrammar gram = RGrammarParser.readGrammar(fis); if(gram.name == null) { - System.err.printf("\tINFO: Naming unnamed grammar loaded from %s off config name %s\n", + System.err.printf("\tINFO: Naming unnamed grammar loaded from %s off config name '%s'\n", convPath, name); gram.name = name; diff --git a/src/main/java/bjc/rgens/parser/ConfigSet.java b/src/main/java/bjc/rgens/parser/ConfigSet.java index 949962c..968cc69 100644 --- a/src/main/java/bjc/rgens/parser/ConfigSet.java +++ b/src/main/java/bjc/rgens/parser/ConfigSet.java @@ -1,5 +1,8 @@ package bjc.rgens.parser; +import java.util.HashMap; +import java.util.Map; + public class ConfigSet { public final Map grammars; public final Map templates; diff --git a/src/main/java/bjc/rgens/parser/GenerationState.java b/src/main/java/bjc/rgens/parser/GenerationState.java index 5e2b449..38a25d5 100644 --- a/src/main/java/bjc/rgens/parser/GenerationState.java +++ b/src/main/java/bjc/rgens/parser/GenerationState.java @@ -16,12 +16,6 @@ public class GenerationState { /** The RNG. */ public Random rnd; - /* - * @NOTE - * - * Once the planned refactor for importing rules happens, this will need - * to be changed. - */ /** The current grammar. */ public RGrammar gram; /** The rules of the grammar. */ @@ -29,11 +23,6 @@ public class GenerationState { /** The rules imported from other grammars. */ public Map importRules; - /* - * @NOTE - * - * For planned features, this will need to be refactored. - */ /** The current set of variables. */ public Map vars; public Map> rlVars; @@ -64,6 +53,8 @@ public class GenerationState { } public void swapGrammar(RGrammar gram) { + if(this.gram == gram) return; + this.gram = gram; rules = gram.getRules(); @@ -74,4 +65,31 @@ public class GenerationState { public GenerationState newBuf() { return new GenerationState(new StringBuilder(), rnd, vars, rlVars, gram); } + + /* + * @TODO 6/5/18 Ben Culkin :ImportRefactor + * + * Change this so that imports in almost all cases have to specify where + * they are importing the rule from, so as to make it clear which rules + * are imported, and which aren't + */ + public IPair findRule(String ruleName, boolean allowImports) { + if(rules.containsKey(ruleName)) { + return new Pair<>(gram, rules.get(ruleName)); + } + + if(allowImports) return findImport(ruleName); + + return null; + } + + public IPair findImport(String ruleName) { + if(importRules.containsKey(ruleName)) { + RGrammar imp = importRules.get(ruleName); + + return new Pair<>(imp, imp.rules.get(ruleName)); + } + + return null; + } } diff --git a/src/main/java/bjc/rgens/parser/GrammarTemplate.java b/src/main/java/bjc/rgens/parser/GrammarTemplate.java index 8d13877..1bd5c53 100644 --- a/src/main/java/bjc/rgens/parser/GrammarTemplate.java +++ b/src/main/java/bjc/rgens/parser/GrammarTemplate.java @@ -1,5 +1,23 @@ package bjc.rgens.parser; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; + public class GrammarTemplate { + public String name; + public final List lines; + + public GrammarTemplate(List lines) { + this.lines = lines; + } + + public static GrammarTemplate readTemplate(Reader rdr) { + List lines = new ArrayList<>(); + + GrammarTemplate template = new GrammarTemplate(lines); + + return template; + } } diff --git a/src/main/java/bjc/rgens/parser/RGrammar.java b/src/main/java/bjc/rgens/parser/RGrammar.java index 381eac1..5ca26a4 100755 --- a/src/main/java/bjc/rgens/parser/RGrammar.java +++ b/src/main/java/bjc/rgens/parser/RGrammar.java @@ -31,6 +31,8 @@ import edu.gatech.gtri.bktree.MutableBkTree; * @author EVE */ public class RGrammar { + public RGrammarSet belongsTo; + public String name; /* The max distance between possible alternate rules. */ @@ -54,7 +56,7 @@ public class RGrammar { } /* The rules of the grammar. */ - private Map rules; + public Map rules; /* The rules imported from other grammars. */ private Map importRules; /* The rules exported from this grammar. */ @@ -158,10 +160,16 @@ public class RGrammar { } } - Rule rl = rules.get(fromRule); + /* + * We don't search imports, so it will always belong to this + * grammar. + */ + Rule rl = state.findRule(fromRule, false).getRight(); + if(rl == null) + throw new GrammarException("Could not find rule " + rl.name); if(rl.doRecur()) { - RuleCase start = rules.get(fromRule).getCase(state.rnd); + RuleCase start = rl.getCase(state.rnd); System.err.printf("\tFINE: Generating %s (from %s in %s)\n", start, fromRule, name); generateCase(start, state); @@ -177,12 +185,11 @@ public class RGrammar { * * Do we want to perform this post-processing here, or elsewhere? */ - String body = state.contents.toString(); + return postprocessRes(state.contents.toString()); + } - /* - * Collapse duplicate spaces. - */ - body = body.replaceAll("\\s+", " "); + private String postprocessRes(String strang) { + String body = strang.replaceAll("\\s+", " "); /* * Remove extraneous spaces around punctutation marks. @@ -221,7 +228,6 @@ public class RGrammar { return body.trim(); } - /** * Generate a rule case. * diff --git a/src/main/java/bjc/rgens/parser/RGrammarSet.java b/src/main/java/bjc/rgens/parser/RGrammarSet.java index b84f5f0..a192da7 100755 --- a/src/main/java/bjc/rgens/parser/RGrammarSet.java +++ b/src/main/java/bjc/rgens/parser/RGrammarSet.java @@ -11,6 +11,10 @@ import java.util.Set; * @author EVE */ public class RGrammarSet { + public String name; + + public ConfigSet belongsTo; + /* Contains all the grammars in this set. */ private Map grammars; @@ -59,6 +63,7 @@ public class RGrammarSet { } grammars.put(grammarName, gram); + gram.belongsTo = this; /* Process exports from the grammar. */ for (Rule export : gram.getExportedRules()) { diff --git a/src/main/java/bjc/rgens/parser/RGrammarTest.java b/src/main/java/bjc/rgens/parser/RGrammarTest.java index 0238fc0..25f76e7 100755 --- a/src/main/java/bjc/rgens/parser/RGrammarTest.java +++ b/src/main/java/bjc/rgens/parser/RGrammarTest.java @@ -24,71 +24,74 @@ public class RGrammarTest { try { /* Load a grammar set. */ Path cfgPath = Paths.get(rsc.toURI()); - RGrammarSet gramSet = ConfigLoader.fromConfigFile(cfgPath); + ConfigSet cfgSet = ConfigLoader.fromConfigFile(cfgPath); - /* Generate rule suggestions for all the grammars in the set. */ - for (String gramName : gramSet.getGrammars()) { - long startSuggTime = System.nanoTime(); + for(RGrammarSet gramSet : cfgSet.grammars.values()) { + testGrammarSet(gramSet); + } + } catch (IOException ioex) { + ioex.printStackTrace(); + } catch (URISyntaxException urisex) { + urisex.printStackTrace(); + } + } - gramSet.getGrammar(gramName).generateSuggestions(); + private static void testGrammarSet(RGrammarSet gramSet) { + /* Generate rule suggestions for all the grammars in the set. */ + for (String gramName : gramSet.getGrammars()) { + long startSuggTime = System.nanoTime(); - long endSuggTime = System.nanoTime(); + gramSet.getGrammar(gramName).generateSuggestions(); - long suggDur = endSuggTime - startSuggTime; + long endSuggTime = System.nanoTime(); - if(gramSet.PERF) - System.err.printf("PERF: Generated rule suggestions for %s in %d ns (%f s)\n", gramName, suggDur, suggDur / 1000000000.0); - } + long suggDur = endSuggTime - startSuggTime; + + System.err.printf("PERF: Generated rule suggestions for %s in %d ns (%f s)\n", gramName, suggDur, suggDur / 1000000000.0); + } + + System.err.printf("\n\n"); + + /* Generate for each exported rule. */ + for (String exportName : gramSet.getExportedRules()) { + /* Where we loaded the rule from. */ + String loadSrc = gramSet.loadedFrom(gramSet.exportedFrom(exportName)); - if(gramSet.PERF) - System.err.printf("\n\n"); - - /* Generate for each exported rule. */ - for (String exportName : gramSet.getExportedRules()) { - /* Where we loaded the rule from. */ - String loadSrc = gramSet.loadedFrom(gramSet.exportedFrom(exportName)); - - System.out.println(); - System.out.printf("Generating for exported rule '%s' from file '%s'\n", exportName, loadSrc); - - RGrammar grammar = gramSet.getExportSource(exportName); - long startGenTime = System.nanoTime(); - for (int i = 0; i < 100; i++) { - try { - String res = grammar.generate(exportName); - if(exportName.contains("+")) res = res.replaceAll("\\s+", ""); - - if(res.length() > 120) { - System.out.printf("\t\n\tContents: %s\n\t\n", res); - } else { - System.out.printf("\tContents: %s\n", res); - } - } catch (GrammarException gex) { - /* Print out errors with generation. */ - String fmt = "Error in exported rule '%s' (loaded from '%s')\n"; - - System.out.printf(fmt, exportName, loadSrc); - System.out.println(); - System.out.println(); - - System.err.printf(fmt, exportName, loadSrc); - gex.printStackTrace(); - - System.err.println(); - System.err.println(); + System.out.println(); + System.out.printf("Generating for exported rule '%s' from file '%s'\n", exportName, loadSrc); + + RGrammar grammar = gramSet.getExportSource(exportName); + long startGenTime = System.nanoTime(); + for (int i = 0; i < 100; i++) { + try { + String res = grammar.generate(exportName); + if(exportName.contains("+")) res = res.replaceAll("\\s+", ""); + + if(res.length() > 120) { + System.out.printf("\t\n\tContents: %s\n\t\n", res); + } else { + System.out.printf("\tContents: %s\n", res); } - } - long endGenTime = System.nanoTime(); + } catch (GrammarException gex) { + /* Print out errors with generation. */ + String fmt = "ERROR: Exported rule %s from %s failed (loaded from '%s')\n"; - long genDur = endGenTime - startGenTime; + System.out.printf(fmt, exportName, grammar.name, loadSrc); + System.out.println(); + System.out.println(); - if(gramSet.PERF) - System.err.printf("PERF: Generated %s 100 times in %d ns (%f s)\n\n\n", exportName, genDur, genDur / 1000000000.0); + System.err.printf(fmt, exportName, grammar.name, loadSrc); + gex.printStackTrace(); + + System.err.println(); + System.err.println(); + } } - } catch (IOException ioex) { - ioex.printStackTrace(); - } catch (URISyntaxException urisex) { - urisex.printStackTrace(); + long endGenTime = System.nanoTime(); + + long genDur = endGenTime - startGenTime; + + System.err.printf("PERF: Generated %s 100 times in %d ns (%f s)\n\n\n", exportName, genDur, genDur / 1000000000.0); } } } diff --git a/src/main/java/bjc/rgens/parser/RGrammars.java b/src/main/java/bjc/rgens/parser/RGrammars.java index ad94388..cc31bad 100755 --- a/src/main/java/bjc/rgens/parser/RGrammars.java +++ b/src/main/java/bjc/rgens/parser/RGrammars.java @@ -16,7 +16,7 @@ import java.util.Map; * @author Ben Culkin */ public class RGrammars { - private static RGrammarSet gramSet; + private static ConfigSet cfgSet; private static void loadSet() { try { @@ -24,12 +24,13 @@ public class RGrammars { Map env = new HashMap<>(); env.put("create", "true"); + /* Ensure we can get at the file we need */ @SuppressWarnings("unused") FileSystem zipfs = FileSystems.newFileSystem(rsc, env); Path cfgPath = Paths.get(rsc); - gramSet = ConfigLoader.fromConfigFile(cfgPath); + cfgSet = ConfigLoader.fromConfigFile(cfgPath); } catch (IOException | URISyntaxException ex) { RuntimeException rtex = new RuntimeException("Could not load grammars"); @@ -49,19 +50,23 @@ public class RGrammars { * If something went wrong. */ public static String generateExport(String exportName) throws GrammarException { - if (gramSet == null) + if (cfgSet == null) loadSet(); - if (!gramSet.getExportedRules().contains(exportName)) { - throw new GrammarException(String.format("No exported rule named %s", exportName)); - } + for(RGrammarSet gramSet : cfgSet.grammars.values()) { + if (!gramSet.getExportedRules().contains(exportName)) { + continue; + } + + RGrammar gram = gramSet.getExportSource(exportName); - RGrammar gram = gramSet.getExportSource(exportName); + String res = gram.generate(exportName); + if (exportName.contains("+")) + res = res.replaceAll("\\s+", ""); - String res = gram.generate(exportName); - if (exportName.contains("+")) - res = res.replaceAll("\\s+", ""); + return res; + } - return res; + throw new GrammarException(String.format("No exported rule named %s", exportName)); } } diff --git a/src/main/java/bjc/rgens/parser/Rule.java b/src/main/java/bjc/rgens/parser/Rule.java index b5a87c3..06dedac 100755 --- a/src/main/java/bjc/rgens/parser/Rule.java +++ b/src/main/java/bjc/rgens/parser/Rule.java @@ -185,7 +185,7 @@ public class Rule { @Override public String toString() { - return String.format("Rule [ruleName='%s', ruleCases=%s]", name, cases); + return String.format("Rule '%s' with %d cases", name, cases.getValues().getSize()); } public boolean doRecur() { diff --git a/src/main/java/bjc/rgens/parser/elements/ExpVariableCaseElement.java b/src/main/java/bjc/rgens/parser/elements/ExpVariableCaseElement.java index 455fce6..ae85139 100755 --- a/src/main/java/bjc/rgens/parser/elements/ExpVariableCaseElement.java +++ b/src/main/java/bjc/rgens/parser/elements/ExpVariableCaseElement.java @@ -1,5 +1,7 @@ package bjc.rgens.parser.elements; +import bjc.utils.data.IPair; + import bjc.rgens.parser.GenerationState; import bjc.rgens.parser.GrammarException; import bjc.rgens.parser.RecurLimitException; @@ -16,24 +18,11 @@ public class ExpVariableCaseElement extends VariableCaseElement { public void generate(GenerationState state) { GenerationState newState = state.newBuf(); - if (state.rules.containsKey(varDef)) { - Rule rl = state.rules.get(varDef); - - if(rl.doRecur()) { - RuleCase destCase = state.rules.get(varDef).getCase(); - System.err.printf("\tFINE: Generating %s (from %s)\n", destCase, varDef); - - state.gram.generateCase(destCase, newState); - - rl.endRecur(); - } else { - throw new RecurLimitException("Rule recurrence limit exceeded"); - } - } else if (state.importRules.containsKey(varDef)) { - RGrammar destGrammar = state.importRules.get(varDef); + IPair par = state.findRule(varDef, true); + if(par != null) { + RGrammar destGrammar = par.getLeft(); newState.swapGrammar(destGrammar); - String res = destGrammar.generate(varDef, state); /* diff --git a/src/main/java/bjc/rgens/parser/elements/RangeCaseElement.java b/src/main/java/bjc/rgens/parser/elements/RangeCaseElement.java index 7cebde9..cf8f161 100755 --- a/src/main/java/bjc/rgens/parser/elements/RangeCaseElement.java +++ b/src/main/java/bjc/rgens/parser/elements/RangeCaseElement.java @@ -14,8 +14,8 @@ public class RangeCaseElement extends CaseElement { } public void generate(GenerationState state) { - int val = state.rnd.nextInt(end - begin); - val += begin; + int val = state.rnd.nextInt(end - begin); + val += begin; state.contents.append(val); } diff --git a/src/main/java/bjc/rgens/parser/elements/RuleCaseElement.java b/src/main/java/bjc/rgens/parser/elements/RuleCaseElement.java index c299963..3e3d182 100755 --- a/src/main/java/bjc/rgens/parser/elements/RuleCaseElement.java +++ b/src/main/java/bjc/rgens/parser/elements/RuleCaseElement.java @@ -1,5 +1,8 @@ package bjc.rgens.parser.elements; +import bjc.utils.data.IPair; +import bjc.utils.data.Pair; + import bjc.rgens.parser.*; import java.util.regex.Matcher; @@ -23,35 +26,30 @@ public abstract class RuleCaseElement extends StringCaseElement { protected void doGenerate(String actName, GenerationState state) { GenerationState newState = state.newBuf(); + IPair par; + if (actName.startsWith("[^")) { actName = "[" + actName.substring(2); - RGrammar dst = state.importRules.get(actName); - - newState.swapGrammar(dst); - - /* :Postprocessing */ - newState.contents = new StringBuilder(dst.generate(actName, state.rnd, state.vars, state.rlVars)); - } else if (state.rules.containsKey(actName)) { - Rule rl = state.rules.get(actName); - - if(rl.doRecur()) { - RuleCase cse = rl.getCase(state.rnd); - System.err.printf("\tFINE: Generating %s (from %s)\n", cse, actName); - - state.gram.generateCase(cse, newState); - - rl.endRecur(); - } else { - throw new RecurLimitException("Rule recurrence limit exceeded"); - } - } else if (state.importRules.containsKey(actName)) { - RGrammar dst = state.importRules.get(actName); + par = state.findImport(actName); + } else { + par = state.findRule(actName, true); + } - newState.swapGrammar(dst); + if(par != null) { + RGrammar destGrammar = par.getLeft(); + newState.swapGrammar(destGrammar); + String res = destGrammar.generate(actName, newState); - /* :Postprocessing */ - newState.contents = new StringBuilder(dst.generate(actName, state.rnd, state.vars, state.rlVars)); + /* + * @NOTE + * + * :Postprocessing + * + * This is because generate() returns a processed + * string, but modifies the passed in StringBuilder. + */ + newState.contents = new StringBuilder(res); } else { /* * @TODO 5/29/18 Ben Culkin :RuleSuggesting diff --git a/src/main/java/bjc/rgens/parser/elements/RuleVariableCaseElement.java b/src/main/java/bjc/rgens/parser/elements/RuleVariableCaseElement.java index d4c677b..fa1783f 100644 --- a/src/main/java/bjc/rgens/parser/elements/RuleVariableCaseElement.java +++ b/src/main/java/bjc/rgens/parser/elements/RuleVariableCaseElement.java @@ -18,35 +18,22 @@ public class RuleVariableCaseElement extends VariableCaseElement { } public void generate(GenerationState state) { - Rule rl; - RGrammar grm; - - if(state.rules.containsKey(varDef)) { - rl = state.rules.get(varDef); - grm = state.gram; - } else if(state.importRules.containsKey(varDef)) { - grm = state.importRules.get(varDef); - rl = grm.getRules().get(varDef); - } else { + IPair par = state.findRule(varDef, true); + + if(par == null) { throw new GrammarException("Can't create variable referencing non-existent rule " + varDef); } - - - if(exhaust) rl = rl.exhaust(); - - if(state.rlVars.containsKey(varName)) { - IPair par = state.rlVars.get(varName); - - System.err.printf("WARN: Shadowing rule variable '%s' (%s with %s)\n", - varName, par.getRight().name, rl.name); + + if(exhaust) { + par = new Pair<>(par.getLeft(), par.getRight().exhaust()); } - state.rlVars.put(varName, new Pair<>(grm, rl)); + state.rlVars.put(varName, par); if(exhaust) { - System.err.printf("\t\tTRACE: Defined exhausted rulevar '%s' ('%s')\n", varName, varDef); + System.err.printf("\t\tFINE: Defined exhausted rulevar '%s' ('%s')\n", varName, varDef); } else { - System.err.printf("\t\tTRACE: Defined rulevar '%s' ('%s')\n", varName, varDef); + System.err.printf("\t\tFINE: Defined rulevar '%s' ('%s')\n", varName, varDef); } } } -- cgit v1.2.3