From 7d435bb5c0cee025d9c5c956ee88cebabaee19cb Mon Sep 17 00:00:00 2001 From: swamikevala Date: Thu, 25 Sep 2014 20:55:57 +0530 Subject: [PATCH 1/2] Implemented feature req #174 (Dump/Restore state) Implemented functionality to save intermediate state of genetic algorithm, so that we can restart from the same point we left off - by dumping full genetic population after each generation. Useful in the event of lost db connection or computer crashing. --- .../no/priv/garshol/duke/ConfigLoader.java | 5 +- .../no/priv/garshol/duke/genetic/Driver.java | 29 ++++++++-- .../duke/genetic/GeneticAlgorithm.java | 54 +++++++++++++++++-- .../duke/genetic/GeneticPopulation.java | 41 +++++++++++++- 4 files changed, 119 insertions(+), 10 deletions(-) diff --git a/src/main/java/no/priv/garshol/duke/ConfigLoader.java b/src/main/java/no/priv/garshol/duke/ConfigLoader.java index 18eb6dad..6bf061f1 100644 --- a/src/main/java/no/priv/garshol/duke/ConfigLoader.java +++ b/src/main/java/no/priv/garshol/duke/ConfigLoader.java @@ -251,8 +251,9 @@ else if (localName.equals("comparator")) { datasource = null; currentobj = null; } else if (localName.equals("object")) { - comparator = (Comparator) currentobj; - config.addCustomComparator(comparator); + if (currentobj instanceof Comparator) + // store custom comparators so genetic algorithm can get them + config.addCustomComparator((Comparator) currentobj); currentobj = null; } else if (localName.equals("database")) diff --git a/src/main/java/no/priv/garshol/duke/genetic/Driver.java b/src/main/java/no/priv/garshol/duke/genetic/Driver.java index e99fa8b3..c936b652 100644 --- a/src/main/java/no/priv/garshol/duke/genetic/Driver.java +++ b/src/main/java/no/priv/garshol/duke/genetic/Driver.java @@ -17,7 +17,7 @@ public class Driver { public static void main(String[] argv) throws IOException, SAXException { // parse command-line CommandLineParser parser = new CommandLineParser(); - parser.setMinimumArguments(1); + parser.setMinimumArguments(0); parser.setMaximumArguments(1); parser.addStringOption("testfile", 'T'); parser.addBooleanOption("scientific", 's'); @@ -25,6 +25,8 @@ public static void main(String[] argv) throws IOException, SAXException { parser.addStringOption("population", 'P'); parser.addStringOption("questions", 'Q'); parser.addStringOption("output", 'O'); + parser.addStringOption("dump-state", 'D'); + parser.addStringOption("restore-state", 'R'); parser.addStringOption("threads", 't'); parser.addBooleanOption("active", 'A'); parser.addStringOption("linkfile", 'l'); @@ -48,9 +50,24 @@ public static void main(String[] argv) throws IOException, SAXException { System.err.println("ERROR: scientific mode requires a test file"); System.exit(1); } + + String restoreStateDir = parser.getOptionValue("restore-state"); + if (argv.length == 0 && restoreStateDir == null) { + System.err.println("ERROR: must specify a config file or be in restore-state mode"); + System.exit(1); + } + + if (argv.length == 1 && restoreStateDir != null) { + System.out.println("WARNING: cannot specify a config file and be in restore-state mode. Ignoring restore-state option"); + } // get started - Configuration config = ConfigLoader.load(argv[0]); + Configuration config; + if (parser.getOptionValue("restore-state") != null) + config = ConfigLoader.load(restoreStateDir + "//config_1.xml"); + else + config = ConfigLoader.load(argv[0]); + GeneticAlgorithm genetic = new GeneticAlgorithm(config, testfile, parser.getOptionState("scientific")); @@ -58,13 +75,16 @@ public static void main(String[] argv) throws IOException, SAXException { genetic.setGenerations(parser.getOptionInteger("generations", 100)); genetic.setQuestions(parser.getOptionInteger("questions", 10)); genetic.setConfigOutput(parser.getOptionValue("output")); + genetic.setDumpStateDir(parser.getOptionValue("dump-state")); + genetic.setRestoreStateDir(parser.getOptionValue("restore-state")); genetic.setThreads(parser.getOptionInteger("threads", 1)); genetic.setSparse(parser.getOptionState("sparse")); genetic.setMutationRate(parser.getOptionInteger("mutation-rate", -1)); genetic.setRecombinationRate(parser.getOptionDouble("recombination-rate", -1.0)); genetic.setEvolveComparators(!parser.getOptionState("no-comparators")); - genetic.setCopiesOfOriginal(parser.getOptionInteger("original", 0)); genetic.setIncompleteTest(parser.getOptionState("incomplete-data")); + //Meaningless when restoring full population from saved state + genetic.setCopiesOfOriginal((restoreStateDir == null) ? parser.getOptionInteger("original", 0) : 0); if (parser.getOptionState("active")) genetic.setActive(true); if (parser.getOptionValue("linkfile") != null) @@ -84,6 +104,9 @@ private static void usage() { System.out.println(" --sparse don't ask questions after every generation"); System.out.println(" --output= file to write best configuration to"); System.out.println(" (a new export after every generation)"); + System.out.println(" --dump-state= directory to dump the current state to"); + System.out.println(" (dumps after every generation)"); + System.out.println(" --restore-state= directory to load state from (restart from saved state)"); System.out.println(" --threads=N number of threads to run"); System.out.println(" --linkfile= write user's answers to this file"); System.out.println(" --scientific test active learning"); diff --git a/src/main/java/no/priv/garshol/duke/genetic/GeneticAlgorithm.java b/src/main/java/no/priv/garshol/duke/genetic/GeneticAlgorithm.java index 128ba36c..35b6edb9 100644 --- a/src/main/java/no/priv/garshol/duke/genetic/GeneticAlgorithm.java +++ b/src/main/java/no/priv/garshol/duke/genetic/GeneticAlgorithm.java @@ -8,8 +8,11 @@ import java.util.Comparator; import java.util.Collection; import java.util.Collections; +import java.io.File; import java.io.IOException; +import org.xml.sax.SAXException; + import no.priv.garshol.duke.Link; import no.priv.garshol.duke.Record; import no.priv.garshol.duke.Database; @@ -43,6 +46,8 @@ public class GeneticAlgorithm { private boolean scientific; private Oracle oracle; private String outfile; // file to write config to + private String dumpStateDir; // file to dump state to + private String restoreStateDir; // file to restore state from private Map sciencetracker; private boolean quiet; // limit output private boolean incomplete; // is test file incomplete? @@ -123,6 +128,21 @@ public void setConfigOutput(String output) { this.outfile = output; } + /** + * Set the directory to dump the state to. The full genetic + * population gets dumped at the end of each generation. + */ + public void setDumpStateDir(String dumpDir) { + this.dumpStateDir = dumpDir; + } + + /** + * Set the directory to restore the state from. + */ + public void setRestoreStateDir(String restoreDir) { + this.restoreStateDir = restoreDir; + } + /** * Sets the number of threads to run the genetic algorithm in. */ @@ -239,9 +259,19 @@ public void run() { } } - // make first, random population - population.create(); - + // make first population (either random, or restored from previously saved state) + if (restoreStateDir == null) + population.create(); + else + try { + population.restore(restoreStateDir); + } + catch (IOException e) { + System.err.println("ERROR: Cannot read files from " + restoreStateDir + ": " + e); + } catch (SAXException e) { + System.err.println("ERROR: Cannot parse xml files from " + restoreStateDir + ": " + e); + } + // run through the required number of generations double prevbest = 0.0; int stuck_for = 0; // number of generations f has remained unchanged @@ -337,6 +367,24 @@ else if (gen_no > 1) } } + // if asked to, dump state + if (dumpStateDir != null) { + String dumpPath = null; + try { + File d = new File(dumpStateDir); + d.mkdirs(); + Configuration cfg; + for (int c = 0; c < population.size(); c++) { + cfg = population.getNthConfiguration(c).getConfiguration(); + dumpPath = dumpStateDir + "//config_" + (c+1) + ".xml"; + ConfigWriter.write(cfg, dumpPath); + } + cfg = null; + } catch (IOException e) { + System.err.println("ERROR: Cannot write to '" + dumpPath + "': " + e); + } + } + // is there any point in evolving? if (active && population.getBestConfiguration().getFNumber() == diff --git a/src/main/java/no/priv/garshol/duke/genetic/GeneticPopulation.java b/src/main/java/no/priv/garshol/duke/genetic/GeneticPopulation.java index 4fd3490b..6c545cbb 100644 --- a/src/main/java/no/priv/garshol/duke/genetic/GeneticPopulation.java +++ b/src/main/java/no/priv/garshol/duke/genetic/GeneticPopulation.java @@ -1,11 +1,17 @@ - package no.priv.garshol.duke.genetic; import java.util.List; import java.util.ArrayList; import java.util.Collections; +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; + +import org.xml.sax.SAXException; + import no.priv.garshol.duke.Configuration; +import no.priv.garshol.duke.ConfigLoader; /** * Keeps track of the population. @@ -29,7 +35,7 @@ public GeneticPopulation(Configuration config) { } /** - * Creates the initial population. + * Creates the initial random population. */ public void create() { GeneticConfiguration cfg = @@ -42,6 +48,30 @@ public void create() { for (; ix < size; ix++) population.add(cfg.makeRandomCopy()); } + + /** + * Creates the initial population from a previously saved state + */ + public void restore(String dirStr) throws IOException, SAXException { + File dir = new File(dirStr); + File[] files = dir.listFiles(new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.matches("config_\\d+\\.xml"); + } + } + ); + + Configuration config; + GeneticConfiguration geneticConfig; + population = new ArrayList(files.length); + + for (int ix = 0; ix < files.length; ix++) { + config = ConfigLoader.load(dir + "//config_" + (ix+1) + ".xml"); + geneticConfig = new GeneticConfiguration(config, mutation_rate, recombination_rate, + evolve_comparators); + population.add(geneticConfig); + } + } /** * Returns all configurations in the current generation. @@ -69,6 +99,13 @@ public void sort() { public GeneticConfiguration getBestConfiguration() { return population.get(0); } + + /** + * Returns the nth configuration. + */ + public GeneticConfiguration getNthConfiguration(int n) { + return population.get(n); + } /** * Returns the worst configuration. From c8b0b87d255f9674a926b3c6116a040c8bde3815 Mon Sep 17 00:00:00 2001 From: swamikevala Date: Sat, 27 Sep 2014 20:44:09 +0530 Subject: [PATCH 2/2] Fix: Genetic evolution parameters not initialized When running the restore-dump, the mutation and recombination rates were not being initialized, so were continuously zero. This has been fixed now. Note that it does not save the evolution parameters state: It again starts from random settings. --- .../java/no/priv/garshol/duke/genetic/GeneticPopulation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/no/priv/garshol/duke/genetic/GeneticPopulation.java b/src/main/java/no/priv/garshol/duke/genetic/GeneticPopulation.java index 6c545cbb..3c99f49d 100644 --- a/src/main/java/no/priv/garshol/duke/genetic/GeneticPopulation.java +++ b/src/main/java/no/priv/garshol/duke/genetic/GeneticPopulation.java @@ -69,7 +69,7 @@ public boolean accept(File dir, String name) { config = ConfigLoader.load(dir + "//config_" + (ix+1) + ".xml"); geneticConfig = new GeneticConfiguration(config, mutation_rate, recombination_rate, evolve_comparators); - population.add(geneticConfig); + population.add(geneticConfig.makeCopy()); } }