diff --git a/src/cspro2sql/Main.java b/src/cspro2sql/Main.java index ef861da..41aa3e3 100644 --- a/src/cspro2sql/Main.java +++ b/src/cspro2sql/Main.java @@ -1,286 +1,293 @@ -package cspro2sql; - -import cspro2sql.bean.Dictionary; -import cspro2sql.reader.DictionaryReader; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; -import java.io.UnsupportedEncodingException; -import java.util.List; -import java.util.Properties; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; - -/** - * Copyright 2017 ISTAT - * - * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by - * the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. You may - * obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl5 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * Licence for the specific language governing permissions and limitations under - * the Licence. - * - * @author Guido Drovandi - * @author Mauro Bruno - * @author Paolo Giacomi - * @version 0.9.18.2 - */ -public class Main { - - private static final String VERSION = "0.9.18.2"; - - public static void main(String[] args) { - CsPro2SqlOptions opts = getCommandLineOptions(args); - boolean error = false; - List dictionaries; - try { - dictionaries = DictionaryReader.parseDictionaries(opts.schema, opts.dictionary, opts.tablePrefix); - } catch (Exception e) { - opts.ps.close(); - opts.printHelp(e.getMessage()); - System.exit(1); - return; - } - - if (opts.schemaEngine) { - error = !SchemaEngine.execute(dictionaries, opts.foreignKeys, opts.ps); - } else if (opts.loaderEngine) { - if (opts.delay == null) { - error = !LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); - } else { - while (true) { - try { - LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); - } catch (Exception ex) { - System.err.println(ex.getMessage()); - } - try { - Thread.sleep(opts.delay); - } catch (Exception ex) { - System.err.println(ex.getMessage()); - } - } - } - } else if (opts.monitorEngine) { - error = !MonitorEngine.execute(dictionaries, opts.ps); - } else if (opts.updateEngine) { - error = !UpdateEngine.execute(opts.prop); - } else if (opts.statusEngine) { - error = !StatusEngine.execute(dictionaries, opts.prop); - } else if (opts.loadAndUpdate) { - while (true) { - try { - LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); - UpdateEngine.execute(opts.prop); - } catch (Exception ex) { - System.err.println(ex.getMessage()); - } - try { - Thread.sleep(opts.delay); - } catch (Exception ex) { - System.err.println(ex.getMessage()); - } - } - } - - if (opts.ps != null) { - opts.ps.close(); - } - - if (error) { - System.exit(1); - } - } - - private static CsPro2SqlOptions getCommandLineOptions(String[] args) { - Options options = new Options(); - options.addOption("a", "all", false, "transfer all the questionnaires"); - options.addOption("cc", "check-constraints", false, "perform constraints check"); - options.addOption("co", "check-only", false, "perform only constraints check (no data transfer)"); - options.addOption("e", "engine", true, "select engine: [loader|schema|monitor|update|status]"); - options.addOption("f", "force", false, "skip check of loader multiple running instances"); - options.addOption("fk", "foreign-keys", false, "create foreign keys to value sets"); - options.addOption("h", "help", false, "display this help"); - options.addOption("o", "output", true, "name of the output file"); - options.addOption("p", "properties", true, "properties file"); - options.addOption("r", "recovery", false, "recover a broken session of the loader"); - options.addOption("v", "version", false, "print the version of the programm"); - options.addOption("d", "delay", true, "perform again after DELAY minutes"); - - CsPro2SqlOptions opts = new CsPro2SqlOptions(options); - try { - CommandLineParser parser = new DefaultParser(); - CommandLine cmd = parser.parse(options, args); - - if (cmd.hasOption("h") || args.length == 0) { - opts.printHelp(); - } - if (cmd.hasOption("v")) { - opts.printVersion(); - } - - if (!cmd.hasOption("e")) { - opts.printHelp("The engine type is mandatory!"); - } - if (!cmd.hasOption("p")) { - opts.printHelp("The properties file is mandatory!"); - } - - opts.propertiesFile = cmd.getOptionValue("p"); - String engine = cmd.getOptionValue("e"); - switch (engine) { - case "schema": - opts.schemaEngine = true; - opts.foreignKeys = cmd.hasOption("fk"); - if (cmd.hasOption("o")) { - opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); - } else { - opts.ps = System.out; - } - break; - case "loader": - case "LU": - opts.loaderEngine = "loader".equals(engine); - opts.loadAndUpdate = "LU".equals(engine); - opts.checkConstraints = cmd.hasOption("cc"); - opts.checkOnly = cmd.hasOption("co"); - opts.allRecords = cmd.hasOption("a"); - opts.force = cmd.hasOption("f"); - opts.recovery = cmd.hasOption("r"); - if (cmd.hasOption("o")) { - opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); - } - if (opts.loadAndUpdate && !cmd.hasOption("d")) { - opts.printHelp("The delay is mandatory!"); - } - if (cmd.hasOption("d")) { - opts.delay = Integer.parseInt(cmd.getOptionValue("d")) * 60 * 1000; - if (opts.delay < 0) { - opts.printHelp("The delay cannot be negative!"); - } - } - break; - case "monitor": - opts.monitorEngine = true; - if (cmd.hasOption("o")) { - opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); - } else { - opts.ps = System.out; - } - break; - case "update": - opts.updateEngine = true; - break; - case "status": - opts.statusEngine = true; - break; - default: - opts.printHelp("Wrong engine type!"); - break; - } - } catch (ParseException | FileNotFoundException | UnsupportedEncodingException e) { - opts.printHelp(); - } - //End parsing command line - - //Load property file - Properties prop = new Properties(); - try (InputStream in = new FileInputStream(opts.propertiesFile)) { - prop.load(in); - } catch (IOException ex) { - System.out.println("Cannot read properties file '" + opts.propertiesFile + "'"); - opts.printHelp(); - } - - opts.prop = prop; - opts.dictionary = prop.getProperty("dictionary"); - opts.schema = prop.getProperty("db.dest.schema"); - if (opts.schema == null || opts.schema.isEmpty()) { - opts.printHelp("The database schema is mandatory!\nPlease set 'db.dest.schema' into the properties file"); - } - opts.tablePrefix = prop.getProperty("dictionary.prefix", ""); - - return opts; - } - - public static class CsPro2SqlOptions { - - boolean schemaEngine; - boolean loaderEngine; - boolean monitorEngine; - boolean updateEngine; - boolean statusEngine; - boolean loadAndUpdate; - boolean allRecords; - boolean foreignKeys; - boolean checkConstraints; - boolean checkOnly; - boolean force; - boolean recovery; - String dictionary; - String schema; - String tablePrefix; - String propertiesFile; - Integer delay; - PrintStream ps = null; - Properties prop; - private final Options options; - - CsPro2SqlOptions(Options options) { - this.options = options; - } - - void printHelp() { - printHelp(null); - } - - void printHelp(String errMessage) { - HelpFormatter formatter = new HelpFormatter(); - if (errMessage != null) { - System.err.println(errMessage); - } - System.out.println("CsPro2Sql - version " + VERSION + "\n"); - formatter.printHelp("\n\n" - + "CsPro2Sql -e schema -p PROPERTIES_FILE [-fk] [-o OUTPUT_FILE]\n" - + "CsPro2Sql -e loader -p PROPERTIES_FILE [-a] [-cc] [-co] [-f|-r] [-o OUTPUT_FILE] [-d DELAY]\n" - + "CsPro2Sql -e monitor -p PROPERTIES_FILE [-o OUTPUT_FILE]\n" - + "CsPro2Sql -e update -p PROPERTIES_FILE\n" - + "CsPro2Sql -e status -p PROPERTIES_FILE\n" - + "CsPro2Sql -e LU -p PROPERTIES_FILE -d DELAY [-a] [-cc] [-co] [-f|-r] [-o OUTPUT_FILE]\n" - + "CsPro2Sql -v\n" - + "\n" - + "Engines descriptions:\n" - + " - schema: create the sql script for microdata\n" - + " - loader: load microdata into the sql database\n" - + " - monitor: create the sql script to setup the monitoring system\n" - + " - update: update the reports of the monitoring system\n" - + " - status: print the loader status\n" - + " - LU: load and update\n" - + "\n", options); - if (errMessage == null) { - System.exit(0); - } else { - System.exit(1); - } - } - - void printVersion() { - System.out.println("CsPro2Sql - version " + VERSION); - System.exit(0); - } - } - -} +package cspro2sql; + +import cspro2sql.bean.AreaNameFile; +import cspro2sql.bean.Dictionary; +import cspro2sql.reader.AreaNameFileReader; +import cspro2sql.reader.DictionaryReader; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Properties; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +/** + * Copyright 2017 ISTAT + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by + * the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. You may + * obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl5 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * Licence for the specific language governing permissions and limitations under + * the Licence. + * + * @author Guido Drovandi + * @author Mauro Bruno + * @author Paolo Giacomi + * @version 0.9.18.2 + */ +public class Main { + + private static final String VERSION = "0.9.18.2"; + + public static void main(String[] args) { + CsPro2SqlOptions opts = getCommandLineOptions(args); + boolean error = false; + List dictionaries; + AreaNameFile areaNames = null; + try { + dictionaries = DictionaryReader.parseDictionaries(opts.schema, opts.dictionary, opts.tablePrefix); + + if (opts.prop.getProperty("geography") != null) + areaNames = AreaNameFileReader.parseAreaNamesFile(opts.prop.getProperty("geography")); + + } catch (Exception e) { + opts.ps.close(); + opts.printHelp(e.getMessage()); + System.exit(1); + return; + } + + if (opts.schemaEngine) { + error = !SchemaEngine.execute(dictionaries, opts.foreignKeys, opts.ps); + } else if (opts.loaderEngine) { + if (opts.delay == null) { + error = !LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); + } else { + while (true) { + try { + LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); + } catch (Exception ex) { + System.err.println(ex.getMessage()); + } + try { + Thread.sleep(opts.delay); + } catch (Exception ex) { + System.err.println(ex.getMessage()); + } + } + } + } else if (opts.monitorEngine) { + error = !MonitorEngine.execute(dictionaries, areaNames, opts.prop, opts.ps); + } else if (opts.updateEngine) { + error = !UpdateEngine.execute(opts.prop); + } else if (opts.statusEngine) { + error = !StatusEngine.execute(dictionaries, opts.prop); + } else if (opts.loadAndUpdate) { + while (true) { + try { + LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); + UpdateEngine.execute(opts.prop); + } catch (Exception ex) { + System.err.println(ex.getMessage()); + } + try { + Thread.sleep(opts.delay); + } catch (Exception ex) { + System.err.println(ex.getMessage()); + } + } + } + + if (opts.ps != null) { + opts.ps.close(); + } + + if (error) { + System.exit(1); + } + } + + private static CsPro2SqlOptions getCommandLineOptions(String[] args) { + Options options = new Options(); + options.addOption("a", "all", false, "transfer all the questionnaires"); + options.addOption("cc", "check-constraints", false, "perform constraints check"); + options.addOption("co", "check-only", false, "perform only constraints check (no data transfer)"); + options.addOption("e", "engine", true, "select engine: [loader|schema|monitor|update|status]"); + options.addOption("f", "force", false, "skip check of loader multiple running instances"); + options.addOption("fk", "foreign-keys", false, "create foreign keys to value sets"); + options.addOption("h", "help", false, "display this help"); + options.addOption("o", "output", true, "name of the output file"); + options.addOption("p", "properties", true, "properties file"); + options.addOption("r", "recovery", false, "recover a broken session of the loader"); + options.addOption("v", "version", false, "print the version of the programm"); + options.addOption("d", "delay", true, "perform again after DELAY minutes"); + + CsPro2SqlOptions opts = new CsPro2SqlOptions(options); + try { + CommandLineParser parser = new DefaultParser(); + CommandLine cmd = parser.parse(options, args); + + if (cmd.hasOption("h") || args.length == 0) { + opts.printHelp(); + } + if (cmd.hasOption("v")) { + opts.printVersion(); + } + + if (!cmd.hasOption("e")) { + opts.printHelp("The engine type is mandatory!"); + } + if (!cmd.hasOption("p")) { + opts.printHelp("The properties file is mandatory!"); + } + + opts.propertiesFile = cmd.getOptionValue("p"); + String engine = cmd.getOptionValue("e"); + switch (engine) { + case "schema": + opts.schemaEngine = true; + opts.foreignKeys = cmd.hasOption("fk"); + if (cmd.hasOption("o")) { + opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); + } else { + opts.ps = System.out; + } + break; + case "loader": + case "LU": + opts.loaderEngine = "loader".equals(engine); + opts.loadAndUpdate = "LU".equals(engine); + opts.checkConstraints = cmd.hasOption("cc"); + opts.checkOnly = cmd.hasOption("co"); + opts.allRecords = cmd.hasOption("a"); + opts.force = cmd.hasOption("f"); + opts.recovery = cmd.hasOption("r"); + if (cmd.hasOption("o")) { + opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); + } + if (opts.loadAndUpdate && !cmd.hasOption("d")) { + opts.printHelp("The delay is mandatory!"); + } + if (cmd.hasOption("d")) { + opts.delay = Integer.parseInt(cmd.getOptionValue("d")) * 60 * 1000; + if (opts.delay < 0) { + opts.printHelp("The delay cannot be negative!"); + } + } + break; + case "monitor": + opts.monitorEngine = true; + if (cmd.hasOption("o")) { + opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); + } else { + opts.ps = System.out; + } + break; + case "update": + opts.updateEngine = true; + break; + case "status": + opts.statusEngine = true; + break; + default: + opts.printHelp("Wrong engine type!"); + break; + } + } catch (ParseException | FileNotFoundException | UnsupportedEncodingException e) { + opts.printHelp(); + } + //End parsing command line + + //Load property file + Properties prop = new Properties(); + try (InputStream in = new FileInputStream(opts.propertiesFile)) { + prop.load(in); + } catch (IOException ex) { + System.out.println("Cannot read properties file '" + opts.propertiesFile + "'"); + opts.printHelp(); + } + + opts.prop = prop; + opts.dictionary = prop.getProperty("dictionary"); + opts.schema = prop.getProperty("db.dest.schema"); + if (opts.schema == null || opts.schema.isEmpty()) { + opts.printHelp("The database schema is mandatory!\nPlease set 'db.dest.schema' into the properties file"); + } + opts.tablePrefix = prop.getProperty("dictionary.prefix", ""); + + return opts; + } + + public static class CsPro2SqlOptions { + + boolean schemaEngine; + boolean loaderEngine; + boolean monitorEngine; + boolean updateEngine; + boolean statusEngine; + boolean loadAndUpdate; + boolean allRecords; + boolean foreignKeys; + boolean checkConstraints; + boolean checkOnly; + boolean force; + boolean recovery; + String dictionary; + String schema; + String tablePrefix; + String propertiesFile; + Integer delay; + PrintStream ps = null; + Properties prop; + private final Options options; + + CsPro2SqlOptions(Options options) { + this.options = options; + } + + void printHelp() { + printHelp(null); + } + + void printHelp(String errMessage) { + HelpFormatter formatter = new HelpFormatter(); + if (errMessage != null) { + System.err.println(errMessage); + } + System.out.println("CsPro2Sql - version " + VERSION + "\n"); + formatter.printHelp("\n\n" + + "CsPro2Sql -e schema -p PROPERTIES_FILE [-fk] [-o OUTPUT_FILE]\n" + + "CsPro2Sql -e loader -p PROPERTIES_FILE [-a] [-cc] [-co] [-f|-r] [-o OUTPUT_FILE] [-d DELAY]\n" + + "CsPro2Sql -e monitor -p PROPERTIES_FILE [-o OUTPUT_FILE]\n" + + "CsPro2Sql -e update -p PROPERTIES_FILE\n" + + "CsPro2Sql -e status -p PROPERTIES_FILE\n" + + "CsPro2Sql -e LU -p PROPERTIES_FILE -d DELAY [-a] [-cc] [-co] [-f|-r] [-o OUTPUT_FILE]\n" + + "CsPro2Sql -v\n" + + "\n" + + "Engines descriptions:\n" + + " - schema: create the sql script for microdata\n" + + " - loader: load microdata into the sql database\n" + + " - monitor: create the sql script to setup the monitoring system\n" + + " - update: update the reports of the monitoring system\n" + + " - status: print the loader status\n" + + " - LU: load and update\n" + + "\n", options); + if (errMessage == null) { + System.exit(0); + } else { + System.exit(1); + } + } + + void printVersion() { + System.out.println("CsPro2Sql - version " + VERSION); + System.exit(0); + } + } + +} diff --git a/src/cspro2sql/MonitorEngine.java b/src/cspro2sql/MonitorEngine.java index cae6faa..2700a40 100644 --- a/src/cspro2sql/MonitorEngine.java +++ b/src/cspro2sql/MonitorEngine.java @@ -1,6 +1,8 @@ package cspro2sql; +import cspro2sql.bean.AreaNameFile; import cspro2sql.bean.Dictionary; +import cspro2sql.reader.AreaNameFileReader; import cspro2sql.reader.DictionaryReader; import cspro2sql.sql.TemplateManager; import cspro2sql.writer.MonitorWriter; @@ -44,24 +46,39 @@ public static void main(String[] args) throws Exception { prop.getProperty("db.dest.schema"), prop.getProperty("dictionary"), prop.getProperty("dictionary.prefix")); - execute(dictionaries, System.out); + + final AreaNameFile areaNames = prop.getProperty("geography") != null + ? AreaNameFileReader.parseAreaNamesFile(prop.getProperty("geography")) + : null; + execute(dictionaries, areaNames, prop, System.out); } catch (Exception ex) { System.exit(1); } } - static boolean execute(List dictionaries, PrintStream out) { - TemplateManager tmFieldwork = null, tmListing = null, tmExpected = null; - for (Dictionary dictionary : dictionaries) { - if (dictionary.hasTag(Dictionary.TAG_FIELDWORK)) { - tmFieldwork = new TemplateManager(dictionary); - } else if (dictionary.hasTag(Dictionary.TAG_LISTING)) { - tmListing = new TemplateManager(dictionary); - } else if (dictionary.hasTag(Dictionary.TAG_EXPECTED)) { - tmExpected = new TemplateManager(dictionary); + static boolean execute(List dictionaries, AreaNameFile areaNames, Properties prop, PrintStream out) { + + try { + TemplateManager tmFieldwork = null, tmListing = null, + tmExpected = null, tmEaStatus = null; + for (Dictionary dictionary : dictionaries) { + if (dictionary.hasTag(Dictionary.TAG_FIELDWORK)) { + tmFieldwork = new TemplateManager(dictionary); + } else if (dictionary.hasTag(Dictionary.TAG_LISTING)) { + tmListing = new TemplateManager(dictionary); + } else if (dictionary.hasTag(Dictionary.TAG_EXPECTED)) { + tmExpected = new TemplateManager(dictionary); + } else if (dictionary.hasTag(Dictionary.TAG_EA_STATUS)) { + tmEaStatus = new TemplateManager(dictionary); + } } + + boolean gisEnabled = prop.getProperty("gis.enabled") != null && (prop.getProperty("gis.enabled").equalsIgnoreCase("true") || prop.getProperty("gis.enabled").equals("1")); + return MonitorWriter.write(tmFieldwork, tmListing, tmExpected, tmEaStatus, areaNames, gisEnabled ,out); + } catch (IOException ex) { + System.err.println(ex.getMessage()); + return false; } - return MonitorWriter.write(tmFieldwork, tmListing, tmExpected, out); } } diff --git a/src/cspro2sql/SchemaEngine.java b/src/cspro2sql/SchemaEngine.java index ee06eb5..e31a09a 100644 --- a/src/cspro2sql/SchemaEngine.java +++ b/src/cspro2sql/SchemaEngine.java @@ -61,10 +61,15 @@ public static void main(String[] args) throws Exception { } static boolean execute(List dictionaries, boolean foreignKeys, PrintStream out) { - for (Dictionary dictionary : dictionaries) { - SchemaWriter.write(dictionary, foreignKeys, out); + try { + for (Dictionary dictionary : dictionaries) { + SchemaWriter.write(dictionary, foreignKeys, out); + } + return true; + } catch (IOException ex) { + System.err.println(ex.getMessage()); + return false; } - return true; } } diff --git a/src/cspro2sql/UpdateEngine.java b/src/cspro2sql/UpdateEngine.java index f1763bc..ab2305f 100644 --- a/src/cspro2sql/UpdateEngine.java +++ b/src/cspro2sql/UpdateEngine.java @@ -63,9 +63,9 @@ static boolean execute(Properties prop) { while (rs.next()) { String template = rs.getString(1); System.out.print("Updating " + template + "... "); - writeDst.executeUpdate("DROP TABLE IF EXISTS " + schema + ".m" + template); + writeDst.executeUpdate("DROP TABLE IF EXISTS " + schema + ".`m" + template + "`"); writeDst.executeQuery("SELECT 0 INTO @ID"); - writeDst.executeUpdate("CREATE TABLE " + schema + ".m" + template + " (PRIMARY KEY (ID)) AS SELECT @ID := @ID + 1 ID, " + template + ".* FROM " + schema + "." + template); + writeDst.executeUpdate("CREATE TABLE " + schema + ".`m" + template + "` (PRIMARY KEY (ID)) AS SELECT @ID := @ID + 1 ID, `" + template + "`.* FROM " + schema + ".`" + template + "`"); connDst.commit(); System.out.println("done"); } diff --git a/src/cspro2sql/bean/AreaNameFile.java b/src/cspro2sql/bean/AreaNameFile.java new file mode 100644 index 0000000..d890181 --- /dev/null +++ b/src/cspro2sql/bean/AreaNameFile.java @@ -0,0 +1,45 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package cspro2sql.bean; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * + * @author josh + */ +public class AreaNameFile { + + private final String[] levels; + private final NamedArea[] areas; + private final Map, NamedArea> codeToArea; + + public AreaNameFile(String[] levels, NamedArea[] areas) + { + this.levels = levels; + this.areas = areas; + codeToArea = new HashMap<>(); + for (NamedArea area : areas) + codeToArea.put(area.getCodes(), area); + } + + public String[] getLevels() + { + return levels; + } + + public NamedArea[] getAreas() + { + return areas; + } + + public NamedArea lookup(List codes) + { + return codeToArea.get(codes); + } +} diff --git a/src/cspro2sql/bean/BeanFactory.java b/src/cspro2sql/bean/BeanFactory.java index c3b258a..cfc23d4 100644 --- a/src/cspro2sql/bean/BeanFactory.java +++ b/src/cspro2sql/bean/BeanFactory.java @@ -37,6 +37,10 @@ public static void parseDictionary(BufferedReader br, Dictionary dictionary) thr dictionary.setName(getValue(line)); } else if (line.startsWith(Dictionary.DICT_NOTE)) { parseNote(dictionary, getValue(line)); + } else if (line.startsWith(Dictionary.HEADER_RECSTART)) { + dictionary.setRecordTypeStart(Integer.parseInt(getValue(line))); + } else if (line.startsWith(Dictionary.HEADER_RECLEN)) { + dictionary.setRecordTypeLength(Integer.parseInt(getValue(line))); } else if (line.isEmpty()) { break; } diff --git a/src/cspro2sql/bean/Dictionary.java b/src/cspro2sql/bean/Dictionary.java index b717246..02e8ee5 100644 --- a/src/cspro2sql/bean/Dictionary.java +++ b/src/cspro2sql/bean/Dictionary.java @@ -1,314 +1,335 @@ -package cspro2sql.bean; - -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -/** - * Copyright 2017 ISTAT - * - * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by - * the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. You may - * obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl5 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * Licence for the specific language governing permissions and limitations under - * the Licence. - * - * @author Guido Drovandi - * @author Mauro Bruno - * @author Paolo Giacomi - * @version 0.9.15 - */ -public final class Dictionary extends Taggable { - - public static final String DICT_HEADER = "[Dictionary]"; - public static final String DICT_LANGUAGES = "[Languages]"; - public static final String DICT_LEVEL = "[Level]"; - public static final String DICT_RECORD = "[Record]"; - public static final String DICT_IDITEMS = "[IdItems]"; - public static final String DICT_ITEM = "[Item]"; - public static final String DICT_VALUESET = "[ValueSet]"; - public static final String DICT_RELATION = "[Relation]"; - - public static final String DICT_LABEL = "Label"; - public static final char DICT_LABEL_LANGUAGE_SEPARATOR = '|'; - public static final String DICT_NAME = "Name"; - public static final String DICT_NOTE = "Note"; - public static final String DICT_NOTENEWLINE = "\r\n"; - public static final String DICT_NEWLINE_REGEXP = "(? records = new LinkedList<>(); - private final Map recordsByName = new LinkedHashMap<>(); - private final Map valueSets = new HashMap<>(); - private final Map> tags = new HashMap<>(); - - private String name; - private Record lastRecord; - private List lastItems; - private List lastItemsNotSubItem; - - public Dictionary(String schema, String prefix) { - this.schema = schema; - this.prefix = prefix; - } - - public String getSchema() { - return schema; - } - - public String getPrefix() { - return prefix; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public void addRecord(Record record) { - if (record.hasTag(TAG_IGNORE)) { - return; - } - if (this.records.isEmpty()) { - // idItems record - record.setIsMainRecord(true); - } else { - record.setMainRecord(getMainRecord()); - } - this.lastRecord = record; - this.records.add(this.lastRecord); - this.recordsByName.put(record.getRecordTypeValue(), record); - } - - public boolean addItem(Item item) { - if (item.hasTag(TAG_IGNORE)) { - return false; - } - if (item.getOccurrences() > 1) { - List its = new LinkedList<>(); - for (int i = 0; i < item.getOccurrences(); i++) { - Item it = item.clone(); - it.setOccurenceNumber(i); - it.setName(it.getName() + "_" + i); - it.setValueSetName(item.getName()); - it.setStart(it.getStart() + i * it.getLength()); - its.add(it); - } - if (item.isSubItem()) { - addSubItems(its); - } else { - addLastItemsNotSubItem(its); - this.lastRecord.addItems(its); - } - addLastItems(its); - } else { - if (item.isSubItem()) { - addSubItem(item); - } else { - addLastItemNotSubItem(item); - this.lastRecord.addItem(item); - } - addLastItem(item); - } - return true; - } - - public void addValueSet(ValueSet valueSet) { - if (valueSet != null) { - if (valueSet.getLink() != null && !valueSet.getLink().isEmpty() - && valueSets.containsKey(valueSet.getLink())) { - valueSet = valueSets.get(valueSet.getLink()).clone(); - } else { - if (valueSet.isEmpty()) { - return; - } - if (valueSet.getLink() != null && !valueSet.getLink().isEmpty()) { - this.valueSets.put(valueSet.getLink(), valueSet); - } - } - addValueSetToLastItems(valueSet); - } - } - - private void addLastItem(Item item) { - this.lastItems = new LinkedList<>(); - this.lastItems.add(item); - } - - private void addLastItems(List items) { - this.lastItems = items; - } - - private void addValueSetToLastItems(ValueSet valueSet) { - for (Item item : this.lastItems) { - if (ITEM_ALPHA.equals(item.getDataType()) && item.hasTag(TAG_MULTIPLE_RESPONSE)) { - List splits = item.splitIntoColumns(valueSet); - this.lastRecord.replaceItemWithSplit(item, splits); - } else { - if (!ITEM_ALPHA.equals(item.getDataType())) { - valueSet.removeEmptyValues(); - } - item.addValueSet(valueSet); - } - } - } - - private void addLastItemNotSubItem(Item item) { - this.lastItemsNotSubItem = new LinkedList<>(); - this.lastItemsNotSubItem.add(item); - } - - private void addLastItemsNotSubItem(List items) { - this.lastItemsNotSubItem = items; - } - - private void addSubItem(Item subItem) { - for (Item item : this.lastItemsNotSubItem) { - item.addSubItem(subItem); - } - } - - private void addSubItems(List subItems) { - for (Item item : this.lastItemsNotSubItem) { - for (Item subItem : subItems) { - item.addSubItem(subItem); - } - } - } - - public Record getMainRecord() { - return this.records.get(0); - } - - public List getRecords() { - return records; - } - - public Record getRecord(char name) { - return recordsByName.get("" + name); - } - - public Record getRecord(String name) { - return recordsByName.get(name); - } - - public void addTagged(Tag tag, Taggable o) { - if (!tags.containsKey(tag)) { - tags.put(tag, new LinkedList()); - } - tags.get(tag).add(o); - } - - public boolean hasTagged(Tag tag) { - return tags.containsKey(tag); - } - - public Record getTaggedRecord(Tag tag) { - return (Record) tags.get(tag).get(0); - } - - public Item getTaggedItem(Tag tag) { - if (tags.containsKey(tag)) { - return (Item) tags.get(tag).get(0); - } - return null; - } - - public Iterable getTaggedItems(Tag tag) { - return (Iterable) (Object) tags.get(tag); - } - - public ValueSet getTaggedValueSet(Tag tag) { - return (ValueSet) tags.get(tag).get(0); - } - - public ValueSetValue getTaggedValueSetValue(Tag tag) { - return (ValueSetValue) tags.get(tag).get(0); - } - - public Tag getTag(Tag tag) { - for (Tag t : tags.keySet()) { - if (t.equals(tag)) { - return t; - } - } - return null; - } - -} +package cspro2sql.bean; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Copyright 2017 ISTAT + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by + * the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. You may + * obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl5 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * Licence for the specific language governing permissions and limitations under + * the Licence. + * + * @author Guido Drovandi + * @author Mauro Bruno + * @author Paolo Giacomi + * @version 0.9.15 + */ +public final class Dictionary extends Taggable { + + public static final String DICT_HEADER = "[Dictionary]"; + public static final String DICT_LANGUAGES = "[Languages]"; + public static final String DICT_LEVEL = "[Level]"; + public static final String DICT_RECORD = "[Record]"; + public static final String DICT_IDITEMS = "[IdItems]"; + public static final String DICT_ITEM = "[Item]"; + public static final String DICT_VALUESET = "[ValueSet]"; + public static final String DICT_RELATION = "[Relation]"; + + public static final String DICT_LABEL = "Label"; + public static final char DICT_LABEL_LANGUAGE_SEPARATOR = '|'; + public static final String DICT_NAME = "Name"; + public static final String DICT_NOTE = "Note"; + public static final String DICT_NOTENEWLINE = "\r\n"; + public static final String DICT_NEWLINE_REGEXP = "(? records = new LinkedList<>(); + private final Map recordsByName = new LinkedHashMap<>(); + private final Map valueSets = new HashMap<>(); + private final Map> tags = new HashMap<>(); + + private String name; + private int recordTypeStart; + private int recordTypeLength; + private Record lastRecord; + private List lastItems; + private List lastItemsNotSubItem; + + public Dictionary(String schema, String prefix) { + this.schema = schema; + this.prefix = prefix; + } + + public String getSchema() { + return schema; + } + + public String getPrefix() { + return prefix; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public int getRecordTypeStart() { + return recordTypeStart; + } + + public void setRecordTypeStart(int start) { + this.recordTypeStart = start; + } + + public int getRecordTypeLength() { + return recordTypeLength; + } + + public void setRecordTypeLength(int length) { + this.recordTypeLength = length; + } + + public void addRecord(Record record) { + if (record.hasTag(TAG_IGNORE)) { + return; + } + if (this.records.isEmpty()) { + // idItems record + record.setIsMainRecord(true); + } else { + record.setMainRecord(getMainRecord()); + } + this.lastRecord = record; + this.records.add(this.lastRecord); + this.recordsByName.put(record.getRecordTypeValue(), record); + } + + public boolean addItem(Item item) { + if (item.hasTag(TAG_IGNORE)) { + return false; + } + if (item.getOccurrences() > 1) { + List its = new LinkedList<>(); + for (int i = 0; i < item.getOccurrences(); i++) { + Item it = item.clone(); + it.setOccurenceNumber(i); + it.setName(it.getName() + "_" + i); + it.setValueSetName(item.getName()); + it.setStart(it.getStart() + i * it.getLength()); + its.add(it); + } + if (item.isSubItem()) { + addSubItems(its); + } else { + addLastItemsNotSubItem(its); + this.lastRecord.addItems(its); + } + addLastItems(its); + } else { + if (item.isSubItem()) { + addSubItem(item); + } else { + addLastItemNotSubItem(item); + this.lastRecord.addItem(item); + } + addLastItem(item); + } + return true; + } + + public void addValueSet(ValueSet valueSet) { + if (valueSet != null) { + if (valueSet.getLink() != null && !valueSet.getLink().isEmpty() + && valueSets.containsKey(valueSet.getLink())) { + valueSet = valueSets.get(valueSet.getLink()).clone(); + } else { + if (valueSet.isEmpty()) { + return; + } + if (valueSet.getLink() != null && !valueSet.getLink().isEmpty()) { + this.valueSets.put(valueSet.getLink(), valueSet); + } + } + addValueSetToLastItems(valueSet); + } + } + + private void addLastItem(Item item) { + this.lastItems = new LinkedList<>(); + this.lastItems.add(item); + } + + private void addLastItems(List items) { + this.lastItems = items; + } + + private void addValueSetToLastItems(ValueSet valueSet) { + for (Item item : this.lastItems) { + if (ITEM_ALPHA.equals(item.getDataType()) && item.hasTag(TAG_MULTIPLE_RESPONSE)) { + List splits = item.splitIntoColumns(valueSet); + this.lastRecord.replaceItemWithSplit(item, splits); + } else { + if (!ITEM_ALPHA.equals(item.getDataType())) { + valueSet.removeEmptyValues(); + } + item.addValueSet(valueSet); + } + } + } + + private void addLastItemNotSubItem(Item item) { + this.lastItemsNotSubItem = new LinkedList<>(); + this.lastItemsNotSubItem.add(item); + } + + private void addLastItemsNotSubItem(List items) { + this.lastItemsNotSubItem = items; + } + + private void addSubItem(Item subItem) { + for (Item item : this.lastItemsNotSubItem) { + item.addSubItem(subItem); + } + } + + private void addSubItems(List subItems) { + for (Item item : this.lastItemsNotSubItem) { + for (Item subItem : subItems) { + item.addSubItem(subItem); + } + } + } + + public Record getMainRecord() { + return this.records.get(0); + } + + public List getRecords() { + return records; + } + + public Record getRecord(char name) { + return recordsByName.get("" + name); + } + + public Record getRecord(String name) { + return recordsByName.get(name); + } + + public void addTagged(Tag tag, Taggable o) { + if (!tags.containsKey(tag)) { + tags.put(tag, new LinkedList()); + } + tags.get(tag).add(o); + } + + public boolean hasTagged(Tag tag) { + return tags.containsKey(tag); + } + + public Record getTaggedRecord(Tag tag) { + return (Record) tags.get(tag).get(0); + } + + public Item getTaggedItem(Tag tag) { + if (tags.containsKey(tag)) { + return (Item) tags.get(tag).get(0); + } + return null; + } + + public Iterable getTaggedItems(Tag tag) { + return (Iterable) (Object) tags.get(tag); + } + + public ValueSet getTaggedValueSet(Tag tag) { + return (ValueSet) tags.get(tag).get(0); + } + + public ValueSetValue getTaggedValueSetValue(Tag tag) { + return (ValueSetValue) tags.get(tag).get(0); + } + + public Tag getTag(Tag tag) { + for (Tag t : tags.keySet()) { + if (t.equals(tag)) { + return t; + } + } + return null; + } + +} diff --git a/src/cspro2sql/bean/NamedArea.java b/src/cspro2sql/bean/NamedArea.java new file mode 100644 index 0000000..b7304eb --- /dev/null +++ b/src/cspro2sql/bean/NamedArea.java @@ -0,0 +1,39 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package cspro2sql.bean; + +import java.util.List; + +/** + * + * @author josh + */ +public class NamedArea { + + private final String name; + private final List codes; + + public NamedArea(String name, List codes) + { + this.name = name; + this.codes = codes; + } + + public String getName() + { + return name; + } + + public List getCodes() + { + return codes; + } + + public int level() + { + return codes.size(); + } +} diff --git a/src/cspro2sql/bean/Territory.java b/src/cspro2sql/bean/Territory.java index 37ff753..8fec90d 100644 --- a/src/cspro2sql/bean/Territory.java +++ b/src/cspro2sql/bean/Territory.java @@ -1,6 +1,9 @@ package cspro2sql.bean; +import java.io.IOException; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -28,32 +31,63 @@ */ public class Territory { - private final Map items; private final List itemsList; public Territory() { - this.items = new LinkedHashMap<>(); this.itemsList = new LinkedList<>(); } + + public void addItems(Iterable dictItems) throws IOException { - public void addItem(Item item) { - String name = null; - TerritoryItem parent = null; - Tag tag = item.getTag(Dictionary.TAG_TERRITORY); - if (tag.getValue() != null) { - String[] tagValues = tag.getValue().split(","); - name = tagValues[0]; - if (tagValues.length > 1 && !tagValues[1].isEmpty()) { - parent = items.get(tagValues[1]); + ArrayList dictItemList = new ArrayList<>(); + final Map dictItemToParentName = new LinkedHashMap<>(); + final Map dictItemToName = new LinkedHashMap<>(); + for (Item dictItem : dictItems) { + dictItemList.add(dictItem); + Tag tag = dictItem.getTag(Dictionary.TAG_TERRITORY); + if (tag.getValue() != null) { + String[] tagValues = tag.getValue().split(","); + dictItemToName.put(dictItem, tagValues[0].trim()); + if (tagValues.length > 1 && !tagValues[1].isEmpty()) { + dictItemToParentName.put(dictItem, tagValues[1].trim()); + } } } - TerritoryItem territoryItem = new TerritoryItem(item, parent, name); - items.put(item.getName(), territoryItem); - itemsList.add(territoryItem); + + int i = 0; + while (i < dictItemList.size()) + { + Item dictItem = dictItemList.get(i); + String parentName = dictItemToParentName.getOrDefault(dictItem, null); + String prevName = i == 0 ? null : dictItemList.get(i-1).getName(); + if (parentName == null || parentName.compareTo(prevName) == 0) { + ++i; + } else { + dictItemList.remove(i); + int parentPos = 0; + for (; parentPos < dictItemList.size() && dictItemList.get(parentPos).getName().compareTo(parentName) != 0; ++parentPos) + { + } + if (parentPos == dictItemList.size()) { + throw new IOException("Territory " + dictItem.getName() + " has parent " + parentName + " that is NOT in found in territories"); + } + dictItemList.add(parentPos + 1, dictItem); + } + } + + Map nameToTerritoryItem = new LinkedHashMap<>(); + for (Item dictItem : dictItemList) + { + String parentName = dictItemToParentName.getOrDefault(dictItem, null); + TerritoryItem parent = parentName != null ? nameToTerritoryItem.get(parentName) : null; + TerritoryItem territoryItem = new TerritoryItem(dictItem, parent, dictItemToName.get(dictItem)); + nameToTerritoryItem.put(dictItem.getName(), territoryItem); + itemsList.add(territoryItem); + } } public boolean isEmpty() { - return this.items.isEmpty(); + return this.itemsList.isEmpty(); } public TerritoryItem getFirst() { @@ -68,4 +102,28 @@ public int size() { return this.itemsList.size(); } + public Iterable getItemRecords() + { + LinkedHashSet itemRecords = new LinkedHashSet<>(); + for (int i = 0; i < itemsList.size(); i++) { + itemRecords.add(itemsList.get(i).getItem().getRecord()); + } + return itemRecords; + } + + public String getFromClause() + { + if (itemsList.isEmpty()) + return null; + + Record mainRecord = itemsList.get(0).getItem().getRecord().getMainRecord(); + String from = mainRecord.getTableName(); + + for (Record record : getItemRecords()) { + if (!record.isMainRecord()) + from += " JOIN " + record.getTableName() + " ON " + mainRecord.getTableName() + ".ID = " + record.getTableName() + "." + mainRecord.getName(); + } + + return from; + } } diff --git a/src/cspro2sql/reader/AreaNameFileReader.java b/src/cspro2sql/reader/AreaNameFileReader.java new file mode 100644 index 0000000..e86a759 --- /dev/null +++ b/src/cspro2sql/reader/AreaNameFileReader.java @@ -0,0 +1,150 @@ + +package cspro2sql.reader; + +import cspro2sql.bean.AreaNameFile; +import cspro2sql.bean.NamedArea; +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** +* Copyright 2017 ISTAT + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by + * the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. You may + * obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl5 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * Licence for the specific language governing permissions and limitations under + * the Licence. + * + * @author Josh Handley + */ +public class AreaNameFileReader { + private static final String AREA_NAMES = "[Area Names]"; + private static final String LEVELS = "[Levels]"; + private static final String AREAS = "[Areas]"; + private static final Pattern VERSION_LINE = Pattern.compile("Version\\s*=\\s*CSPro ([0-9]+\\.[0-9]+)", Pattern.CASE_INSENSITIVE); + private static final Pattern LEVEL_NAME = Pattern.compile("Name\\s*=\\s*(.+)", Pattern.CASE_INSENSITIVE); + private static final Pattern AREA_LINE = Pattern.compile("((?:(?:X|[0-9]+)\\s*,?\\s*)+)=\\s*(.+)", Pattern.CASE_INSENSITIVE); + private static final Pattern AREA_CODE = Pattern.compile("(X|[0-9]+)\\s*,?\\s*", Pattern.CASE_INSENSITIVE); + + public static AreaNameFile parseAreaNamesFile(String filename) throws Exception + { + try { + return read(filename); + } catch (IOException ex) { + throw new Exception("Impossible to read dictionary " + filename + " (" + ex.getMessage() + ")", ex); + } + } + + private static AreaNameFile read(String fileName) throws IOException + { + try (InputStream in = new FileInputStream(fileName)) { + try (InputStreamReader fr = new InputStreamReader(in, "UTF-8")) { + try (BufferedReader br = new BufferedReader(fr)) { + return read(br); + } + } + } + } + + private static AreaNameFile read(BufferedReader br) throws IOException { + + br.mark(4); + if ('\ufeff' != br.read()) + br.reset(); + readThroughSectionHeader(br, AREA_NAMES); + String version = readVersion(br); + if (version == null) + throw new IOException("Missing version"); + String[] levels = readLevels(br); + NamedArea[] areas = readAreas(br); + return new AreaNameFile(levels, areas); + } + + private static void readThroughSectionHeader(BufferedReader br, String header) throws IOException + { + String line; + while ((line = br.readLine().trim()) != null) { + if (line.equalsIgnoreCase(header)) + return; + else if (!line.isEmpty() ) + throw new IOException("Expected " + header + " got " + line ); + } + } + + private static String readVersion(BufferedReader br) throws IOException { + String line; + String version = null; + Matcher matcher; + while ((line = br.readLine().trim()) != null) { + if (line.equalsIgnoreCase(LEVELS)) + { + break; + } + else if ((matcher = VERSION_LINE.matcher(line)).matches()) + { + version = matcher.group(1); + } else if (!line.isEmpty()) + { + throw new IOException("Expected 'name=level' got " + line ); + } + } + return version; + } + + private static String[] readLevels(BufferedReader br) throws IOException { + ArrayList levels = new ArrayList<>(); + String line; + Matcher matcher; + while ((line = br.readLine().trim()) != null) { + if (line.equalsIgnoreCase(AREAS)) { + break; + } else if ((matcher = LEVEL_NAME.matcher(line)).matches()) + { + levels.add(matcher.group(1)); + } else if (!line.isEmpty()) + { + throw new IOException("Expected 'name=level' got " + line ); + } + } + return levels.toArray(new String[0]); + } + + private static NamedArea[] readAreas(BufferedReader br) throws IOException { + ArrayList areas = new ArrayList<>(); + String line; + Matcher lineMatcher; + while ((line = br.readLine()) != null) { + if ((lineMatcher = AREA_LINE.matcher(line)).matches()) + { + String name = lineMatcher.group(2); + String codesString = lineMatcher.group(1); + ArrayList codes = new ArrayList<>(); + Matcher codeMatcher = AREA_CODE.matcher(codesString); + while (codeMatcher.find()) { + String code = codeMatcher.group(1); + if (code.equals("X")) + break; + else + codes.add(Integer.parseInt(code)); + } + areas.add(new NamedArea(name, codes)); + } else if (!line.isEmpty()) { + throw new IOException("Expected 'name=level' got " + line ); + } + } + return areas.toArray(new NamedArea[0]); + } +} diff --git a/src/cspro2sql/reader/QuestionnaireReader.java b/src/cspro2sql/reader/QuestionnaireReader.java index 823f683..27816de 100644 --- a/src/cspro2sql/reader/QuestionnaireReader.java +++ b/src/cspro2sql/reader/QuestionnaireReader.java @@ -43,7 +43,7 @@ public static Questionnaire parse(Dictionary dictionary, String questionnaire, S parseItem(item, rows[0], values); } for (String row : rows) { - record = dictionary.getRecord(row.charAt(0)); + record = dictionary.getRecord(getRecordType(dictionary, row)); valuesList = result.getRecordValues(record); if (valuesList == null) { valuesList = new LinkedList<>(); @@ -58,6 +58,17 @@ record = dictionary.getRecord(row.charAt(0)); return result; } + private static String getRecordType(Dictionary dictionary, String dataRow) + { + if (dictionary.getRecordTypeLength() == 0) { + // Dictionaries with only one record may omit the record type + // by setting record length to zero. + return ""; + } else { + return dataRow.substring(dictionary.getRecordTypeStart() - 1, dictionary.getRecordTypeStart() - 1 + dictionary.getRecordTypeLength()); + } + } + private static void parseItem(Item item, String row, List values) { if (row.length() < item.getStart()) { values.add(new Answer(item, null)); diff --git a/src/cspro2sql/sql/TemplateManager.java b/src/cspro2sql/sql/TemplateManager.java index 088b35c..72f0762 100644 --- a/src/cspro2sql/sql/TemplateManager.java +++ b/src/cspro2sql/sql/TemplateManager.java @@ -1,7 +1,6 @@ package cspro2sql.sql; import cspro2sql.bean.Dictionary; -import cspro2sql.bean.Item; import cspro2sql.bean.Record; import cspro2sql.bean.Tag; import cspro2sql.bean.Territory; @@ -43,7 +42,7 @@ public class TemplateManager { private final int[] ageRange; private Territory territory; - public TemplateManager(Dictionary dictionary) { + public TemplateManager(Dictionary dictionary) throws IOException { this.dictionary = dictionary; Record mainRecord = dictionary.getMainRecord(); String schema = dictionary.getSchema(); @@ -56,10 +55,12 @@ public TemplateManager(Dictionary dictionary) { this.territory = new Territory(); if (dictionary.hasTagged(Dictionary.TAG_TERRITORY)) { - Iterable territories = dictionary.getTaggedItems(Dictionary.TAG_TERRITORY); - for (Item territory : territories) { - this.territory.addItem(territory); - } + this.territory.addItems(dictionary.getTaggedItems(Dictionary.TAG_TERRITORY)); + + // Assume that geographic ids are in order in dictionary so region is + // the first one. Depending on country may not be a region but it should + // be the highest level of geography which is what we want for reports. + this.params.put("@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO", territory.getFirst().getItemName()); } if (dictionary.hasTagged(Dictionary.TAG_AGE)) { @@ -99,6 +100,9 @@ public TemplateManager(Dictionary dictionary) { if (dictionary.hasTagged(Dictionary.TAG_FEMALE)) { this.params.put("@INDIVIDUAL_VALUE_SEX_FEMALE", dictionary.getTaggedValueSetValue(Dictionary.TAG_FEMALE).getKey()); } + if (dictionary.hasTagged(Dictionary.TAG_EA_COMPLETED)) { + this.params.put("@EA_STATUS_VALUE_COMPLETE", dictionary.getTaggedValueSetValue(Dictionary.TAG_EA_COMPLETED).getKey()); + } } public Dictionary getDictionary() { diff --git a/src/cspro2sql/sql/template/dashboard_info.sql b/src/cspro2sql/sql/template/dashboard_info.sql index 530230c..48c8495 100644 --- a/src/cspro2sql/sql/template/dashboard_info.sql +++ b/src/cspro2sql/sql/template/dashboard_info.sql @@ -2,7 +2,9 @@ CREATE TABLE IF NOT EXISTS @SCHEMA.DASHBOARD_INFO ( `ID` int(1) NOT NULL DEFAULT 0, `LISTING` int(1) NOT NULL DEFAULT 0, `EXPECTED` int(1) NOT NULL DEFAULT 0, + `EA_STATUS` int(1) NOT NULL DEFAULT 0, + `GIS` int(1) NOT NULL DEFAULT 0, PRIMARY KEY (`ID`) ) ENGINE=INNODB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -INSERT INTO @SCHEMA.DASHBOARD_INFO values (0, @LISTING, @EXPECTED); +INSERT INTO @SCHEMA.DASHBOARD_INFO values (0, @LISTING, @EXPECTED, @EA_STATUS, @GIS); diff --git a/src/cspro2sql/sql/template/r_first_level_geography.sql b/src/cspro2sql/sql/template/r_first_level_geography.sql new file mode 100644 index 0000000..d1008b7 --- /dev/null +++ b/src/cspro2sql/sql/template/r_first_level_geography.sql @@ -0,0 +1,4 @@ +CREATE OR REPLACE VIEW @SCHEMA.r_first_level_geography AS +SELECT @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS CODE, @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO_NAME AS NAME +FROM TERRITORY +GROUP BY @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO, @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO_NAME; \ No newline at end of file diff --git a/src/cspro2sql/sql/template/r_questionnaire_info_region.sql b/src/cspro2sql/sql/template/r_questionnaire_info_region.sql new file mode 100644 index 0000000..5ac14b3 --- /dev/null +++ b/src/cspro2sql/sql/template/r_questionnaire_info_region.sql @@ -0,0 +1,23 @@ +CREATE OR REPLACE VIEW @SCHEMA.`r_questionnaire_info_region` AS + SELECT + `b`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS `region`, + `b`.`total` AS `total`, + `avg_individual`.`avg_individual` AS `avg_individual` + FROM + (((SELECT + @SCHEMA.@QUESTIONNAIRE_TABLE.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO, + COUNT(0) AS `total` + FROM + @SCHEMA.@QUESTIONNAIRE_TABLE + GROUP BY @SCHEMA.@QUESTIONNAIRE_TABLE.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO)) `b` + JOIN (SELECT + `a`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO, AVG(`a`.`num`) AS `avg_individual` + FROM + (SELECT + `q`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO, COUNT(0) AS `num` + FROM + (@SCHEMA.@INDIVIDUAL_TABLE `h` + JOIN @SCHEMA.@QUESTIONNAIRE_TABLE `q` ON ((`h`.@QUESTIONNAIRE_COLUMN_BASE = `q`.`ID`))) + GROUP BY `q`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO , `h`.@QUESTIONNAIRE_COLUMN_BASE) `a` + GROUP BY `a`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO) `avg_individual` ON ((`b`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO = `avg_individual`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO))) + GROUP BY `b`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO; diff --git a/src/cspro2sql/sql/template/r_religion.sql b/src/cspro2sql/sql/template/r_religion.sql index b42153c..5f6623a 100644 --- a/src/cspro2sql/sql/template/r_religion.sql +++ b/src/cspro2sql/sql/template/r_religion.sql @@ -1,8 +1,5 @@ CREATE OR REPLACE VIEW @SCHEMA.`r_religion` AS SELECT - `vs`.`VALUE` AS `RELIGION`, COUNT(0) AS `INDIVIDUALS` - FROM - (@SCHEMA.@INDIVIDUAL_TABLE `i` - JOIN @SCHEMA.@VALUESET_RELIGION `vs` ON ((`i`.@INDIVIDUAL_COLUMN_RELIGION = `vs`.`ID`))) - GROUP BY `vs`.`VALUE`; - + `vs`.`VALUE` AS `RELIGION`, `i`.`INDIVIDUALS` + FROM (SELECT @INDIVIDUAL_COLUMN_RELIGION, COUNT(0) AS `INDIVIDUALS` FROM @SCHEMA.@INDIVIDUAL_TABLE GROUP BY @INDIVIDUAL_COLUMN_RELIGION) `i` + JOIN @SCHEMA.@VALUESET_RELIGION `vs` ON `i`.@INDIVIDUAL_COLUMN_RELIGION = `vs`.`ID`; \ No newline at end of file diff --git a/src/cspro2sql/sql/template/r_sex_by_age_group_region.sql b/src/cspro2sql/sql/template/r_sex_by_age_group_region.sql new file mode 100644 index 0000000..078aab5 --- /dev/null +++ b/src/cspro2sql/sql/template/r_sex_by_age_group_region.sql @@ -0,0 +1,533 @@ +CREATE OR REPLACE VIEW @SCHEMA.`aux_sex_age_region` AS + SELECT + `q`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS `first_level_geo`, + `h`.@INDIVIDUAL_COLUMN_SEX AS `sex`, + `h`.@INDIVIDUAL_COLUMN_AGE AS `age` + FROM + (@SCHEMA.@INDIVIDUAL_TABLE `h` + JOIN @SCHEMA.@QUESTIONNAIRE_TABLE `q` ON ((`h`.@QUESTIONNAIRE_COLUMN_BASE = `q`.`ID`))); + +CREATE OR REPLACE VIEW @SCHEMA.`r_sex_by_age_group_region` AS + SELECT + `a`.`first_level_geo` AS `region`, + '0 to 4' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 0) + AND (`aux_sex_age_region`.`age` < 5)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 0) + AND (`aux_sex_age_region`.`age` < 5)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '5 to 9' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 5) + AND (`aux_sex_age_region`.`age` < 10)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 5) + AND (`aux_sex_age_region`.`age` < 10)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '10 to 14' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = 1) + AND (`aux_sex_age_region`.`age` >= 10) + AND (`aux_sex_age_region`.`age` < 15)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 10) + AND (`aux_sex_age_region`.`age` < 15)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '15 to 19' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 15) + AND (`aux_sex_age_region`.`age` < 20)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 15) + AND (`aux_sex_age_region`.`age` < 20)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '20 to 24' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 20) + AND (`aux_sex_age_region`.`age` < 25)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 20) + AND (`aux_sex_age_region`.`age` < 25)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '25 to 29' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 25) + AND (`aux_sex_age_region`.`age` < 30)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 25) + AND (`aux_sex_age_region`.`age` < 30)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '30 to 34' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 30) + AND (`aux_sex_age_region`.`age` < 35)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 30) + AND (`aux_sex_age_region`.`age` < 35)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '35 to 39' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 35) + AND (`aux_sex_age_region`.`age` < 40)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 35) + AND (`aux_sex_age_region`.`age` < 40)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '40 to 44' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 40) + AND (`aux_sex_age_region`.`age` < 45)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 40) + AND (`aux_sex_age_region`.`age` < 45)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '45 to 49' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 45) + AND (`aux_sex_age_region`.`age` < 50)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 45) + AND (`aux_sex_age_region`.`age` < 50)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '50 to 54' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 50) + AND (`aux_sex_age_region`.`age` < 55)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 50) + AND (`aux_sex_age_region`.`age` < 55)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '55 to 59' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 55) + AND (`aux_sex_age_region`.`age` < 60)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 55) + AND (`aux_sex_age_region`.`age` < 60)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '60 to 64' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 60) + AND (`aux_sex_age_region`.`age` < 65)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 60) + AND (`aux_sex_age_region`.`age` < 65)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '65 to 69' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 65) + AND (`aux_sex_age_region`.`age` < 70)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 65) + AND (`aux_sex_age_region`.`age` < 70)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '70 to 74' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 70) + AND (`aux_sex_age_region`.`age` < 75)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 70) + AND (`aux_sex_age_region`.`age` < 75)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '75 to 79' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 75) + AND (`aux_sex_age_region`.`age` < 80)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 75) + AND (`aux_sex_age_region`.`age` < 80)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '80 to 84' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 80) + AND (`aux_sex_age_region`.`age` < 85)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 80) + AND (`aux_sex_age_region`.`age` < 85)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '85 to 89' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 85) + AND (`aux_sex_age_region`.`age` < 90)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 85) + AND (`aux_sex_age_region`.`age` < 90)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '90 to 94' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 90) + AND (`aux_sex_age_region`.`age` < 95)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 90) + AND (`aux_sex_age_region`.`age` < 95)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '95 to 97' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 95) + AND (`aux_sex_age_region`.`age` < 97)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 95) + AND (`aux_sex_age_region`.`age` < 97)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '97+' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 97)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 97)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))); diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index efe0a2f..d5bafac 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -1,14 +1,18 @@ package cspro2sql.writer; +import cspro2sql.bean.AreaNameFile; import cspro2sql.bean.Dictionary; import cspro2sql.bean.Item; +import cspro2sql.bean.NamedArea; import cspro2sql.bean.Record; import cspro2sql.bean.Territory; import cspro2sql.bean.TerritoryItem; import cspro2sql.sql.TemplateManager; import java.io.IOException; import java.io.PrintStream; +import java.util.Formatter; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; /** @@ -37,23 +41,40 @@ public class MonitorWriter { "r_questionnaire_info", "r_individual_info", "r_religion", - "r_sex_by_age" + "r_sex_by_age", + "r_first_level_geography", + "r_sex_by_age_group_region" }; private static int reportCount = 0; - public static boolean write(TemplateManager tm, TemplateManager tmListing, TemplateManager tmExpected, PrintStream out) { + public static boolean write(TemplateManager tm, TemplateManager tmListing, + TemplateManager tmExpected, TemplateManager tmEaStatus, + AreaNameFile areaNames, boolean gisEnabled, PrintStream out) { + String schema = tm.getDictionary().getSchema(); tm.addParam("@LISTING", tmListing == null ? "0" : "1"); tm.addParam("@EXPECTED", tmExpected == null ? "0" : "1"); - + tm.addParam("@EA_STATUS", tmEaStatus == null ? "0" : "1"); + tm.addParam("@GIS", gisEnabled ? "1" : "0"); + out.println("USE " + schema + ";"); out.println(); Territory territory = tm.getTerritory(); int[] ageRange = tm.getAgeRange(); + if (!territory.isEmpty()) { + try { + printTerritoryTable(territory, areaNames, out); + + } catch (IOException ex) { + System.err.println(ex.getMessage()); + return false; + } + } + try { tm.printTemplate("c_user", out); tm.printTemplate("cspro2sql_report", out); @@ -73,36 +94,29 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ } if (!territory.isEmpty()) { - out.println("CREATE TABLE IF NOT EXISTS `territory` ("); - String idx = ""; - for (int i = 0; i < territory.size(); i++) { - TerritoryItem territoryItem = territory.get(i); - String name = territoryItem.getItemName(); - out.println(" `" + name + "_NAME` text COLLATE utf8mb4_unicode_ci,"); - out.println(" `" + name + "` int(11) DEFAULT NULL,"); - if (i > 0) { - idx += ","; - } - idx += "`" + name + "`"; + try { + if (tm.printTemplate("r_questionnaire_info_region", out)) + printMaterialized(schema, "r_questionnaire_info_region", out); + } catch (IOException ex) { + System.err.println(ex.getMessage()); + return false; } - out.println(" `TERRITORY_CODE` text COLLATE utf8mb4_unicode_ci,"); - out.println(" KEY `idx_territory` (" + idx + ")"); - out.println(") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); - out.println(); } if (!territory.isEmpty()) { - TerritoryItem territoryItem = territory.getFirst(); + TerritoryItem firstTerritoryItem = territory.getFirst(); out.println("CREATE OR REPLACE VIEW " + schema + ".`r_regional_area` AS"); - String groupBy = territoryItem.getItemName(); - String name = territoryItem.getName(); - out.print(" SELECT '" + name + "' name, COUNT(0) value FROM (SELECT COUNT(0) FROM " + schema + "." + tm.getParam("@QUESTIONNAIRE_TABLE") + " GROUP BY " + groupBy + ") a0"); + String groupBy = firstTerritoryItem.getItemName(); + String name = firstTerritoryItem.getName(); + String fromClause = territory.getFromClause(); + + out.print(" SELECT '" + name + "' name, COUNT(0) value FROM (SELECT COUNT(0) FROM " + fromClause + " GROUP BY " + groupBy + ") a0"); for (int i = 1; i < territory.size(); i++) { - territoryItem = territory.get(i); + TerritoryItem territoryItem = territory.get(i); name = territoryItem.getName(); groupBy += "," + territoryItem.getItemName(); out.println(" UNION"); - out.print(" SELECT '" + name + "', COUNT(0) FROM (SELECT COUNT(0) FROM " + schema + "." + tm.getParam("@QUESTIONNAIRE_TABLE") + " GROUP BY " + groupBy + ") a" + i); + out.print(" SELECT '" + name + "', COUNT(0) FROM (SELECT COUNT(0) FROM " + fromClause + " GROUP BY " + groupBy + ") a" + i); } out.println(); out.println(";"); @@ -112,23 +126,12 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ if (ageRange != null && tm.hasParam("@INDIVIDUAL_TABLE") && tm.hasParam("@INDIVIDUAL_COLUMN_SEX") && tm.hasParam("@INDIVIDUAL_VALUE_SEX_MALE") && tm.hasParam("@INDIVIDUAL_VALUE_SEX_FEMALE") && tm.hasParam("@INDIVIDUAL_COLUMN_AGE")) { - out.println("CREATE OR REPLACE VIEW " + schema + ".`r_sex_by_age_group` AS"); - out.print(" SELECT '" + ageRange[0] + " to " + (ageRange[1] - 1) + "' as 'range', a.male, b.female FROM " - + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[0] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[1] + ") a," - + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[0] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[1] + ") b"); - for (int i = 1; i < ageRange.length - 1; i++) { - out.println(" UNION"); - out.print(" SELECT '" + ageRange[i] + " to " + (ageRange[i + 1] - 1) + "' as 'range', a.male, b.female FROM " - + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[i] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[i + 1] + ") a," - + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[i] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[i + 1] + ") b"); - } - out.println(" UNION"); - out.print(" SELECT '" + ageRange[ageRange.length - 1] + "+' as 'range', a.male, b.female FROM " - + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[ageRange.length - 1] + ") a," - + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[ageRange.length - 1] + ") b"); - out.println(); - out.println(";"); - printMaterialized(schema, "r_sex_by_age_group", out); + + printAgeBySexReport("r_sex_by_age_group", tm, ageRange, schema, out); + + int[] broadAgeRange = new int[] {0, 5, 18}; + printAgeBySexReport("r_sex_by_broad_age_group", tm, broadAgeRange, schema, out); + } if (!territory.isEmpty()) { @@ -150,7 +153,12 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ out.print(",'#'," + territoryItem.selectDescription()); } out.println(") as name, COUNT(0) AS `household`"); - out.println(" FROM " + schema + "." + tm.getParam("@QUESTIONNAIRE_TABLE") + " `h`"); + out.print(" FROM (SELECT " + territory.getFirst().getItem().getColunmFullName()); + for (int i = 1; i < territory.size(); i++) { + out.print(", " + territory.get(i).getItem().getColunmFullName()); + } + + out.println(" FROM " + territory.getFromClause() + ") `h`"); out.print(" GROUP BY "); territoryItem = territory.getFirst(); out.print("`h`." + territoryItem.getItemName()); @@ -163,36 +171,154 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ } if (!territory.isEmpty()) { - printAuxTable(tm, tm, "aux_household_returned", "returned", out); - printAuxTable(tm, (tmListing != null) ? tmListing : tm, "aux_listing_returned", "returned", out); - printAuxTable(tm, (tmExpected != null) ? tmExpected : tm, "aux_household_expected", "expected", out); + try { + printAuxTable(tm, tm, "aux_household_returned", "returned", out); + printAuxTable(tm, (tmListing != null) ? tmListing : tm, "aux_listing_returned", "returned", out); + printAuxTable(tm, (tmExpected != null) ? tmExpected : tm, "aux_household_expected", "expected", out); - int upTo = 1; - for (int i = 0; i < territory.size(); i++) { - TerritoryItem territoryItem = territory.get(i); - printExpectedReport(tm, "r_household_expected_by_" + territoryItem.getName().toLowerCase(), upTo++, out); - printMaterialized(schema, "r_household_expected_by_" + territoryItem.getName().toLowerCase(), out); - } + boolean createEaTables = tmEaStatus != null && tmExpected != null && tmEaStatus.hasParam("@EA_STATUS_VALUE_COMPLETE"); + + if (createEaTables) { + printAuxEaStatusTable(tm, tmExpected, "aux_ea_expected", "expected", out); + printAuxEaStatusTable(tm, tmEaStatus, "aux_ea_completed", "completed", out); + } - printTotalReport(tm, (tmListing != null) ? tmListing : tm, out); - printMaterialized(schema, "r_total", out); - } + for (int i = 0; i < territory.size(); i++) { + final int upTo = i + 1; + + TerritoryItem territoryItem = territory.get(i); + printExpectedReport(tm, "r_household_expected_by_" + territoryItem.getName().toLowerCase(), upTo, out); + printMaterialized(schema, "r_household_expected_by_" + territoryItem.getName().toLowerCase(), out); + + if (createEaTables && i < territory.size() - 1) { + printExpectedEaReport(tm, "r_ea_expected_by_" + territoryItem.getName().toLowerCase(), upTo, out); + printMaterialized(schema, "r_ea_expected_by_" + territoryItem.getName().toLowerCase(), out); + } + } + printTotalReport(tm, (tmListing != null) ? tmListing : tm, createEaTables, out); + printMaterialized(schema, "r_total", out); + } catch (IOException ex) { + System.err.println(ex.getMessage()); + return false; + } + } + return true; } + private static void printAgeBySexReport(String reportName, TemplateManager tm, int [] ageRange, String schema, PrintStream out) { + out.println("CREATE OR REPLACE VIEW " + schema + ".`" + reportName + "` AS"); + out.print(" SELECT '" + ageRange[0] + " to " + (ageRange[1] - 1) + "' as 'range', a.male, b.female FROM " + + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[0] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[1] + ") a," + + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[0] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[1] + ") b"); + for (int i = 1; i < ageRange.length - 1; i++) { + out.println(" UNION"); + out.print(" SELECT '" + ageRange[i] + " to " + (ageRange[i + 1] - 1) + "' as 'range', a.male, b.female FROM " + + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[i] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[i + 1] + ") a," + + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[i] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[i + 1] + ") b"); + } + out.println(" UNION"); + out.print(" SELECT '" + ageRange[ageRange.length - 1] + "+' as 'range', a.male, b.female FROM " + + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[ageRange.length - 1] + ") a," + + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[ageRange.length - 1] + ") b"); + out.println(); + out.println(";"); + printMaterialized(schema, reportName, out); + } + + private static void printTerritoryTable(Territory territory, AreaNameFile areaNames, PrintStream out) throws IOException { + out.println("CREATE TABLE IF NOT EXISTS `territory` ("); + String idx = ""; + for (int i = 0; i < territory.size(); i++) { + TerritoryItem territoryItem = territory.get(i); + String name = territoryItem.getItemName(); + out.println(" `" + name + "_NAME` text COLLATE utf8mb4_unicode_ci,"); + out.println(" `" + name + "` int(11) DEFAULT NULL,"); + if (i > 0) { + idx += ","; + } + idx += "`" + name + "`"; + } + out.println(" `TERRITORY_CODE` text COLLATE utf8mb4_unicode_ci,"); + out.println(" KEY `idx_territory` (" + idx + ")"); + out.println(") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); + out.println(); + + if (areaNames != null) { + final int numLevels = areaNames.getLevels().length; + if (territory.size() != numLevels) { + throw new IOException("Area names file does not match #territory items in dictionary"); + } + + out.println("TRUNCATE TABLE `territory`;"); + + StringBuilder insertPreambleBuilder = new StringBuilder(); + insertPreambleBuilder.append("INSERT INTO `territory` ("); + for (int i = 0; i < territory.size(); i++) { + TerritoryItem territoryItem = territory.get(i); + String name = territoryItem.getItemName(); + insertPreambleBuilder.append("`"); + insertPreambleBuilder.append(name); + insertPreambleBuilder.append("_NAME`,`"); + insertPreambleBuilder.append(name); + insertPreambleBuilder.append("`"); + if (i != territory.size() - 1) + insertPreambleBuilder.append(","); + } + insertPreambleBuilder.append(",`TERRITORY_CODE`) VALUES ("); + final String insertPreamble = insertPreambleBuilder.toString(); + String[] formatSpecs = new String[numLevels]; + for (int i = 0; i < numLevels; i++) { + Item territoryDictItem = territory.get(i).getItem(); + if (territoryDictItem.isZeroFill()) + formatSpecs[i] = "%0" + territoryDictItem.getLength() + "d"; + else + formatSpecs[i] = "%" + territoryDictItem.getLength() + "d"; + } + for (NamedArea area : areaNames.getAreas()) { + + // Ignore higher level entities (province, district) + // only insert lowest level entities (EA) + if (area.level() == numLevels) { + out.print(insertPreamble); + for (int i = 0; i < numLevels - 1; i++) { + List ancestorCode = area.getCodes().subList(0, i + 1); + NamedArea ancestor = areaNames.lookup(ancestorCode); + String ancestorName = ancestor != null ? ancestor.getName().replace("'", "''") : String.valueOf(area.getCodes().get(i)); + out.print("'" + ancestorName + "'," + area.getCodes().get(i) + ","); + } + + StringBuilder fullCodeBuilder = new StringBuilder(); + Formatter fmt = new Formatter(fullCodeBuilder); + for (int i = 0; i < numLevels; i++) { + fmt.format(formatSpecs[i], area.getCodes().get(i)); + } + + out.println("'" + area.getName().replace("'", "''") + "'," + area.getCodes().get(numLevels - 1) + ",'" + fullCodeBuilder.toString() + "');"); + } + } + } + } + private static void printMaterialized(String schema, String name, PrintStream out) { - out.println("DROP TABLE IF EXISTS " + schema + ".m" + name + ";"); + out.println("DROP TABLE IF EXISTS " + schema + ".`m" + name + "`;"); out.println("SELECT 0 INTO @ID;"); - out.println("CREATE TABLE " + schema + ".m" + name + " (PRIMARY KEY (ID)) AS SELECT @ID := @ID + 1 ID, " + name + ".* FROM " + schema + "." + name + ";"); + out.println("CREATE TABLE " + schema + ".`m" + name + "` (PRIMARY KEY (ID)) AS SELECT @ID := @ID + 1 ID, `" + name + "`.* FROM " + schema + ".`" + name + "`;"); out.println("INSERT INTO " + schema + ".`cspro2sql_report` VALUES ('" + name + "', " + (reportCount++) + ");"); out.println(); } - private static void printAuxTable(TemplateManager mainTm, TemplateManager tm, String auxName, String columnName, PrintStream out) { + private static void printAuxTable(TemplateManager mainTm, TemplateManager tm, String auxName, String columnName, PrintStream out) throws IOException { Set records = new LinkedHashSet<>(); Territory territory = tm.getTerritory(); Territory mainTerritory = mainTm.getTerritory(); + if (territory.size() != mainTerritory.size()) { + throw new IOException("Number of territories in " + tm.getDictionary().getName() + + " is different from number of territories in " + + mainTm.getDictionary().getName() + + ". Fix #territory tags in dictionaries so that they match"); + } out.println("CREATE OR REPLACE VIEW " + tm.getDictionary().getSchema() + "." + auxName + " AS"); out.println(" SELECT "); for (int i = 0; i < territory.size(); i++) { @@ -202,13 +328,14 @@ private static void printAuxTable(TemplateManager mainTm, TemplateManager tm, St out.println(" " + item.getColunmFullName() + " AS " + mainTerritoryItem.getItemName() + ","); records.add(item.getRecord()); } - Record[] recArray = records.toArray(new Record[0]); Item expected = tm.getDictionary().getTaggedItem(Dictionary.TAG_EXPECTED_QUESTIONNAIRES); if (expected == null) { out.println(" COUNT(0) AS `" + columnName + "`"); } else { out.println(" SUM(" + expected.getColunmFullName() + ") AS `" + columnName + "`"); + records.add(expected.getRecord()); } + Record[] recArray = records.toArray(new Record[0]); out.println(" FROM"); out.println(" " + recArray[0].getMainRecord().getFullTableName()); for (Record record : recArray) { @@ -216,6 +343,12 @@ private static void printAuxTable(TemplateManager mainTm, TemplateManager tm, St out.println(" JOIN " + record.getFullTableName() + " ON " + record.getMainRecord().getTableName() + ".ID = " + record.getTableName() + "." + record.getMainRecord().getName()); } } + out.println(" WHERE"); + for (int i = 0; i < territory.size() - 1; i++) { + Item item = territory.get(i).getItem(); + out.println(" " + item.getColunmFullName() + " IS NOT NULL AND"); + } + out.println(" " + territory.get(territory.size() - 1).getItem().getColunmFullName() + " IS NOT NULL"); out.println(" GROUP BY"); for (int i = 0; i < territory.size() - 1; i++) { Item item = territory.get(i).getItem(); @@ -226,6 +359,59 @@ private static void printAuxTable(TemplateManager mainTm, TemplateManager tm, St out.println(); } + private static void printAuxEaStatusTable(TemplateManager mainTm, TemplateManager tm, String auxName, String columnName, PrintStream out) throws IOException { + Set records = new LinkedHashSet<>(); + Territory territory = tm.getTerritory(); + Territory mainTerritory = mainTm.getTerritory(); + if (territory.size() != mainTerritory.size()) { + throw new IOException("Number of territories in " + tm.getDictionary().getName() + + " is different from number of territories in " + + mainTm.getDictionary().getName() + + ". Fix #territory tags in dictionaries so that they match"); + } + out.println("CREATE OR REPLACE VIEW " + tm.getDictionary().getSchema() + "." + auxName + " AS"); + out.println(" SELECT "); + for (int i = 0; i < territory.size() - 1; i++) { + TerritoryItem territoryItem = territory.get(i); + TerritoryItem mainTerritoryItem = mainTerritory.get(i); + Item item = territoryItem.getItem(); + out.println(" " + item.getColunmFullName() + " AS " + mainTerritoryItem.getItemName() + ","); + records.add(item.getRecord()); + } + Item expected = tm.getDictionary().getTaggedItem(Dictionary.TAG_EXPECTED_QUESTIONNAIRES); + Item completed = tm.getDictionary().getTaggedItem(Dictionary.TAG_EA_COMPLETION_STATUS); + + if (expected != null) { + out.println(" COUNT(0) AS `" + columnName + "`"); + records.add(expected.getRecord()); + } else if (completed != null) { + out.println(" SUM(" + completed.getColunmFullName() + " = " + tm.getParam("@EA_STATUS_VALUE_COMPLETE") + ") AS `" + columnName + "`"); + records.add(completed.getRecord()); + } + Record[] recArray = records.toArray(new Record[0]); + out.println(" FROM"); + out.println(" " + recArray[0].getMainRecord().getFullTableName()); + for (Record record : recArray) { + if (!record.isMainRecord()) { + out.println(" JOIN " + record.getFullTableName() + " ON " + record.getMainRecord().getTableName() + ".ID = " + record.getTableName() + "." + record.getMainRecord().getName()); + } + } + out.println(" WHERE"); + for (int i = 0; i < territory.size() - 1; i++) { + Item item = territory.get(i).getItem(); + out.println(" " + item.getColunmFullName() + " IS NOT NULL AND"); + } + out.println(" " + territory.get(territory.size() - 1).getItem().getColunmFullName() + " IS NOT NULL"); + out.println(" GROUP BY"); + for (int i = 0; i < territory.size() - 2; i++) { + Item item = territory.get(i).getItem(); + out.println(" " + item.getColunmFullName() + ","); + } + Item item = territory.get(territory.size() - 2).getItem(); + out.println(" " + item.getColunmFullName() + ";"); + out.println(); + } + private static void printExpectedReport(TemplateManager tm, String reportName, int upTo, PrintStream out) { String schema = tm.getDictionary().getSchema(); Territory territory = tm.getTerritory(); @@ -259,16 +445,16 @@ private static void printExpectedReport(TemplateManager tm, String reportName, i for (int i = 0; i < upTo; i++) { out.println(" `h`." + territory.get(i).getItemName() + " AS `" + territory.get(i).getItemName() + "`,"); } - out.println(" SUM(`h`.`returned`) AS `returned`,"); + out.println(" SUM(`e`.`returned`) AS `returned`,"); out.println(" SUM(`l`.`returned`) AS `returned`,"); - out.println(" SUM(`e`.`expected`) AS `expected`,"); - out.println(" ((SUM(`h`.`returned`) / SUM(`l`.`returned`)) * 100) AS `field_freshlist`,"); - out.println(" ((SUM(`h`.`returned`) / SUM(`e`.`expected`)) * 100) AS `field_expected`,"); - out.println(" ((SUM(`l`.`returned`) / SUM(`e`.`expected`)) * 100) AS `freshlist_expected`"); + out.println(" SUM(`h`.`expected`) AS `expected`,"); + out.println(" ((SUM(`e`.`returned`) / SUM(`l`.`returned`)) * 100) AS `field_freshlist`,"); + out.println(" ((SUM(`e`.`returned`) / SUM(`h`.`expected`)) * 100) AS `field_expected`,"); + out.println(" ((SUM(`l`.`returned`) / SUM(`h`.`expected`)) * 100) AS `freshlist_expected`"); out.println(" FROM"); - printSubTable(tm, "aux_household_returned", "returned", upTo, out); + printSubTable(tm, "aux_household_expected", "expected", upTo, out); out.println(" `h`"); - out.println(" JOIN "); + out.println(" LEFT JOIN "); printSubTable(tm, "aux_listing_returned", "returned", upTo, out); out.println(" `l` ON"); territoryItem = territory.getFirst(); @@ -277,8 +463,8 @@ private static void printExpectedReport(TemplateManager tm, String reportName, i territoryItem = territory.get(i); out.println(" AND (`h`.`" + territoryItem.getItemName() + "` = `l`.`" + territoryItem.getItemName() + "`)"); } - out.println(" JOIN"); - printSubTable(tm, "aux_household_expected", "expected", upTo, out); + out.println(" LEFT JOIN"); + printSubTable(tm, "aux_household_returned", "returned", upTo, out); out.println(" `e` ON"); territoryItem = territory.getFirst(); out.println(" (`h`.`" + territoryItem.getItemName() + "` = `e`.`" + territoryItem.getItemName() + "`)"); @@ -294,6 +480,59 @@ private static void printExpectedReport(TemplateManager tm, String reportName, i out.println(); } + private static void printExpectedEaReport(TemplateManager tm, String reportName, int upTo, PrintStream out) { + String schema = tm.getDictionary().getSchema(); + Territory territory = tm.getTerritory(); + + out.println("CREATE OR REPLACE VIEW " + schema + ".`" + reportName + "` AS"); + out.println(" SELECT "); + out.print(" _utf8mb4 '" + territory.getFirst().getName()); + for (int i = 1; i < upTo; i++) { + out.print("#" + territory.get(i).getName()); + } + out.println("' COLLATE utf8mb4_unicode_ci AS `name`,"); + for (int i = 0; i < upTo; i++) { + out.println(" NULL AS `" + territory.get(i).getItemName() + "`,"); + } + out.println(" NULL AS `completed`,"); + out.println(" NULL AS `expected`,"); + out.println(" NULL AS `completed_expected`"); + out.println(" "); + out.println(" UNION SELECT "); + out.print(" CONCAT("); + TerritoryItem territoryItem = territory.getFirst(); + out.print(territoryItem.selectDescription()); + for (int i = 1; i < upTo; i++) { + territoryItem = territory.get(i); + out.print(",'#'," + territoryItem.selectDescription()); + } + out.println(") AS `name`,"); + for (int i = 0; i < upTo; i++) { + out.println(" `h`." + territory.get(i).getItemName() + " AS `" + territory.get(i).getItemName() + "`,"); + } + out.println(" SUM(COALESCE(`c`.`completed`, 0)) AS `completed`,"); + out.println(" SUM(`h`.`expected`) AS `expected`,"); + out.println(" ((SUM(COALESCE(`c`.`completed`, 0)) / SUM(`h`.`expected`)) * 100) AS `completed_expected`"); + out.println(" FROM"); + printSubTable(tm, "aux_ea_expected", "expected", upTo, out); + out.println(" `h`"); + out.println(" LEFT JOIN "); + printSubTable(tm, "aux_ea_completed", "completed", upTo, out); + out.println(" `c` ON"); + territoryItem = territory.getFirst(); + out.println(" (`h`.`" + territoryItem.getItemName() + "` = `c`.`" + territoryItem.getItemName() + "`)"); + for (int i = 1; i < upTo; i++) { + territoryItem = territory.get(i); + out.println(" AND (`h`.`" + territoryItem.getItemName() + "` = `c`.`" + territoryItem.getItemName() + "`)"); + } + out.print(" GROUP BY `name`"); + for (int i = 0; i < upTo; i++) { + out.print(", `h`.`" + territory.get(i).getItemName() + "`"); + } + out.println(";"); + out.println(); + } + private static void printSubTable(TemplateManager tm, String tableName, String columnName, int upTo, PrintStream out) { String schema = tm.getDictionary().getSchema(); Territory territory = tm.getTerritory(); @@ -308,7 +547,7 @@ private static void printSubTable(TemplateManager tm, String tableName, String c out.println(" FROM `" + schema + "`.`" + tableName + "`"); out.println(" GROUP BY"); out.print(" `" + tableName + "`." + territory.getFirst().getItemName()); - for (int i = 0; i < upTo && i < territory.size(); i++) { + for (int i = 1; i < upTo && i < territory.size(); i++) { TerritoryItem territoryItem = territory.get(i); out.print(",\n `" + tableName + "`." + territoryItem.getItemName()); } @@ -316,7 +555,7 @@ private static void printSubTable(TemplateManager tm, String tableName, String c out.println(" )"); } - private static void printTotalReport(TemplateManager tm, TemplateManager tmListing, PrintStream out) { + private static void printTotalReport(TemplateManager tm, TemplateManager tmListing, boolean hasEACompleted, PrintStream out) { String schema = tm.getDictionary().getSchema(); out.println("CREATE OR REPLACE VIEW `" + schema + "`.`r_total` AS"); @@ -331,6 +570,11 @@ private static void printTotalReport(TemplateManager tm, TemplateManager tmListi out.println(" FROM"); printSubTable(tm, "aux_listing_returned", "returned", 1000, out); out.println(" `a`) AS `ea_freshlist`,"); + if (hasEACompleted) { + out.println(" (SELECT COALESCE(SUM(completed),0) FROM aux_ea_completed) AS `ea_completed`,"); + } else { + out.println(" (SELECT 0 FROM aux_household_expected LIMIT 1) AS `ea_completed`,"); + } out.println(" (SELECT "); out.println(" COUNT(0)"); out.println(" FROM"); @@ -345,7 +589,7 @@ private static void printTotalReport(TemplateManager tm, TemplateManager tmListi out.println(" FROM"); out.println(" " + tmListing.getDictionary().getMainRecord().getFullTableName() + ") AS `household_freshlist`,"); out.println(" (SELECT "); - out.println(" SUM(`a`.`expected`)"); + out.println(" COALESCE(SUM(`a`.`expected`), 0)"); out.println(" FROM"); printSubTable(tm, "aux_household_expected", "expected", 1000, out); out.println(" `a`) AS `household_expected`;"); diff --git a/src/cspro2sql/writer/SchemaWriter.java b/src/cspro2sql/writer/SchemaWriter.java index 65349b6..642b3e9 100644 --- a/src/cspro2sql/writer/SchemaWriter.java +++ b/src/cspro2sql/writer/SchemaWriter.java @@ -1,144 +1,144 @@ -package cspro2sql.writer; - -import cspro2sql.bean.Dictionary; -import cspro2sql.bean.Item; -import cspro2sql.bean.Record; -import cspro2sql.bean.ValueSet; -import cspro2sql.bean.ValueSetValue; -import cspro2sql.sql.TemplateManager; -import java.io.IOException; -import java.io.PrintStream; -import java.util.HashSet; -import java.util.Set; - -/** - * Copyright 2017 ISTAT - * - * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by - * the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. You may - * obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl5 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * Licence for the specific language governing permissions and limitations under - * the Licence. - * - * @author Guido Drovandi - * @author Mauro Bruno - * @version 0.9.17 - */ -public class SchemaWriter { - - public static void write(Dictionary dictionary, boolean foreignKeys, PrintStream ps) { - TemplateManager tm = new TemplateManager(dictionary); - String schema = dictionary.getSchema(); - - ps.println("CREATE SCHEMA IF NOT EXISTS " + schema + " CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci;"); - ps.println(); - ps.println("USE " + schema + ";"); - ps.println(); - - try { - tm.printTemplate("cspro2sql_dictionary", ps); - tm.printTemplate("cspro2sql_error", ps); - } catch (IOException ex) { - return; - } - - for (Record record : dictionary.getRecords()) { - for (Item item : record.getItems()) { - printValueSet(schema, item, ps); - } - } - - for (Record record : dictionary.getRecords()) { - ps.println("CREATE TABLE " + record.getFullTableName() + " ("); - ps.println(" ID INT(9) UNSIGNED AUTO_INCREMENT,"); - if (!record.isMainRecord()) { - ps.println(" " + record.getMainRecord().getName() + " INT(9) UNSIGNED NOT NULL,"); - ps.println(" COUNTER INT(9) UNSIGNED NOT NULL,"); - } - for (Item item : record.getItems()) { - printItem(schema, foreignKeys, item, ps); - } - if (!record.isMainRecord()) { - ps.println(" INDEX (" + record.getMainRecord().getName() + "),"); - ps.println(" FOREIGN KEY (" + record.getMainRecord().getName() + ") REFERENCES " + record.getMainRecord().getFullTableName() + "(id),"); - } - ps.println(" PRIMARY KEY (ID)"); - ps.println(") ENGINE=INNODB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); - ps.println(); - } - } - - private static void printItem(String schema, boolean foreignKeys, Item item, PrintStream ps) { - String name = item.getName(); - int length = item.getLength(); - switch (item.getDataType()) { - case Dictionary.ITEM_ALPHA: - ps.println(" " + name + " CHAR(" + length + "),"); - break; - case Dictionary.ITEM_DECIMAL: - if (item.getDecimal() > 0) { - ps.println(" " + name + " DECIMAL(" + length + ", " + item.getDecimal() + "),"); - } else { - ps.println(" " + name + " DECIMAL(" + length + ", 0),"); - } - break; - default: - } - if (foreignKeys && item.hasValueSets()) { - ps.println(" FOREIGN KEY (" + name + ") REFERENCES " + schema + "." + item.getValueSetName() + "(ID),"); - } - for (Item subItem : item.getSubItems()) { - printItem(schema, foreignKeys, subItem, ps); - } - } - - private static void printValueSet(String schema, Item item, PrintStream ps) { - if (item.hasValueSets() && item.getValueSets().get(0).isNotCreated()) { - ps.println("CREATE TABLE " + schema + "." + item.getValueSetName() + " ("); - switch (item.getDataType()) { - case Dictionary.ITEM_ALPHA: - ps.println(" ID CHAR(" + item.getLength() + "),"); - break; - case Dictionary.ITEM_DECIMAL: - ps.println(" ID INT(" + item.getLength() + "),"); - break; - default: - } - ps.println(" VALUE TEXT,"); - ps.println(" PRIMARY KEY (ID)"); - ps.println(") ENGINE=INNODB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); - ps.println(); - boolean first = true; - Set keys = new HashSet<>(); - ps.println("INSERT INTO " + schema + "." + item.getValueSetName() + "(ID,VALUE) VALUES "); - for (int i = 0; i < item.getValueSets().size(); i++) { - ValueSet valueSet = item.getValueSets().get(i); - valueSet.setCreated(); - for (ValueSetValue e : valueSet.getValues().values()) { - if (keys.contains(e.getKey())) { - continue; - } - if (!first) { - ps.println(","); - } - ps.print(" (\"" + e.getKey() + "\",\"" + e.getValue().replace("\"", "\\\"") + "\")"); - keys.add(e.getKey()); - first = false; - } - } - ps.println(";"); - ps.println(); - } - for (Item subItem : item.getSubItems()) { - printValueSet(schema, subItem, ps); - } - } - -} +package cspro2sql.writer; + +import cspro2sql.bean.Dictionary; +import cspro2sql.bean.Item; +import cspro2sql.bean.Record; +import cspro2sql.bean.ValueSet; +import cspro2sql.bean.ValueSetValue; +import cspro2sql.sql.TemplateManager; +import java.io.IOException; +import java.io.PrintStream; +import java.util.HashSet; +import java.util.Set; + +/** + * Copyright 2017 ISTAT + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by + * the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. You may + * obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl5 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * Licence for the specific language governing permissions and limitations under + * the Licence. + * + * @author Guido Drovandi + * @author Mauro Bruno + * @version 0.9.17 + */ +public class SchemaWriter { + + public static void write(Dictionary dictionary, boolean foreignKeys, PrintStream ps) throws IOException { + TemplateManager tm = new TemplateManager(dictionary); + String schema = dictionary.getSchema(); + + ps.println("CREATE SCHEMA IF NOT EXISTS " + schema + " CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci;"); + ps.println(); + ps.println("USE " + schema + ";"); + ps.println(); + + try { + tm.printTemplate("cspro2sql_dictionary", ps); + tm.printTemplate("cspro2sql_error", ps); + } catch (IOException ex) { + return; + } + + for (Record record : dictionary.getRecords()) { + for (Item item : record.getItems()) { + printValueSet(schema, item, ps); + } + } + + for (Record record : dictionary.getRecords()) { + ps.println("CREATE TABLE " + record.getFullTableName() + " ("); + ps.println(" ID INT(9) UNSIGNED AUTO_INCREMENT,"); + if (!record.isMainRecord()) { + ps.println(" " + record.getMainRecord().getName() + " INT(9) UNSIGNED NOT NULL,"); + ps.println(" COUNTER INT(9) UNSIGNED NOT NULL,"); + } + for (Item item : record.getItems()) { + printItem(schema, foreignKeys, item, ps); + } + if (!record.isMainRecord()) { + ps.println(" INDEX (" + record.getMainRecord().getName() + "),"); + ps.println(" FOREIGN KEY (" + record.getMainRecord().getName() + ") REFERENCES " + record.getMainRecord().getFullTableName() + "(id),"); + } + ps.println(" PRIMARY KEY (ID)"); + ps.println(") ENGINE=INNODB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); + ps.println(); + } + } + + private static void printItem(String schema, boolean foreignKeys, Item item, PrintStream ps) { + String name = item.getName(); + int length = item.getLength(); + switch (item.getDataType()) { + case Dictionary.ITEM_ALPHA: + ps.println(" " + name + " CHAR(" + length + "),"); + break; + case Dictionary.ITEM_DECIMAL: + if (item.getDecimal() > 0) { + ps.println(" " + name + " DECIMAL(" + length + ", " + item.getDecimal() + "),"); + } else { + ps.println(" " + name + " DECIMAL(" + length + ", 0),"); + } + break; + default: + } + if (foreignKeys && item.hasValueSets()) { + ps.println(" FOREIGN KEY (" + name + ") REFERENCES " + schema + "." + item.getValueSetName() + "(ID),"); + } + for (Item subItem : item.getSubItems()) { + printItem(schema, foreignKeys, subItem, ps); + } + } + + private static void printValueSet(String schema, Item item, PrintStream ps) { + if (item.hasValueSets() && item.getValueSets().get(0).isNotCreated()) { + ps.println("CREATE TABLE " + schema + "." + item.getValueSetName() + " ("); + switch (item.getDataType()) { + case Dictionary.ITEM_ALPHA: + ps.println(" ID CHAR(" + item.getLength() + "),"); + break; + case Dictionary.ITEM_DECIMAL: + ps.println(" ID INT(" + item.getLength() + "),"); + break; + default: + } + ps.println(" VALUE TEXT,"); + ps.println(" PRIMARY KEY (ID)"); + ps.println(") ENGINE=INNODB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); + ps.println(); + boolean first = true; + Set keys = new HashSet<>(); + ps.println("INSERT INTO " + schema + "." + item.getValueSetName() + "(ID,VALUE) VALUES "); + for (int i = 0; i < item.getValueSets().size(); i++) { + ValueSet valueSet = item.getValueSets().get(i); + valueSet.setCreated(); + for (ValueSetValue e : valueSet.getValues().values()) { + if (keys.contains(e.getKey())) { + continue; + } + if (!first) { + ps.println(","); + } + ps.print(" (\"" + e.getKey() + "\",\"" + e.getValue().replace("\"", "\\\"") + "\")"); + keys.add(e.getKey()); + first = false; + } + } + ps.println(";"); + ps.println(); + } + for (Item subItem : item.getSubItems()) { + printValueSet(schema, subItem, ps); + } + } + +}