From e1d91f2e28126b483bf44b7cd5fc97e06d6ea142 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Thu, 21 Jun 2018 16:23:56 -0400 Subject: [PATCH 01/15] Fix missing join for expectedCounts When expectedCounts is in record that does not have any territory no join is generated for record containing expectedCounts --- src/cspro2sql/writer/MonitorWriter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index efe0a2f..f877276 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -202,13 +202,14 @@ private static void printAuxTable(TemplateManager mainTm, TemplateManager tm, St out.println(" " + item.getColunmFullName() + " AS " + mainTerritoryItem.getItemName() + ","); records.add(item.getRecord()); } - Record[] recArray = records.toArray(new Record[0]); Item expected = tm.getDictionary().getTaggedItem(Dictionary.TAG_EXPECTED_QUESTIONNAIRES); if (expected == null) { out.println(" COUNT(0) AS `" + columnName + "`"); } else { out.println(" SUM(" + expected.getColunmFullName() + ") AS `" + columnName + "`"); + records.add(expected.getRecord()); } + Record[] recArray = records.toArray(new Record[0]); out.println(" FROM"); out.println(" " + recArray[0].getMainRecord().getFullTableName()); for (Record record : recArray) { From e908fd382d60f83cca218115133e36f477151594 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Thu, 21 Jun 2018 17:55:12 -0400 Subject: [PATCH 02/15] Add gis.enabled to properties file to hide gis menu --- src/cspro2sql/Main.java | 572 +++++++++--------- src/cspro2sql/MonitorEngine.java | 8 +- src/cspro2sql/sql/template/dashboard_info.sql | 3 +- src/cspro2sql/writer/MonitorWriter.java | 5 +- 4 files changed, 296 insertions(+), 292 deletions(-) diff --git a/src/cspro2sql/Main.java b/src/cspro2sql/Main.java index ef861da..5e9fd72 100644 --- a/src/cspro2sql/Main.java +++ b/src/cspro2sql/Main.java @@ -1,286 +1,286 @@ -package cspro2sql; - -import cspro2sql.bean.Dictionary; -import cspro2sql.reader.DictionaryReader; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.PrintStream; -import java.io.UnsupportedEncodingException; -import java.util.List; -import java.util.Properties; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; - -/** - * Copyright 2017 ISTAT - * - * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by - * the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. You may - * obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl5 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * Licence for the specific language governing permissions and limitations under - * the Licence. - * - * @author Guido Drovandi - * @author Mauro Bruno - * @author Paolo Giacomi - * @version 0.9.18.2 - */ -public class Main { - - private static final String VERSION = "0.9.18.2"; - - public static void main(String[] args) { - CsPro2SqlOptions opts = getCommandLineOptions(args); - boolean error = false; - List dictionaries; - try { - dictionaries = DictionaryReader.parseDictionaries(opts.schema, opts.dictionary, opts.tablePrefix); - } catch (Exception e) { - opts.ps.close(); - opts.printHelp(e.getMessage()); - System.exit(1); - return; - } - - if (opts.schemaEngine) { - error = !SchemaEngine.execute(dictionaries, opts.foreignKeys, opts.ps); - } else if (opts.loaderEngine) { - if (opts.delay == null) { - error = !LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); - } else { - while (true) { - try { - LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); - } catch (Exception ex) { - System.err.println(ex.getMessage()); - } - try { - Thread.sleep(opts.delay); - } catch (Exception ex) { - System.err.println(ex.getMessage()); - } - } - } - } else if (opts.monitorEngine) { - error = !MonitorEngine.execute(dictionaries, opts.ps); - } else if (opts.updateEngine) { - error = !UpdateEngine.execute(opts.prop); - } else if (opts.statusEngine) { - error = !StatusEngine.execute(dictionaries, opts.prop); - } else if (opts.loadAndUpdate) { - while (true) { - try { - LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); - UpdateEngine.execute(opts.prop); - } catch (Exception ex) { - System.err.println(ex.getMessage()); - } - try { - Thread.sleep(opts.delay); - } catch (Exception ex) { - System.err.println(ex.getMessage()); - } - } - } - - if (opts.ps != null) { - opts.ps.close(); - } - - if (error) { - System.exit(1); - } - } - - private static CsPro2SqlOptions getCommandLineOptions(String[] args) { - Options options = new Options(); - options.addOption("a", "all", false, "transfer all the questionnaires"); - options.addOption("cc", "check-constraints", false, "perform constraints check"); - options.addOption("co", "check-only", false, "perform only constraints check (no data transfer)"); - options.addOption("e", "engine", true, "select engine: [loader|schema|monitor|update|status]"); - options.addOption("f", "force", false, "skip check of loader multiple running instances"); - options.addOption("fk", "foreign-keys", false, "create foreign keys to value sets"); - options.addOption("h", "help", false, "display this help"); - options.addOption("o", "output", true, "name of the output file"); - options.addOption("p", "properties", true, "properties file"); - options.addOption("r", "recovery", false, "recover a broken session of the loader"); - options.addOption("v", "version", false, "print the version of the programm"); - options.addOption("d", "delay", true, "perform again after DELAY minutes"); - - CsPro2SqlOptions opts = new CsPro2SqlOptions(options); - try { - CommandLineParser parser = new DefaultParser(); - CommandLine cmd = parser.parse(options, args); - - if (cmd.hasOption("h") || args.length == 0) { - opts.printHelp(); - } - if (cmd.hasOption("v")) { - opts.printVersion(); - } - - if (!cmd.hasOption("e")) { - opts.printHelp("The engine type is mandatory!"); - } - if (!cmd.hasOption("p")) { - opts.printHelp("The properties file is mandatory!"); - } - - opts.propertiesFile = cmd.getOptionValue("p"); - String engine = cmd.getOptionValue("e"); - switch (engine) { - case "schema": - opts.schemaEngine = true; - opts.foreignKeys = cmd.hasOption("fk"); - if (cmd.hasOption("o")) { - opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); - } else { - opts.ps = System.out; - } - break; - case "loader": - case "LU": - opts.loaderEngine = "loader".equals(engine); - opts.loadAndUpdate = "LU".equals(engine); - opts.checkConstraints = cmd.hasOption("cc"); - opts.checkOnly = cmd.hasOption("co"); - opts.allRecords = cmd.hasOption("a"); - opts.force = cmd.hasOption("f"); - opts.recovery = cmd.hasOption("r"); - if (cmd.hasOption("o")) { - opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); - } - if (opts.loadAndUpdate && !cmd.hasOption("d")) { - opts.printHelp("The delay is mandatory!"); - } - if (cmd.hasOption("d")) { - opts.delay = Integer.parseInt(cmd.getOptionValue("d")) * 60 * 1000; - if (opts.delay < 0) { - opts.printHelp("The delay cannot be negative!"); - } - } - break; - case "monitor": - opts.monitorEngine = true; - if (cmd.hasOption("o")) { - opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); - } else { - opts.ps = System.out; - } - break; - case "update": - opts.updateEngine = true; - break; - case "status": - opts.statusEngine = true; - break; - default: - opts.printHelp("Wrong engine type!"); - break; - } - } catch (ParseException | FileNotFoundException | UnsupportedEncodingException e) { - opts.printHelp(); - } - //End parsing command line - - //Load property file - Properties prop = new Properties(); - try (InputStream in = new FileInputStream(opts.propertiesFile)) { - prop.load(in); - } catch (IOException ex) { - System.out.println("Cannot read properties file '" + opts.propertiesFile + "'"); - opts.printHelp(); - } - - opts.prop = prop; - opts.dictionary = prop.getProperty("dictionary"); - opts.schema = prop.getProperty("db.dest.schema"); - if (opts.schema == null || opts.schema.isEmpty()) { - opts.printHelp("The database schema is mandatory!\nPlease set 'db.dest.schema' into the properties file"); - } - opts.tablePrefix = prop.getProperty("dictionary.prefix", ""); - - return opts; - } - - public static class CsPro2SqlOptions { - - boolean schemaEngine; - boolean loaderEngine; - boolean monitorEngine; - boolean updateEngine; - boolean statusEngine; - boolean loadAndUpdate; - boolean allRecords; - boolean foreignKeys; - boolean checkConstraints; - boolean checkOnly; - boolean force; - boolean recovery; - String dictionary; - String schema; - String tablePrefix; - String propertiesFile; - Integer delay; - PrintStream ps = null; - Properties prop; - private final Options options; - - CsPro2SqlOptions(Options options) { - this.options = options; - } - - void printHelp() { - printHelp(null); - } - - void printHelp(String errMessage) { - HelpFormatter formatter = new HelpFormatter(); - if (errMessage != null) { - System.err.println(errMessage); - } - System.out.println("CsPro2Sql - version " + VERSION + "\n"); - formatter.printHelp("\n\n" - + "CsPro2Sql -e schema -p PROPERTIES_FILE [-fk] [-o OUTPUT_FILE]\n" - + "CsPro2Sql -e loader -p PROPERTIES_FILE [-a] [-cc] [-co] [-f|-r] [-o OUTPUT_FILE] [-d DELAY]\n" - + "CsPro2Sql -e monitor -p PROPERTIES_FILE [-o OUTPUT_FILE]\n" - + "CsPro2Sql -e update -p PROPERTIES_FILE\n" - + "CsPro2Sql -e status -p PROPERTIES_FILE\n" - + "CsPro2Sql -e LU -p PROPERTIES_FILE -d DELAY [-a] [-cc] [-co] [-f|-r] [-o OUTPUT_FILE]\n" - + "CsPro2Sql -v\n" - + "\n" - + "Engines descriptions:\n" - + " - schema: create the sql script for microdata\n" - + " - loader: load microdata into the sql database\n" - + " - monitor: create the sql script to setup the monitoring system\n" - + " - update: update the reports of the monitoring system\n" - + " - status: print the loader status\n" - + " - LU: load and update\n" - + "\n", options); - if (errMessage == null) { - System.exit(0); - } else { - System.exit(1); - } - } - - void printVersion() { - System.out.println("CsPro2Sql - version " + VERSION); - System.exit(0); - } - } - -} +package cspro2sql; + +import cspro2sql.bean.Dictionary; +import cspro2sql.reader.DictionaryReader; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Properties; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +/** + * Copyright 2017 ISTAT + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by + * the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. You may + * obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl5 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * Licence for the specific language governing permissions and limitations under + * the Licence. + * + * @author Guido Drovandi + * @author Mauro Bruno + * @author Paolo Giacomi + * @version 0.9.18.2 + */ +public class Main { + + private static final String VERSION = "0.9.18.2"; + + public static void main(String[] args) { + CsPro2SqlOptions opts = getCommandLineOptions(args); + boolean error = false; + List dictionaries; + try { + dictionaries = DictionaryReader.parseDictionaries(opts.schema, opts.dictionary, opts.tablePrefix); + } catch (Exception e) { + opts.ps.close(); + opts.printHelp(e.getMessage()); + System.exit(1); + return; + } + + if (opts.schemaEngine) { + error = !SchemaEngine.execute(dictionaries, opts.foreignKeys, opts.ps); + } else if (opts.loaderEngine) { + if (opts.delay == null) { + error = !LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); + } else { + while (true) { + try { + LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); + } catch (Exception ex) { + System.err.println(ex.getMessage()); + } + try { + Thread.sleep(opts.delay); + } catch (Exception ex) { + System.err.println(ex.getMessage()); + } + } + } + } else if (opts.monitorEngine) { + error = !MonitorEngine.execute(dictionaries, opts.prop, opts.ps); + } else if (opts.updateEngine) { + error = !UpdateEngine.execute(opts.prop); + } else if (opts.statusEngine) { + error = !StatusEngine.execute(dictionaries, opts.prop); + } else if (opts.loadAndUpdate) { + while (true) { + try { + LoaderEngine.execute(dictionaries, opts.prop, opts.allRecords, opts.checkConstraints, opts.checkOnly, opts.force, opts.recovery, opts.ps); + UpdateEngine.execute(opts.prop); + } catch (Exception ex) { + System.err.println(ex.getMessage()); + } + try { + Thread.sleep(opts.delay); + } catch (Exception ex) { + System.err.println(ex.getMessage()); + } + } + } + + if (opts.ps != null) { + opts.ps.close(); + } + + if (error) { + System.exit(1); + } + } + + private static CsPro2SqlOptions getCommandLineOptions(String[] args) { + Options options = new Options(); + options.addOption("a", "all", false, "transfer all the questionnaires"); + options.addOption("cc", "check-constraints", false, "perform constraints check"); + options.addOption("co", "check-only", false, "perform only constraints check (no data transfer)"); + options.addOption("e", "engine", true, "select engine: [loader|schema|monitor|update|status]"); + options.addOption("f", "force", false, "skip check of loader multiple running instances"); + options.addOption("fk", "foreign-keys", false, "create foreign keys to value sets"); + options.addOption("h", "help", false, "display this help"); + options.addOption("o", "output", true, "name of the output file"); + options.addOption("p", "properties", true, "properties file"); + options.addOption("r", "recovery", false, "recover a broken session of the loader"); + options.addOption("v", "version", false, "print the version of the programm"); + options.addOption("d", "delay", true, "perform again after DELAY minutes"); + + CsPro2SqlOptions opts = new CsPro2SqlOptions(options); + try { + CommandLineParser parser = new DefaultParser(); + CommandLine cmd = parser.parse(options, args); + + if (cmd.hasOption("h") || args.length == 0) { + opts.printHelp(); + } + if (cmd.hasOption("v")) { + opts.printVersion(); + } + + if (!cmd.hasOption("e")) { + opts.printHelp("The engine type is mandatory!"); + } + if (!cmd.hasOption("p")) { + opts.printHelp("The properties file is mandatory!"); + } + + opts.propertiesFile = cmd.getOptionValue("p"); + String engine = cmd.getOptionValue("e"); + switch (engine) { + case "schema": + opts.schemaEngine = true; + opts.foreignKeys = cmd.hasOption("fk"); + if (cmd.hasOption("o")) { + opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); + } else { + opts.ps = System.out; + } + break; + case "loader": + case "LU": + opts.loaderEngine = "loader".equals(engine); + opts.loadAndUpdate = "LU".equals(engine); + opts.checkConstraints = cmd.hasOption("cc"); + opts.checkOnly = cmd.hasOption("co"); + opts.allRecords = cmd.hasOption("a"); + opts.force = cmd.hasOption("f"); + opts.recovery = cmd.hasOption("r"); + if (cmd.hasOption("o")) { + opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); + } + if (opts.loadAndUpdate && !cmd.hasOption("d")) { + opts.printHelp("The delay is mandatory!"); + } + if (cmd.hasOption("d")) { + opts.delay = Integer.parseInt(cmd.getOptionValue("d")) * 60 * 1000; + if (opts.delay < 0) { + opts.printHelp("The delay cannot be negative!"); + } + } + break; + case "monitor": + opts.monitorEngine = true; + if (cmd.hasOption("o")) { + opts.ps = new PrintStream(cmd.getOptionValue("o"), "UTF-8"); + } else { + opts.ps = System.out; + } + break; + case "update": + opts.updateEngine = true; + break; + case "status": + opts.statusEngine = true; + break; + default: + opts.printHelp("Wrong engine type!"); + break; + } + } catch (ParseException | FileNotFoundException | UnsupportedEncodingException e) { + opts.printHelp(); + } + //End parsing command line + + //Load property file + Properties prop = new Properties(); + try (InputStream in = new FileInputStream(opts.propertiesFile)) { + prop.load(in); + } catch (IOException ex) { + System.out.println("Cannot read properties file '" + opts.propertiesFile + "'"); + opts.printHelp(); + } + + opts.prop = prop; + opts.dictionary = prop.getProperty("dictionary"); + opts.schema = prop.getProperty("db.dest.schema"); + if (opts.schema == null || opts.schema.isEmpty()) { + opts.printHelp("The database schema is mandatory!\nPlease set 'db.dest.schema' into the properties file"); + } + opts.tablePrefix = prop.getProperty("dictionary.prefix", ""); + + return opts; + } + + public static class CsPro2SqlOptions { + + boolean schemaEngine; + boolean loaderEngine; + boolean monitorEngine; + boolean updateEngine; + boolean statusEngine; + boolean loadAndUpdate; + boolean allRecords; + boolean foreignKeys; + boolean checkConstraints; + boolean checkOnly; + boolean force; + boolean recovery; + String dictionary; + String schema; + String tablePrefix; + String propertiesFile; + Integer delay; + PrintStream ps = null; + Properties prop; + private final Options options; + + CsPro2SqlOptions(Options options) { + this.options = options; + } + + void printHelp() { + printHelp(null); + } + + void printHelp(String errMessage) { + HelpFormatter formatter = new HelpFormatter(); + if (errMessage != null) { + System.err.println(errMessage); + } + System.out.println("CsPro2Sql - version " + VERSION + "\n"); + formatter.printHelp("\n\n" + + "CsPro2Sql -e schema -p PROPERTIES_FILE [-fk] [-o OUTPUT_FILE]\n" + + "CsPro2Sql -e loader -p PROPERTIES_FILE [-a] [-cc] [-co] [-f|-r] [-o OUTPUT_FILE] [-d DELAY]\n" + + "CsPro2Sql -e monitor -p PROPERTIES_FILE [-o OUTPUT_FILE]\n" + + "CsPro2Sql -e update -p PROPERTIES_FILE\n" + + "CsPro2Sql -e status -p PROPERTIES_FILE\n" + + "CsPro2Sql -e LU -p PROPERTIES_FILE -d DELAY [-a] [-cc] [-co] [-f|-r] [-o OUTPUT_FILE]\n" + + "CsPro2Sql -v\n" + + "\n" + + "Engines descriptions:\n" + + " - schema: create the sql script for microdata\n" + + " - loader: load microdata into the sql database\n" + + " - monitor: create the sql script to setup the monitoring system\n" + + " - update: update the reports of the monitoring system\n" + + " - status: print the loader status\n" + + " - LU: load and update\n" + + "\n", options); + if (errMessage == null) { + System.exit(0); + } else { + System.exit(1); + } + } + + void printVersion() { + System.out.println("CsPro2Sql - version " + VERSION); + System.exit(0); + } + } + +} diff --git a/src/cspro2sql/MonitorEngine.java b/src/cspro2sql/MonitorEngine.java index cae6faa..84228c7 100644 --- a/src/cspro2sql/MonitorEngine.java +++ b/src/cspro2sql/MonitorEngine.java @@ -44,13 +44,13 @@ public static void main(String[] args) throws Exception { prop.getProperty("db.dest.schema"), prop.getProperty("dictionary"), prop.getProperty("dictionary.prefix")); - execute(dictionaries, System.out); + execute(dictionaries, prop, System.out); } catch (Exception ex) { System.exit(1); } } - static boolean execute(List dictionaries, PrintStream out) { + static boolean execute(List dictionaries, Properties prop, PrintStream out) { TemplateManager tmFieldwork = null, tmListing = null, tmExpected = null; for (Dictionary dictionary : dictionaries) { if (dictionary.hasTag(Dictionary.TAG_FIELDWORK)) { @@ -61,7 +61,9 @@ static boolean execute(List dictionaries, PrintStream out) { tmExpected = new TemplateManager(dictionary); } } - return MonitorWriter.write(tmFieldwork, tmListing, tmExpected, out); + + boolean gisEnabled = prop.getProperty("gis.enabled") != null && (prop.getProperty("gis.enabled").equalsIgnoreCase("true") || prop.getProperty("gis.enabled").equals("1")); + return MonitorWriter.write(tmFieldwork, tmListing, tmExpected, gisEnabled ,out); } } diff --git a/src/cspro2sql/sql/template/dashboard_info.sql b/src/cspro2sql/sql/template/dashboard_info.sql index 530230c..fb68731 100644 --- a/src/cspro2sql/sql/template/dashboard_info.sql +++ b/src/cspro2sql/sql/template/dashboard_info.sql @@ -2,7 +2,8 @@ CREATE TABLE IF NOT EXISTS @SCHEMA.DASHBOARD_INFO ( `ID` int(1) NOT NULL DEFAULT 0, `LISTING` int(1) NOT NULL DEFAULT 0, `EXPECTED` int(1) NOT NULL DEFAULT 0, + `GIS` int(1) NOT NULL DEFAULT 0, PRIMARY KEY (`ID`) ) ENGINE=INNODB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -INSERT INTO @SCHEMA.DASHBOARD_INFO values (0, @LISTING, @EXPECTED); +INSERT INTO @SCHEMA.DASHBOARD_INFO values (0, @LISTING, @EXPECTED, @GIS); diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index f877276..09e7c87 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -42,12 +42,13 @@ public class MonitorWriter { private static int reportCount = 0; - public static boolean write(TemplateManager tm, TemplateManager tmListing, TemplateManager tmExpected, PrintStream out) { + public static boolean write(TemplateManager tm, TemplateManager tmListing, TemplateManager tmExpected, boolean gisEnabled, PrintStream out) { String schema = tm.getDictionary().getSchema(); tm.addParam("@LISTING", tmListing == null ? "0" : "1"); tm.addParam("@EXPECTED", tmExpected == null ? "0" : "1"); - + tm.addParam("@GIS", gisEnabled ? "1" : "0"); + out.println("USE " + schema + ";"); out.println(); From 38ec6f19072b57ddf8f21e8a85e75cc7df9a9d39 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Sun, 24 Jun 2018 14:46:15 -0400 Subject: [PATCH 03/15] Fill territory table by specifying CSPro anm file in properties --- src/cspro2sql/Main.java | 11 +- src/cspro2sql/MonitorEngine.java | 12 +- src/cspro2sql/bean/AreaNameFile.java | 45 ++++++ src/cspro2sql/bean/NamedArea.java | 39 +++++ src/cspro2sql/reader/AreaNameFileReader.java | 150 +++++++++++++++++++ src/cspro2sql/writer/MonitorWriter.java | 100 ++++++++++--- 6 files changed, 335 insertions(+), 22 deletions(-) create mode 100644 src/cspro2sql/bean/AreaNameFile.java create mode 100644 src/cspro2sql/bean/NamedArea.java create mode 100644 src/cspro2sql/reader/AreaNameFileReader.java diff --git a/src/cspro2sql/Main.java b/src/cspro2sql/Main.java index 5e9fd72..41aa3e3 100644 --- a/src/cspro2sql/Main.java +++ b/src/cspro2sql/Main.java @@ -1,6 +1,8 @@ package cspro2sql; +import cspro2sql.bean.AreaNameFile; import cspro2sql.bean.Dictionary; +import cspro2sql.reader.AreaNameFileReader; import cspro2sql.reader.DictionaryReader; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -46,15 +48,20 @@ public static void main(String[] args) { CsPro2SqlOptions opts = getCommandLineOptions(args); boolean error = false; List dictionaries; + AreaNameFile areaNames = null; try { dictionaries = DictionaryReader.parseDictionaries(opts.schema, opts.dictionary, opts.tablePrefix); + + if (opts.prop.getProperty("geography") != null) + areaNames = AreaNameFileReader.parseAreaNamesFile(opts.prop.getProperty("geography")); + } catch (Exception e) { opts.ps.close(); opts.printHelp(e.getMessage()); System.exit(1); return; } - + if (opts.schemaEngine) { error = !SchemaEngine.execute(dictionaries, opts.foreignKeys, opts.ps); } else if (opts.loaderEngine) { @@ -75,7 +82,7 @@ public static void main(String[] args) { } } } else if (opts.monitorEngine) { - error = !MonitorEngine.execute(dictionaries, opts.prop, opts.ps); + error = !MonitorEngine.execute(dictionaries, areaNames, opts.prop, opts.ps); } else if (opts.updateEngine) { error = !UpdateEngine.execute(opts.prop); } else if (opts.statusEngine) { diff --git a/src/cspro2sql/MonitorEngine.java b/src/cspro2sql/MonitorEngine.java index 84228c7..6d918d0 100644 --- a/src/cspro2sql/MonitorEngine.java +++ b/src/cspro2sql/MonitorEngine.java @@ -1,6 +1,8 @@ package cspro2sql; +import cspro2sql.bean.AreaNameFile; import cspro2sql.bean.Dictionary; +import cspro2sql.reader.AreaNameFileReader; import cspro2sql.reader.DictionaryReader; import cspro2sql.sql.TemplateManager; import cspro2sql.writer.MonitorWriter; @@ -44,13 +46,17 @@ public static void main(String[] args) throws Exception { prop.getProperty("db.dest.schema"), prop.getProperty("dictionary"), prop.getProperty("dictionary.prefix")); - execute(dictionaries, prop, System.out); + + final AreaNameFile areaNames = prop.getProperty("geography") != null + ? AreaNameFileReader.parseAreaNamesFile(prop.getProperty("geography")) + : null; + execute(dictionaries, areaNames, prop, System.out); } catch (Exception ex) { System.exit(1); } } - static boolean execute(List dictionaries, Properties prop, PrintStream out) { + static boolean execute(List dictionaries, AreaNameFile areaNames, Properties prop, PrintStream out) { TemplateManager tmFieldwork = null, tmListing = null, tmExpected = null; for (Dictionary dictionary : dictionaries) { if (dictionary.hasTag(Dictionary.TAG_FIELDWORK)) { @@ -63,7 +69,7 @@ static boolean execute(List dictionaries, Properties prop, PrintStre } boolean gisEnabled = prop.getProperty("gis.enabled") != null && (prop.getProperty("gis.enabled").equalsIgnoreCase("true") || prop.getProperty("gis.enabled").equals("1")); - return MonitorWriter.write(tmFieldwork, tmListing, tmExpected, gisEnabled ,out); + return MonitorWriter.write(tmFieldwork, tmListing, tmExpected, areaNames, gisEnabled ,out); } } diff --git a/src/cspro2sql/bean/AreaNameFile.java b/src/cspro2sql/bean/AreaNameFile.java new file mode 100644 index 0000000..d890181 --- /dev/null +++ b/src/cspro2sql/bean/AreaNameFile.java @@ -0,0 +1,45 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package cspro2sql.bean; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * + * @author josh + */ +public class AreaNameFile { + + private final String[] levels; + private final NamedArea[] areas; + private final Map, NamedArea> codeToArea; + + public AreaNameFile(String[] levels, NamedArea[] areas) + { + this.levels = levels; + this.areas = areas; + codeToArea = new HashMap<>(); + for (NamedArea area : areas) + codeToArea.put(area.getCodes(), area); + } + + public String[] getLevels() + { + return levels; + } + + public NamedArea[] getAreas() + { + return areas; + } + + public NamedArea lookup(List codes) + { + return codeToArea.get(codes); + } +} diff --git a/src/cspro2sql/bean/NamedArea.java b/src/cspro2sql/bean/NamedArea.java new file mode 100644 index 0000000..b7304eb --- /dev/null +++ b/src/cspro2sql/bean/NamedArea.java @@ -0,0 +1,39 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package cspro2sql.bean; + +import java.util.List; + +/** + * + * @author josh + */ +public class NamedArea { + + private final String name; + private final List codes; + + public NamedArea(String name, List codes) + { + this.name = name; + this.codes = codes; + } + + public String getName() + { + return name; + } + + public List getCodes() + { + return codes; + } + + public int level() + { + return codes.size(); + } +} diff --git a/src/cspro2sql/reader/AreaNameFileReader.java b/src/cspro2sql/reader/AreaNameFileReader.java new file mode 100644 index 0000000..e86a759 --- /dev/null +++ b/src/cspro2sql/reader/AreaNameFileReader.java @@ -0,0 +1,150 @@ + +package cspro2sql.reader; + +import cspro2sql.bean.AreaNameFile; +import cspro2sql.bean.NamedArea; +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** +* Copyright 2017 ISTAT + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by + * the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. You may + * obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl5 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * Licence for the specific language governing permissions and limitations under + * the Licence. + * + * @author Josh Handley + */ +public class AreaNameFileReader { + private static final String AREA_NAMES = "[Area Names]"; + private static final String LEVELS = "[Levels]"; + private static final String AREAS = "[Areas]"; + private static final Pattern VERSION_LINE = Pattern.compile("Version\\s*=\\s*CSPro ([0-9]+\\.[0-9]+)", Pattern.CASE_INSENSITIVE); + private static final Pattern LEVEL_NAME = Pattern.compile("Name\\s*=\\s*(.+)", Pattern.CASE_INSENSITIVE); + private static final Pattern AREA_LINE = Pattern.compile("((?:(?:X|[0-9]+)\\s*,?\\s*)+)=\\s*(.+)", Pattern.CASE_INSENSITIVE); + private static final Pattern AREA_CODE = Pattern.compile("(X|[0-9]+)\\s*,?\\s*", Pattern.CASE_INSENSITIVE); + + public static AreaNameFile parseAreaNamesFile(String filename) throws Exception + { + try { + return read(filename); + } catch (IOException ex) { + throw new Exception("Impossible to read dictionary " + filename + " (" + ex.getMessage() + ")", ex); + } + } + + private static AreaNameFile read(String fileName) throws IOException + { + try (InputStream in = new FileInputStream(fileName)) { + try (InputStreamReader fr = new InputStreamReader(in, "UTF-8")) { + try (BufferedReader br = new BufferedReader(fr)) { + return read(br); + } + } + } + } + + private static AreaNameFile read(BufferedReader br) throws IOException { + + br.mark(4); + if ('\ufeff' != br.read()) + br.reset(); + readThroughSectionHeader(br, AREA_NAMES); + String version = readVersion(br); + if (version == null) + throw new IOException("Missing version"); + String[] levels = readLevels(br); + NamedArea[] areas = readAreas(br); + return new AreaNameFile(levels, areas); + } + + private static void readThroughSectionHeader(BufferedReader br, String header) throws IOException + { + String line; + while ((line = br.readLine().trim()) != null) { + if (line.equalsIgnoreCase(header)) + return; + else if (!line.isEmpty() ) + throw new IOException("Expected " + header + " got " + line ); + } + } + + private static String readVersion(BufferedReader br) throws IOException { + String line; + String version = null; + Matcher matcher; + while ((line = br.readLine().trim()) != null) { + if (line.equalsIgnoreCase(LEVELS)) + { + break; + } + else if ((matcher = VERSION_LINE.matcher(line)).matches()) + { + version = matcher.group(1); + } else if (!line.isEmpty()) + { + throw new IOException("Expected 'name=level' got " + line ); + } + } + return version; + } + + private static String[] readLevels(BufferedReader br) throws IOException { + ArrayList levels = new ArrayList<>(); + String line; + Matcher matcher; + while ((line = br.readLine().trim()) != null) { + if (line.equalsIgnoreCase(AREAS)) { + break; + } else if ((matcher = LEVEL_NAME.matcher(line)).matches()) + { + levels.add(matcher.group(1)); + } else if (!line.isEmpty()) + { + throw new IOException("Expected 'name=level' got " + line ); + } + } + return levels.toArray(new String[0]); + } + + private static NamedArea[] readAreas(BufferedReader br) throws IOException { + ArrayList areas = new ArrayList<>(); + String line; + Matcher lineMatcher; + while ((line = br.readLine()) != null) { + if ((lineMatcher = AREA_LINE.matcher(line)).matches()) + { + String name = lineMatcher.group(2); + String codesString = lineMatcher.group(1); + ArrayList codes = new ArrayList<>(); + Matcher codeMatcher = AREA_CODE.matcher(codesString); + while (codeMatcher.find()) { + String code = codeMatcher.group(1); + if (code.equals("X")) + break; + else + codes.add(Integer.parseInt(code)); + } + areas.add(new NamedArea(name, codes)); + } else if (!line.isEmpty()) { + throw new IOException("Expected 'name=level' got " + line ); + } + } + return areas.toArray(new NamedArea[0]); + } +} diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index 09e7c87..e80212b 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -1,14 +1,18 @@ package cspro2sql.writer; +import cspro2sql.bean.AreaNameFile; import cspro2sql.bean.Dictionary; import cspro2sql.bean.Item; +import cspro2sql.bean.NamedArea; import cspro2sql.bean.Record; import cspro2sql.bean.Territory; import cspro2sql.bean.TerritoryItem; import cspro2sql.sql.TemplateManager; import java.io.IOException; import java.io.PrintStream; +import java.util.Formatter; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; /** @@ -42,7 +46,7 @@ public class MonitorWriter { private static int reportCount = 0; - public static boolean write(TemplateManager tm, TemplateManager tmListing, TemplateManager tmExpected, boolean gisEnabled, PrintStream out) { + public static boolean write(TemplateManager tm, TemplateManager tmListing, TemplateManager tmExpected, AreaNameFile areaNames, boolean gisEnabled, PrintStream out) { String schema = tm.getDictionary().getSchema(); tm.addParam("@LISTING", tmListing == null ? "0" : "1"); @@ -74,22 +78,12 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ } if (!territory.isEmpty()) { - out.println("CREATE TABLE IF NOT EXISTS `territory` ("); - String idx = ""; - for (int i = 0; i < territory.size(); i++) { - TerritoryItem territoryItem = territory.get(i); - String name = territoryItem.getItemName(); - out.println(" `" + name + "_NAME` text COLLATE utf8mb4_unicode_ci,"); - out.println(" `" + name + "` int(11) DEFAULT NULL,"); - if (i > 0) { - idx += ","; - } - idx += "`" + name + "`"; + try { + printTerritoryTable(territory, areaNames, out); + } catch (IOException ex) { + System.err.println(ex.getMessage()); + return false; } - out.println(" `TERRITORY_CODE` text COLLATE utf8mb4_unicode_ci,"); - out.println(" KEY `idx_territory` (" + idx + ")"); - out.println(") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); - out.println(); } if (!territory.isEmpty()) { @@ -178,10 +172,82 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ printTotalReport(tm, (tmListing != null) ? tmListing : tm, out); printMaterialized(schema, "r_total", out); } - + return true; } + private static void printTerritoryTable(Territory territory, AreaNameFile areaNames, PrintStream out) throws IOException { + out.println("CREATE TABLE IF NOT EXISTS `territory` ("); + String idx = ""; + for (int i = 0; i < territory.size(); i++) { + TerritoryItem territoryItem = territory.get(i); + String name = territoryItem.getItemName(); + out.println(" `" + name + "_NAME` text COLLATE utf8mb4_unicode_ci,"); + out.println(" `" + name + "` int(11) DEFAULT NULL,"); + if (i > 0) { + idx += ","; + } + idx += "`" + name + "`"; + } + out.println(" `TERRITORY_CODE` text COLLATE utf8mb4_unicode_ci,"); + out.println(" KEY `idx_territory` (" + idx + ")"); + out.println(") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); + out.println(); + + if (areaNames != null) { + final int numLevels = areaNames.getLevels().length; + if (territory.size() != numLevels) { + throw new IOException("Area names file does not match #territory items in dictionary"); + } + + StringBuilder insertPreambleBuilder = new StringBuilder(); + insertPreambleBuilder.append("INSERT INTO `territory` ("); + for (int i = 0; i < territory.size(); i++) { + TerritoryItem territoryItem = territory.get(i); + String name = territoryItem.getItemName(); + insertPreambleBuilder.append("`"); + insertPreambleBuilder.append(name); + insertPreambleBuilder.append("_NAME`,`"); + insertPreambleBuilder.append(name); + insertPreambleBuilder.append("`"); + if (i != territory.size() - 1) + insertPreambleBuilder.append(","); + } + insertPreambleBuilder.append(",`TERRITORY_CODE`) VALUES ("); + final String insertPreamble = insertPreambleBuilder.toString(); + String[] formatSpecs = new String[numLevels]; + for (int i = 0; i < numLevels; i++) { + Item territoryDictItem = territory.get(i).getItem(); + if (territoryDictItem.isZeroFill()) + formatSpecs[i] = "%0" + territoryDictItem.getLength() + "d"; + else + formatSpecs[i] = "%" + territoryDictItem.getLength() + "d"; + } + for (NamedArea area : areaNames.getAreas()) { + + // Ignore higher level entities (province, district) + // only insert lowest level entities (EA) + if (area.level() == numLevels) { + out.print(insertPreamble); + for (int i = 0; i < numLevels - 1; i++) { + List ancestorCode = area.getCodes().subList(0, i + 1); + NamedArea ancestor = areaNames.lookup(ancestorCode); + String ancestorName = ancestor != null ? ancestor.getName().replace("'", "''") : String.valueOf(area.getCodes().get(i)); + out.print("'" + ancestorName + "'," + area.getCodes().get(i) + ","); + } + + StringBuilder fullCodeBuilder = new StringBuilder(); + Formatter fmt = new Formatter(fullCodeBuilder); + for (int i = 0; i < numLevels; i++) { + fmt.format(formatSpecs[i], area.getCodes().get(i)); + } + + out.println("'" + area.getName() + "'," + area.getCodes().get(numLevels - 1) + ",'" + fullCodeBuilder.toString() + "');"); + } + } + } + } + private static void printMaterialized(String schema, String name, PrintStream out) { out.println("DROP TABLE IF EXISTS " + schema + ".m" + name + ";"); out.println("SELECT 0 INTO @ID;"); From ea6ca4115f82be4db21bb50394b6f52e19af0609 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Mon, 25 Jun 2018 11:35:18 -0400 Subject: [PATCH 04/15] Generate questionniare_info_region, region tables --- src/cspro2sql/sql/TemplateManager.java | 5 ++++ .../sql/template/r_first_level_geography.sql | 4 ++++ .../template/r_questionnaire_info_region.sql | 23 +++++++++++++++++++ src/cspro2sql/writer/MonitorWriter.java | 22 ++++++++++++++---- 4 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 src/cspro2sql/sql/template/r_first_level_geography.sql create mode 100644 src/cspro2sql/sql/template/r_questionnaire_info_region.sql diff --git a/src/cspro2sql/sql/TemplateManager.java b/src/cspro2sql/sql/TemplateManager.java index 088b35c..02efdeb 100644 --- a/src/cspro2sql/sql/TemplateManager.java +++ b/src/cspro2sql/sql/TemplateManager.java @@ -60,6 +60,11 @@ public TemplateManager(Dictionary dictionary) { for (Item territory : territories) { this.territory.addItem(territory); } + + // Assume that geographic ids are in order in dictionary so region is + // the first one. Depending on country may not be a region but it should + // be the highest level of geography which is what we want for reports. + this.params.put("@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO", territory.getFirst().getItemName()); } if (dictionary.hasTagged(Dictionary.TAG_AGE)) { diff --git a/src/cspro2sql/sql/template/r_first_level_geography.sql b/src/cspro2sql/sql/template/r_first_level_geography.sql new file mode 100644 index 0000000..d1008b7 --- /dev/null +++ b/src/cspro2sql/sql/template/r_first_level_geography.sql @@ -0,0 +1,4 @@ +CREATE OR REPLACE VIEW @SCHEMA.r_first_level_geography AS +SELECT @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS CODE, @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO_NAME AS NAME +FROM TERRITORY +GROUP BY @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO, @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO_NAME; \ No newline at end of file diff --git a/src/cspro2sql/sql/template/r_questionnaire_info_region.sql b/src/cspro2sql/sql/template/r_questionnaire_info_region.sql new file mode 100644 index 0000000..5ac14b3 --- /dev/null +++ b/src/cspro2sql/sql/template/r_questionnaire_info_region.sql @@ -0,0 +1,23 @@ +CREATE OR REPLACE VIEW @SCHEMA.`r_questionnaire_info_region` AS + SELECT + `b`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS `region`, + `b`.`total` AS `total`, + `avg_individual`.`avg_individual` AS `avg_individual` + FROM + (((SELECT + @SCHEMA.@QUESTIONNAIRE_TABLE.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO, + COUNT(0) AS `total` + FROM + @SCHEMA.@QUESTIONNAIRE_TABLE + GROUP BY @SCHEMA.@QUESTIONNAIRE_TABLE.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO)) `b` + JOIN (SELECT + `a`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO, AVG(`a`.`num`) AS `avg_individual` + FROM + (SELECT + `q`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS @QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO, COUNT(0) AS `num` + FROM + (@SCHEMA.@INDIVIDUAL_TABLE `h` + JOIN @SCHEMA.@QUESTIONNAIRE_TABLE `q` ON ((`h`.@QUESTIONNAIRE_COLUMN_BASE = `q`.`ID`))) + GROUP BY `q`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO , `h`.@QUESTIONNAIRE_COLUMN_BASE) `a` + GROUP BY `a`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO) `avg_individual` ON ((`b`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO = `avg_individual`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO))) + GROUP BY `b`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO; diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index e80212b..19afa9a 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -41,7 +41,8 @@ public class MonitorWriter { "r_questionnaire_info", "r_individual_info", "r_religion", - "r_sex_by_age" + "r_sex_by_age", + "r_first_level_geography" }; private static int reportCount = 0; @@ -59,6 +60,16 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ Territory territory = tm.getTerritory(); int[] ageRange = tm.getAgeRange(); + if (!territory.isEmpty()) { + try { + printTerritoryTable(territory, areaNames, out); + + } catch (IOException ex) { + System.err.println(ex.getMessage()); + return false; + } + } + try { tm.printTemplate("c_user", out); tm.printTemplate("cspro2sql_report", out); @@ -78,8 +89,9 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ } if (!territory.isEmpty()) { - try { - printTerritoryTable(territory, areaNames, out); + try { + if (tm.printTemplate("r_questionnaire_info_region", out)) + printMaterialized(schema, "r_questionnaire_info_region", out); } catch (IOException ex) { System.err.println(ex.getMessage()); return false; @@ -200,6 +212,8 @@ private static void printTerritoryTable(Territory territory, AreaNameFile areaNa throw new IOException("Area names file does not match #territory items in dictionary"); } + out.println("TRUNCATE TABLE `territory`;"); + StringBuilder insertPreambleBuilder = new StringBuilder(); insertPreambleBuilder.append("INSERT INTO `territory` ("); for (int i = 0; i < territory.size(); i++) { @@ -247,7 +261,7 @@ private static void printTerritoryTable(Territory territory, AreaNameFile areaNa } } } - + private static void printMaterialized(String schema, String name, PrintStream out) { out.println("DROP TABLE IF EXISTS " + schema + ".m" + name + ";"); out.println("SELECT 0 INTO @ID;"); From 8b447258ac6588475030ffd41ebc161a17d32f27 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Tue, 26 Jun 2018 06:30:21 -0400 Subject: [PATCH 05/15] Generate sex_by_age_group_region report --- .../template/r_sex_by_age_group_region.sql | 533 ++++++++++++++++++ src/cspro2sql/writer/MonitorWriter.java | 3 +- 2 files changed, 535 insertions(+), 1 deletion(-) create mode 100644 src/cspro2sql/sql/template/r_sex_by_age_group_region.sql diff --git a/src/cspro2sql/sql/template/r_sex_by_age_group_region.sql b/src/cspro2sql/sql/template/r_sex_by_age_group_region.sql new file mode 100644 index 0000000..078aab5 --- /dev/null +++ b/src/cspro2sql/sql/template/r_sex_by_age_group_region.sql @@ -0,0 +1,533 @@ +CREATE OR REPLACE VIEW @SCHEMA.`aux_sex_age_region` AS + SELECT + `q`.@QUESTIONNAIRE_COLUMN_FIRST_LEVEL_GEO AS `first_level_geo`, + `h`.@INDIVIDUAL_COLUMN_SEX AS `sex`, + `h`.@INDIVIDUAL_COLUMN_AGE AS `age` + FROM + (@SCHEMA.@INDIVIDUAL_TABLE `h` + JOIN @SCHEMA.@QUESTIONNAIRE_TABLE `q` ON ((`h`.@QUESTIONNAIRE_COLUMN_BASE = `q`.`ID`))); + +CREATE OR REPLACE VIEW @SCHEMA.`r_sex_by_age_group_region` AS + SELECT + `a`.`first_level_geo` AS `region`, + '0 to 4' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 0) + AND (`aux_sex_age_region`.`age` < 5)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 0) + AND (`aux_sex_age_region`.`age` < 5)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '5 to 9' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 5) + AND (`aux_sex_age_region`.`age` < 10)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 5) + AND (`aux_sex_age_region`.`age` < 10)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '10 to 14' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = 1) + AND (`aux_sex_age_region`.`age` >= 10) + AND (`aux_sex_age_region`.`age` < 15)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 10) + AND (`aux_sex_age_region`.`age` < 15)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '15 to 19' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 15) + AND (`aux_sex_age_region`.`age` < 20)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 15) + AND (`aux_sex_age_region`.`age` < 20)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '20 to 24' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 20) + AND (`aux_sex_age_region`.`age` < 25)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 20) + AND (`aux_sex_age_region`.`age` < 25)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '25 to 29' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 25) + AND (`aux_sex_age_region`.`age` < 30)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 25) + AND (`aux_sex_age_region`.`age` < 30)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '30 to 34' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 30) + AND (`aux_sex_age_region`.`age` < 35)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 30) + AND (`aux_sex_age_region`.`age` < 35)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '35 to 39' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 35) + AND (`aux_sex_age_region`.`age` < 40)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 35) + AND (`aux_sex_age_region`.`age` < 40)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '40 to 44' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 40) + AND (`aux_sex_age_region`.`age` < 45)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 40) + AND (`aux_sex_age_region`.`age` < 45)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '45 to 49' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 45) + AND (`aux_sex_age_region`.`age` < 50)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 45) + AND (`aux_sex_age_region`.`age` < 50)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '50 to 54' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 50) + AND (`aux_sex_age_region`.`age` < 55)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 50) + AND (`aux_sex_age_region`.`age` < 55)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '55 to 59' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 55) + AND (`aux_sex_age_region`.`age` < 60)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 55) + AND (`aux_sex_age_region`.`age` < 60)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '60 to 64' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 60) + AND (`aux_sex_age_region`.`age` < 65)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 60) + AND (`aux_sex_age_region`.`age` < 65)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '65 to 69' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 65) + AND (`aux_sex_age_region`.`age` < 70)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 65) + AND (`aux_sex_age_region`.`age` < 70)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '70 to 74' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 70) + AND (`aux_sex_age_region`.`age` < 75)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 70) + AND (`aux_sex_age_region`.`age` < 75)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '75 to 79' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 75) + AND (`aux_sex_age_region`.`age` < 80)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 75) + AND (`aux_sex_age_region`.`age` < 80)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '80 to 84' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 80) + AND (`aux_sex_age_region`.`age` < 85)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 80) + AND (`aux_sex_age_region`.`age` < 85)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '85 to 89' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 85) + AND (`aux_sex_age_region`.`age` < 90)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 85) + AND (`aux_sex_age_region`.`age` < 90)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '90 to 94' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 90) + AND (`aux_sex_age_region`.`age` < 95)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 90) + AND (`aux_sex_age_region`.`age` < 95)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '95 to 97' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 95) + AND (`aux_sex_age_region`.`age` < 97)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 95) + AND (`aux_sex_age_region`.`age` < 97)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))) + UNION SELECT + `a`.`first_level_geo` AS `region`, + '97+' AS `range`, + `a`.`male` AS `male`, + `b`.`female` AS `female` + FROM + (((SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, COUNT(0) AS `male` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_MALE) + AND (`aux_sex_age_region`.`age` >= 97)) + GROUP BY `aux_sex_age_region`.`first_level_geo`)) `a` + JOIN (SELECT + `aux_sex_age_region`.`first_level_geo` AS `first_level_geo`, + COUNT(0) AS `female` + FROM + @SCHEMA.`aux_sex_age_region` + WHERE + ((`aux_sex_age_region`.`sex` = @INDIVIDUAL_VALUE_SEX_FEMALE) + AND (`aux_sex_age_region`.`age` >= 97)) + GROUP BY `aux_sex_age_region`.`first_level_geo`) `b` ON ((`a`.`first_level_geo` = `b`.`first_level_geo`))); diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index 19afa9a..7374d3d 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -42,7 +42,8 @@ public class MonitorWriter { "r_individual_info", "r_religion", "r_sex_by_age", - "r_first_level_geography" + "r_first_level_geography", + "r_sex_by_age_group_region" }; private static int reportCount = 0; From b2e0ca2d75f2787a2c8c6ff010be7ee74f6ce033 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Thu, 12 Jul 2018 21:23:54 +0200 Subject: [PATCH 06/15] Allow territory tag on items that are not ids and/or are out of order --- src/cspro2sql/MonitorEngine.java | 28 ++- src/cspro2sql/SchemaEngine.java | 11 +- src/cspro2sql/bean/Territory.java | 88 ++++++-- src/cspro2sql/sql/TemplateManager.java | 8 +- src/cspro2sql/writer/MonitorWriter.java | 56 +++-- src/cspro2sql/writer/SchemaWriter.java | 288 ++++++++++++------------ 6 files changed, 281 insertions(+), 198 deletions(-) diff --git a/src/cspro2sql/MonitorEngine.java b/src/cspro2sql/MonitorEngine.java index 6d918d0..4641108 100644 --- a/src/cspro2sql/MonitorEngine.java +++ b/src/cspro2sql/MonitorEngine.java @@ -57,19 +57,25 @@ public static void main(String[] args) throws Exception { } static boolean execute(List dictionaries, AreaNameFile areaNames, Properties prop, PrintStream out) { - TemplateManager tmFieldwork = null, tmListing = null, tmExpected = null; - for (Dictionary dictionary : dictionaries) { - if (dictionary.hasTag(Dictionary.TAG_FIELDWORK)) { - tmFieldwork = new TemplateManager(dictionary); - } else if (dictionary.hasTag(Dictionary.TAG_LISTING)) { - tmListing = new TemplateManager(dictionary); - } else if (dictionary.hasTag(Dictionary.TAG_EXPECTED)) { - tmExpected = new TemplateManager(dictionary); + + try { + TemplateManager tmFieldwork = null, tmListing = null, tmExpected = null; + for (Dictionary dictionary : dictionaries) { + if (dictionary.hasTag(Dictionary.TAG_FIELDWORK)) { + tmFieldwork = new TemplateManager(dictionary); + } else if (dictionary.hasTag(Dictionary.TAG_LISTING)) { + tmListing = new TemplateManager(dictionary); + } else if (dictionary.hasTag(Dictionary.TAG_EXPECTED)) { + tmExpected = new TemplateManager(dictionary); + } } + + boolean gisEnabled = prop.getProperty("gis.enabled") != null && (prop.getProperty("gis.enabled").equalsIgnoreCase("true") || prop.getProperty("gis.enabled").equals("1")); + return MonitorWriter.write(tmFieldwork, tmListing, tmExpected, areaNames, gisEnabled ,out); + } catch (IOException ex) { + System.err.println(ex.getMessage()); + return false; } - - boolean gisEnabled = prop.getProperty("gis.enabled") != null && (prop.getProperty("gis.enabled").equalsIgnoreCase("true") || prop.getProperty("gis.enabled").equals("1")); - return MonitorWriter.write(tmFieldwork, tmListing, tmExpected, areaNames, gisEnabled ,out); } } diff --git a/src/cspro2sql/SchemaEngine.java b/src/cspro2sql/SchemaEngine.java index ee06eb5..e31a09a 100644 --- a/src/cspro2sql/SchemaEngine.java +++ b/src/cspro2sql/SchemaEngine.java @@ -61,10 +61,15 @@ public static void main(String[] args) throws Exception { } static boolean execute(List dictionaries, boolean foreignKeys, PrintStream out) { - for (Dictionary dictionary : dictionaries) { - SchemaWriter.write(dictionary, foreignKeys, out); + try { + for (Dictionary dictionary : dictionaries) { + SchemaWriter.write(dictionary, foreignKeys, out); + } + return true; + } catch (IOException ex) { + System.err.println(ex.getMessage()); + return false; } - return true; } } diff --git a/src/cspro2sql/bean/Territory.java b/src/cspro2sql/bean/Territory.java index 37ff753..ab46902 100644 --- a/src/cspro2sql/bean/Territory.java +++ b/src/cspro2sql/bean/Territory.java @@ -1,6 +1,9 @@ package cspro2sql.bean; +import java.io.IOException; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -28,32 +31,63 @@ */ public class Territory { - private final Map items; private final List itemsList; public Territory() { - this.items = new LinkedHashMap<>(); this.itemsList = new LinkedList<>(); } + + public void addItems(Iterable dictItems) throws IOException { - public void addItem(Item item) { - String name = null; - TerritoryItem parent = null; - Tag tag = item.getTag(Dictionary.TAG_TERRITORY); - if (tag.getValue() != null) { - String[] tagValues = tag.getValue().split(","); - name = tagValues[0]; - if (tagValues.length > 1 && !tagValues[1].isEmpty()) { - parent = items.get(tagValues[1]); + ArrayList dictItemList = new ArrayList<>(); + final Map dictItemToParentName = new LinkedHashMap<>(); + final Map dictItemToName = new LinkedHashMap<>(); + for (Item dictItem : dictItems) { + dictItemList.add(dictItem); + Tag tag = dictItem.getTag(Dictionary.TAG_TERRITORY); + if (tag.getValue() != null) { + String[] tagValues = tag.getValue().split(","); + dictItemToName.put(dictItem, tagValues[0]); + if (tagValues.length > 1 && !tagValues[1].isEmpty()) { + dictItemToParentName.put(dictItem, tagValues[1]); + } } } - TerritoryItem territoryItem = new TerritoryItem(item, parent, name); - items.put(item.getName(), territoryItem); - itemsList.add(territoryItem); + + int i = 0; + while (i < dictItemList.size()) + { + Item dictItem = dictItemList.get(i); + String parentName = dictItemToParentName.getOrDefault(dictItem, null); + String prevName = i == 0 ? null : dictItemList.get(i-1).getName(); + if (parentName == null || parentName.compareTo(prevName) == 0) { + ++i; + } else { + dictItemList.remove(i); + int parentPos = 0; + for (; parentPos < dictItemList.size() && dictItemList.get(parentPos).getName().compareTo(parentName) != 0; ++parentPos) + { + } + if (parentPos == dictItemList.size()) { + throw new IOException("Territory " + dictItem.getName() + " has parent " + parentName + " that is NOT in found in territories"); + } + dictItemList.add(parentPos + 1, dictItem); + } + } + + Map nameToTerritoryItem = new LinkedHashMap<>(); + for (Item dictItem : dictItemList) + { + String parentName = dictItemToParentName.getOrDefault(dictItem, null); + TerritoryItem parent = parentName != null ? nameToTerritoryItem.get(parentName) : null; + TerritoryItem territoryItem = new TerritoryItem(dictItem, parent, dictItemToName.get(dictItem)); + nameToTerritoryItem.put(dictItem.getName(), territoryItem); + itemsList.add(territoryItem); + } } public boolean isEmpty() { - return this.items.isEmpty(); + return this.itemsList.isEmpty(); } public TerritoryItem getFirst() { @@ -68,4 +102,28 @@ public int size() { return this.itemsList.size(); } + public Iterable getItemRecords() + { + LinkedHashSet itemRecords = new LinkedHashSet<>(); + for (int i = 0; i < itemsList.size(); i++) { + itemRecords.add(itemsList.get(i).getItem().getRecord()); + } + return itemRecords; + } + + public String getFromClause() + { + if (itemsList.isEmpty()) + return null; + + Record mainRecord = itemsList.get(0).getItem().getRecord().getMainRecord(); + String from = mainRecord.getTableName(); + + for (Record record : getItemRecords()) { + if (!record.isMainRecord()) + from += " JOIN " + record.getTableName() + " ON " + mainRecord.getTableName() + ".ID = " + record.getTableName() + "." + mainRecord.getName(); + } + + return from; + } } diff --git a/src/cspro2sql/sql/TemplateManager.java b/src/cspro2sql/sql/TemplateManager.java index 02efdeb..4ade2d6 100644 --- a/src/cspro2sql/sql/TemplateManager.java +++ b/src/cspro2sql/sql/TemplateManager.java @@ -1,7 +1,6 @@ package cspro2sql.sql; import cspro2sql.bean.Dictionary; -import cspro2sql.bean.Item; import cspro2sql.bean.Record; import cspro2sql.bean.Tag; import cspro2sql.bean.Territory; @@ -43,7 +42,7 @@ public class TemplateManager { private final int[] ageRange; private Territory territory; - public TemplateManager(Dictionary dictionary) { + public TemplateManager(Dictionary dictionary) throws IOException { this.dictionary = dictionary; Record mainRecord = dictionary.getMainRecord(); String schema = dictionary.getSchema(); @@ -56,10 +55,7 @@ public TemplateManager(Dictionary dictionary) { this.territory = new Territory(); if (dictionary.hasTagged(Dictionary.TAG_TERRITORY)) { - Iterable territories = dictionary.getTaggedItems(Dictionary.TAG_TERRITORY); - for (Item territory : territories) { - this.territory.addItem(territory); - } + this.territory.addItems(dictionary.getTaggedItems(Dictionary.TAG_TERRITORY)); // Assume that geographic ids are in order in dictionary so region is // the first one. Depending on country may not be a region but it should diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index 7374d3d..dc73787 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -100,17 +100,19 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ } if (!territory.isEmpty()) { - TerritoryItem territoryItem = territory.getFirst(); + TerritoryItem firstTerritoryItem = territory.getFirst(); out.println("CREATE OR REPLACE VIEW " + schema + ".`r_regional_area` AS"); - String groupBy = territoryItem.getItemName(); - String name = territoryItem.getName(); - out.print(" SELECT '" + name + "' name, COUNT(0) value FROM (SELECT COUNT(0) FROM " + schema + "." + tm.getParam("@QUESTIONNAIRE_TABLE") + " GROUP BY " + groupBy + ") a0"); + String groupBy = firstTerritoryItem.getItemName(); + String name = firstTerritoryItem.getName(); + String fromClause = territory.getFromClause(); + + out.print(" SELECT '" + name + "' name, COUNT(0) value FROM (SELECT COUNT(0) FROM " + fromClause + " GROUP BY " + groupBy + ") a0"); for (int i = 1; i < territory.size(); i++) { - territoryItem = territory.get(i); + TerritoryItem territoryItem = territory.get(i); name = territoryItem.getName(); groupBy += "," + territoryItem.getItemName(); out.println(" UNION"); - out.print(" SELECT '" + name + "', COUNT(0) FROM (SELECT COUNT(0) FROM " + schema + "." + tm.getParam("@QUESTIONNAIRE_TABLE") + " GROUP BY " + groupBy + ") a" + i); + out.print(" SELECT '" + name + "', COUNT(0) FROM (SELECT COUNT(0) FROM " + fromClause + " GROUP BY " + groupBy + ") a" + i); } out.println(); out.println(";"); @@ -158,7 +160,12 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ out.print(",'#'," + territoryItem.selectDescription()); } out.println(") as name, COUNT(0) AS `household`"); - out.println(" FROM " + schema + "." + tm.getParam("@QUESTIONNAIRE_TABLE") + " `h`"); + out.print(" FROM (SELECT " + territory.getFirst().getItem().getColunmFullName()); + for (int i = 1; i < territory.size(); i++) { + out.print(", " + territory.get(i).getItem().getColunmFullName()); + } + + out.println(" FROM " + territory.getFromClause() + ") `h`"); out.print(" GROUP BY "); territoryItem = territory.getFirst(); out.print("`h`." + territoryItem.getItemName()); @@ -171,19 +178,24 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ } if (!territory.isEmpty()) { - printAuxTable(tm, tm, "aux_household_returned", "returned", out); - printAuxTable(tm, (tmListing != null) ? tmListing : tm, "aux_listing_returned", "returned", out); - printAuxTable(tm, (tmExpected != null) ? tmExpected : tm, "aux_household_expected", "expected", out); + try { + printAuxTable(tm, tm, "aux_household_returned", "returned", out); + printAuxTable(tm, (tmListing != null) ? tmListing : tm, "aux_listing_returned", "returned", out); + printAuxTable(tm, (tmExpected != null) ? tmExpected : tm, "aux_household_expected", "expected", out); - int upTo = 1; - for (int i = 0; i < territory.size(); i++) { - TerritoryItem territoryItem = territory.get(i); - printExpectedReport(tm, "r_household_expected_by_" + territoryItem.getName().toLowerCase(), upTo++, out); - printMaterialized(schema, "r_household_expected_by_" + territoryItem.getName().toLowerCase(), out); - } + int upTo = 1; + for (int i = 0; i < territory.size(); i++) { + TerritoryItem territoryItem = territory.get(i); + printExpectedReport(tm, "r_household_expected_by_" + territoryItem.getName().toLowerCase(), upTo++, out); + printMaterialized(schema, "r_household_expected_by_" + territoryItem.getName().toLowerCase(), out); + } - printTotalReport(tm, (tmListing != null) ? tmListing : tm, out); - printMaterialized(schema, "r_total", out); + printTotalReport(tm, (tmListing != null) ? tmListing : tm, out); + printMaterialized(schema, "r_total", out); + } catch (IOException ex) { + System.err.println(ex.getMessage()); + return false; + } } return true; @@ -271,10 +283,16 @@ private static void printMaterialized(String schema, String name, PrintStream ou out.println(); } - private static void printAuxTable(TemplateManager mainTm, TemplateManager tm, String auxName, String columnName, PrintStream out) { + private static void printAuxTable(TemplateManager mainTm, TemplateManager tm, String auxName, String columnName, PrintStream out) throws IOException { Set records = new LinkedHashSet<>(); Territory territory = tm.getTerritory(); Territory mainTerritory = mainTm.getTerritory(); + if (territory.size() != mainTerritory.size()) { + throw new IOException("Number of territories in " + tm.getDictionary().getName() + + " is different from number of territories in " + + mainTm.getDictionary().getName() + + ". Fix #territory tags in dictionaries so that they match"); + } out.println("CREATE OR REPLACE VIEW " + tm.getDictionary().getSchema() + "." + auxName + " AS"); out.println(" SELECT "); for (int i = 0; i < territory.size(); i++) { diff --git a/src/cspro2sql/writer/SchemaWriter.java b/src/cspro2sql/writer/SchemaWriter.java index 65349b6..642b3e9 100644 --- a/src/cspro2sql/writer/SchemaWriter.java +++ b/src/cspro2sql/writer/SchemaWriter.java @@ -1,144 +1,144 @@ -package cspro2sql.writer; - -import cspro2sql.bean.Dictionary; -import cspro2sql.bean.Item; -import cspro2sql.bean.Record; -import cspro2sql.bean.ValueSet; -import cspro2sql.bean.ValueSetValue; -import cspro2sql.sql.TemplateManager; -import java.io.IOException; -import java.io.PrintStream; -import java.util.HashSet; -import java.util.Set; - -/** - * Copyright 2017 ISTAT - * - * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by - * the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. You may - * obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl5 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * Licence for the specific language governing permissions and limitations under - * the Licence. - * - * @author Guido Drovandi - * @author Mauro Bruno - * @version 0.9.17 - */ -public class SchemaWriter { - - public static void write(Dictionary dictionary, boolean foreignKeys, PrintStream ps) { - TemplateManager tm = new TemplateManager(dictionary); - String schema = dictionary.getSchema(); - - ps.println("CREATE SCHEMA IF NOT EXISTS " + schema + " CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci;"); - ps.println(); - ps.println("USE " + schema + ";"); - ps.println(); - - try { - tm.printTemplate("cspro2sql_dictionary", ps); - tm.printTemplate("cspro2sql_error", ps); - } catch (IOException ex) { - return; - } - - for (Record record : dictionary.getRecords()) { - for (Item item : record.getItems()) { - printValueSet(schema, item, ps); - } - } - - for (Record record : dictionary.getRecords()) { - ps.println("CREATE TABLE " + record.getFullTableName() + " ("); - ps.println(" ID INT(9) UNSIGNED AUTO_INCREMENT,"); - if (!record.isMainRecord()) { - ps.println(" " + record.getMainRecord().getName() + " INT(9) UNSIGNED NOT NULL,"); - ps.println(" COUNTER INT(9) UNSIGNED NOT NULL,"); - } - for (Item item : record.getItems()) { - printItem(schema, foreignKeys, item, ps); - } - if (!record.isMainRecord()) { - ps.println(" INDEX (" + record.getMainRecord().getName() + "),"); - ps.println(" FOREIGN KEY (" + record.getMainRecord().getName() + ") REFERENCES " + record.getMainRecord().getFullTableName() + "(id),"); - } - ps.println(" PRIMARY KEY (ID)"); - ps.println(") ENGINE=INNODB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); - ps.println(); - } - } - - private static void printItem(String schema, boolean foreignKeys, Item item, PrintStream ps) { - String name = item.getName(); - int length = item.getLength(); - switch (item.getDataType()) { - case Dictionary.ITEM_ALPHA: - ps.println(" " + name + " CHAR(" + length + "),"); - break; - case Dictionary.ITEM_DECIMAL: - if (item.getDecimal() > 0) { - ps.println(" " + name + " DECIMAL(" + length + ", " + item.getDecimal() + "),"); - } else { - ps.println(" " + name + " DECIMAL(" + length + ", 0),"); - } - break; - default: - } - if (foreignKeys && item.hasValueSets()) { - ps.println(" FOREIGN KEY (" + name + ") REFERENCES " + schema + "." + item.getValueSetName() + "(ID),"); - } - for (Item subItem : item.getSubItems()) { - printItem(schema, foreignKeys, subItem, ps); - } - } - - private static void printValueSet(String schema, Item item, PrintStream ps) { - if (item.hasValueSets() && item.getValueSets().get(0).isNotCreated()) { - ps.println("CREATE TABLE " + schema + "." + item.getValueSetName() + " ("); - switch (item.getDataType()) { - case Dictionary.ITEM_ALPHA: - ps.println(" ID CHAR(" + item.getLength() + "),"); - break; - case Dictionary.ITEM_DECIMAL: - ps.println(" ID INT(" + item.getLength() + "),"); - break; - default: - } - ps.println(" VALUE TEXT,"); - ps.println(" PRIMARY KEY (ID)"); - ps.println(") ENGINE=INNODB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); - ps.println(); - boolean first = true; - Set keys = new HashSet<>(); - ps.println("INSERT INTO " + schema + "." + item.getValueSetName() + "(ID,VALUE) VALUES "); - for (int i = 0; i < item.getValueSets().size(); i++) { - ValueSet valueSet = item.getValueSets().get(i); - valueSet.setCreated(); - for (ValueSetValue e : valueSet.getValues().values()) { - if (keys.contains(e.getKey())) { - continue; - } - if (!first) { - ps.println(","); - } - ps.print(" (\"" + e.getKey() + "\",\"" + e.getValue().replace("\"", "\\\"") + "\")"); - keys.add(e.getKey()); - first = false; - } - } - ps.println(";"); - ps.println(); - } - for (Item subItem : item.getSubItems()) { - printValueSet(schema, subItem, ps); - } - } - -} +package cspro2sql.writer; + +import cspro2sql.bean.Dictionary; +import cspro2sql.bean.Item; +import cspro2sql.bean.Record; +import cspro2sql.bean.ValueSet; +import cspro2sql.bean.ValueSetValue; +import cspro2sql.sql.TemplateManager; +import java.io.IOException; +import java.io.PrintStream; +import java.util.HashSet; +import java.util.Set; + +/** + * Copyright 2017 ISTAT + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by + * the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. You may + * obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl5 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * Licence for the specific language governing permissions and limitations under + * the Licence. + * + * @author Guido Drovandi + * @author Mauro Bruno + * @version 0.9.17 + */ +public class SchemaWriter { + + public static void write(Dictionary dictionary, boolean foreignKeys, PrintStream ps) throws IOException { + TemplateManager tm = new TemplateManager(dictionary); + String schema = dictionary.getSchema(); + + ps.println("CREATE SCHEMA IF NOT EXISTS " + schema + " CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci;"); + ps.println(); + ps.println("USE " + schema + ";"); + ps.println(); + + try { + tm.printTemplate("cspro2sql_dictionary", ps); + tm.printTemplate("cspro2sql_error", ps); + } catch (IOException ex) { + return; + } + + for (Record record : dictionary.getRecords()) { + for (Item item : record.getItems()) { + printValueSet(schema, item, ps); + } + } + + for (Record record : dictionary.getRecords()) { + ps.println("CREATE TABLE " + record.getFullTableName() + " ("); + ps.println(" ID INT(9) UNSIGNED AUTO_INCREMENT,"); + if (!record.isMainRecord()) { + ps.println(" " + record.getMainRecord().getName() + " INT(9) UNSIGNED NOT NULL,"); + ps.println(" COUNTER INT(9) UNSIGNED NOT NULL,"); + } + for (Item item : record.getItems()) { + printItem(schema, foreignKeys, item, ps); + } + if (!record.isMainRecord()) { + ps.println(" INDEX (" + record.getMainRecord().getName() + "),"); + ps.println(" FOREIGN KEY (" + record.getMainRecord().getName() + ") REFERENCES " + record.getMainRecord().getFullTableName() + "(id),"); + } + ps.println(" PRIMARY KEY (ID)"); + ps.println(") ENGINE=INNODB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); + ps.println(); + } + } + + private static void printItem(String schema, boolean foreignKeys, Item item, PrintStream ps) { + String name = item.getName(); + int length = item.getLength(); + switch (item.getDataType()) { + case Dictionary.ITEM_ALPHA: + ps.println(" " + name + " CHAR(" + length + "),"); + break; + case Dictionary.ITEM_DECIMAL: + if (item.getDecimal() > 0) { + ps.println(" " + name + " DECIMAL(" + length + ", " + item.getDecimal() + "),"); + } else { + ps.println(" " + name + " DECIMAL(" + length + ", 0),"); + } + break; + default: + } + if (foreignKeys && item.hasValueSets()) { + ps.println(" FOREIGN KEY (" + name + ") REFERENCES " + schema + "." + item.getValueSetName() + "(ID),"); + } + for (Item subItem : item.getSubItems()) { + printItem(schema, foreignKeys, subItem, ps); + } + } + + private static void printValueSet(String schema, Item item, PrintStream ps) { + if (item.hasValueSets() && item.getValueSets().get(0).isNotCreated()) { + ps.println("CREATE TABLE " + schema + "." + item.getValueSetName() + " ("); + switch (item.getDataType()) { + case Dictionary.ITEM_ALPHA: + ps.println(" ID CHAR(" + item.getLength() + "),"); + break; + case Dictionary.ITEM_DECIMAL: + ps.println(" ID INT(" + item.getLength() + "),"); + break; + default: + } + ps.println(" VALUE TEXT,"); + ps.println(" PRIMARY KEY (ID)"); + ps.println(") ENGINE=INNODB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;"); + ps.println(); + boolean first = true; + Set keys = new HashSet<>(); + ps.println("INSERT INTO " + schema + "." + item.getValueSetName() + "(ID,VALUE) VALUES "); + for (int i = 0; i < item.getValueSets().size(); i++) { + ValueSet valueSet = item.getValueSets().get(i); + valueSet.setCreated(); + for (ValueSetValue e : valueSet.getValues().values()) { + if (keys.contains(e.getKey())) { + continue; + } + if (!first) { + ps.println(","); + } + ps.print(" (\"" + e.getKey() + "\",\"" + e.getValue().replace("\"", "\\\"") + "\")"); + keys.add(e.getKey()); + first = false; + } + } + ps.println(";"); + ps.println(); + } + for (Item subItem : item.getSubItems()) { + printValueSet(schema, subItem, ps); + } + } + +} From 9be1480d3f840e08086e2a1e05da3b401cd376b5 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Wed, 18 Jul 2018 13:57:43 +0200 Subject: [PATCH 07/15] Add EA status dict, handle geocode with partials Add another dictionary that contains EA completion status (complete/incomplete). Handle expected dictionary that has records for higher level entities where the lower codes are null e.g. just region, district with EA null to represent name of district as this is a common format in CSPro. --- src/cspro2sql/MonitorEngine.java | 7 +- src/cspro2sql/bean/Dictionary.java | 631 +++++++++--------- src/cspro2sql/sql/TemplateManager.java | 3 + src/cspro2sql/sql/template/dashboard_info.sql | 3 +- src/cspro2sql/writer/MonitorWriter.java | 142 +++- 5 files changed, 463 insertions(+), 323 deletions(-) diff --git a/src/cspro2sql/MonitorEngine.java b/src/cspro2sql/MonitorEngine.java index 4641108..2700a40 100644 --- a/src/cspro2sql/MonitorEngine.java +++ b/src/cspro2sql/MonitorEngine.java @@ -59,7 +59,8 @@ public static void main(String[] args) throws Exception { static boolean execute(List dictionaries, AreaNameFile areaNames, Properties prop, PrintStream out) { try { - TemplateManager tmFieldwork = null, tmListing = null, tmExpected = null; + TemplateManager tmFieldwork = null, tmListing = null, + tmExpected = null, tmEaStatus = null; for (Dictionary dictionary : dictionaries) { if (dictionary.hasTag(Dictionary.TAG_FIELDWORK)) { tmFieldwork = new TemplateManager(dictionary); @@ -67,11 +68,13 @@ static boolean execute(List dictionaries, AreaNameFile areaNames, Pr tmListing = new TemplateManager(dictionary); } else if (dictionary.hasTag(Dictionary.TAG_EXPECTED)) { tmExpected = new TemplateManager(dictionary); + } else if (dictionary.hasTag(Dictionary.TAG_EA_STATUS)) { + tmEaStatus = new TemplateManager(dictionary); } } boolean gisEnabled = prop.getProperty("gis.enabled") != null && (prop.getProperty("gis.enabled").equalsIgnoreCase("true") || prop.getProperty("gis.enabled").equals("1")); - return MonitorWriter.write(tmFieldwork, tmListing, tmExpected, areaNames, gisEnabled ,out); + return MonitorWriter.write(tmFieldwork, tmListing, tmExpected, tmEaStatus, areaNames, gisEnabled ,out); } catch (IOException ex) { System.err.println(ex.getMessage()); return false; diff --git a/src/cspro2sql/bean/Dictionary.java b/src/cspro2sql/bean/Dictionary.java index b717246..2184964 100644 --- a/src/cspro2sql/bean/Dictionary.java +++ b/src/cspro2sql/bean/Dictionary.java @@ -1,314 +1,317 @@ -package cspro2sql.bean; - -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -/** - * Copyright 2017 ISTAT - * - * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by - * the European Commission - subsequent versions of the EUPL (the "Licence"); - * You may not use this work except in compliance with the Licence. You may - * obtain a copy of the Licence at: - * - * http://ec.europa.eu/idabc/eupl5 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * Licence for the specific language governing permissions and limitations under - * the Licence. - * - * @author Guido Drovandi - * @author Mauro Bruno - * @author Paolo Giacomi - * @version 0.9.15 - */ -public final class Dictionary extends Taggable { - - public static final String DICT_HEADER = "[Dictionary]"; - public static final String DICT_LANGUAGES = "[Languages]"; - public static final String DICT_LEVEL = "[Level]"; - public static final String DICT_RECORD = "[Record]"; - public static final String DICT_IDITEMS = "[IdItems]"; - public static final String DICT_ITEM = "[Item]"; - public static final String DICT_VALUESET = "[ValueSet]"; - public static final String DICT_RELATION = "[Relation]"; - - public static final String DICT_LABEL = "Label"; - public static final char DICT_LABEL_LANGUAGE_SEPARATOR = '|'; - public static final String DICT_NAME = "Name"; - public static final String DICT_NOTE = "Note"; - public static final String DICT_NOTENEWLINE = "\r\n"; - public static final String DICT_NEWLINE_REGEXP = "(? records = new LinkedList<>(); - private final Map recordsByName = new LinkedHashMap<>(); - private final Map valueSets = new HashMap<>(); - private final Map> tags = new HashMap<>(); - - private String name; - private Record lastRecord; - private List lastItems; - private List lastItemsNotSubItem; - - public Dictionary(String schema, String prefix) { - this.schema = schema; - this.prefix = prefix; - } - - public String getSchema() { - return schema; - } - - public String getPrefix() { - return prefix; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public void addRecord(Record record) { - if (record.hasTag(TAG_IGNORE)) { - return; - } - if (this.records.isEmpty()) { - // idItems record - record.setIsMainRecord(true); - } else { - record.setMainRecord(getMainRecord()); - } - this.lastRecord = record; - this.records.add(this.lastRecord); - this.recordsByName.put(record.getRecordTypeValue(), record); - } - - public boolean addItem(Item item) { - if (item.hasTag(TAG_IGNORE)) { - return false; - } - if (item.getOccurrences() > 1) { - List its = new LinkedList<>(); - for (int i = 0; i < item.getOccurrences(); i++) { - Item it = item.clone(); - it.setOccurenceNumber(i); - it.setName(it.getName() + "_" + i); - it.setValueSetName(item.getName()); - it.setStart(it.getStart() + i * it.getLength()); - its.add(it); - } - if (item.isSubItem()) { - addSubItems(its); - } else { - addLastItemsNotSubItem(its); - this.lastRecord.addItems(its); - } - addLastItems(its); - } else { - if (item.isSubItem()) { - addSubItem(item); - } else { - addLastItemNotSubItem(item); - this.lastRecord.addItem(item); - } - addLastItem(item); - } - return true; - } - - public void addValueSet(ValueSet valueSet) { - if (valueSet != null) { - if (valueSet.getLink() != null && !valueSet.getLink().isEmpty() - && valueSets.containsKey(valueSet.getLink())) { - valueSet = valueSets.get(valueSet.getLink()).clone(); - } else { - if (valueSet.isEmpty()) { - return; - } - if (valueSet.getLink() != null && !valueSet.getLink().isEmpty()) { - this.valueSets.put(valueSet.getLink(), valueSet); - } - } - addValueSetToLastItems(valueSet); - } - } - - private void addLastItem(Item item) { - this.lastItems = new LinkedList<>(); - this.lastItems.add(item); - } - - private void addLastItems(List items) { - this.lastItems = items; - } - - private void addValueSetToLastItems(ValueSet valueSet) { - for (Item item : this.lastItems) { - if (ITEM_ALPHA.equals(item.getDataType()) && item.hasTag(TAG_MULTIPLE_RESPONSE)) { - List splits = item.splitIntoColumns(valueSet); - this.lastRecord.replaceItemWithSplit(item, splits); - } else { - if (!ITEM_ALPHA.equals(item.getDataType())) { - valueSet.removeEmptyValues(); - } - item.addValueSet(valueSet); - } - } - } - - private void addLastItemNotSubItem(Item item) { - this.lastItemsNotSubItem = new LinkedList<>(); - this.lastItemsNotSubItem.add(item); - } - - private void addLastItemsNotSubItem(List items) { - this.lastItemsNotSubItem = items; - } - - private void addSubItem(Item subItem) { - for (Item item : this.lastItemsNotSubItem) { - item.addSubItem(subItem); - } - } - - private void addSubItems(List subItems) { - for (Item item : this.lastItemsNotSubItem) { - for (Item subItem : subItems) { - item.addSubItem(subItem); - } - } - } - - public Record getMainRecord() { - return this.records.get(0); - } - - public List getRecords() { - return records; - } - - public Record getRecord(char name) { - return recordsByName.get("" + name); - } - - public Record getRecord(String name) { - return recordsByName.get(name); - } - - public void addTagged(Tag tag, Taggable o) { - if (!tags.containsKey(tag)) { - tags.put(tag, new LinkedList()); - } - tags.get(tag).add(o); - } - - public boolean hasTagged(Tag tag) { - return tags.containsKey(tag); - } - - public Record getTaggedRecord(Tag tag) { - return (Record) tags.get(tag).get(0); - } - - public Item getTaggedItem(Tag tag) { - if (tags.containsKey(tag)) { - return (Item) tags.get(tag).get(0); - } - return null; - } - - public Iterable getTaggedItems(Tag tag) { - return (Iterable) (Object) tags.get(tag); - } - - public ValueSet getTaggedValueSet(Tag tag) { - return (ValueSet) tags.get(tag).get(0); - } - - public ValueSetValue getTaggedValueSetValue(Tag tag) { - return (ValueSetValue) tags.get(tag).get(0); - } - - public Tag getTag(Tag tag) { - for (Tag t : tags.keySet()) { - if (t.equals(tag)) { - return t; - } - } - return null; - } - -} +package cspro2sql.bean; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Copyright 2017 ISTAT + * + * Licensed under the EUPL, Version 1.1 or – as soon they will be approved by + * the European Commission - subsequent versions of the EUPL (the "Licence"); + * You may not use this work except in compliance with the Licence. You may + * obtain a copy of the Licence at: + * + * http://ec.europa.eu/idabc/eupl5 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the Licence is distributed on an "AS IS" basis, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * Licence for the specific language governing permissions and limitations under + * the Licence. + * + * @author Guido Drovandi + * @author Mauro Bruno + * @author Paolo Giacomi + * @version 0.9.15 + */ +public final class Dictionary extends Taggable { + + public static final String DICT_HEADER = "[Dictionary]"; + public static final String DICT_LANGUAGES = "[Languages]"; + public static final String DICT_LEVEL = "[Level]"; + public static final String DICT_RECORD = "[Record]"; + public static final String DICT_IDITEMS = "[IdItems]"; + public static final String DICT_ITEM = "[Item]"; + public static final String DICT_VALUESET = "[ValueSet]"; + public static final String DICT_RELATION = "[Relation]"; + + public static final String DICT_LABEL = "Label"; + public static final char DICT_LABEL_LANGUAGE_SEPARATOR = '|'; + public static final String DICT_NAME = "Name"; + public static final String DICT_NOTE = "Note"; + public static final String DICT_NOTENEWLINE = "\r\n"; + public static final String DICT_NEWLINE_REGEXP = "(? records = new LinkedList<>(); + private final Map recordsByName = new LinkedHashMap<>(); + private final Map valueSets = new HashMap<>(); + private final Map> tags = new HashMap<>(); + + private String name; + private Record lastRecord; + private List lastItems; + private List lastItemsNotSubItem; + + public Dictionary(String schema, String prefix) { + this.schema = schema; + this.prefix = prefix; + } + + public String getSchema() { + return schema; + } + + public String getPrefix() { + return prefix; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public void addRecord(Record record) { + if (record.hasTag(TAG_IGNORE)) { + return; + } + if (this.records.isEmpty()) { + // idItems record + record.setIsMainRecord(true); + } else { + record.setMainRecord(getMainRecord()); + } + this.lastRecord = record; + this.records.add(this.lastRecord); + this.recordsByName.put(record.getRecordTypeValue(), record); + } + + public boolean addItem(Item item) { + if (item.hasTag(TAG_IGNORE)) { + return false; + } + if (item.getOccurrences() > 1) { + List its = new LinkedList<>(); + for (int i = 0; i < item.getOccurrences(); i++) { + Item it = item.clone(); + it.setOccurenceNumber(i); + it.setName(it.getName() + "_" + i); + it.setValueSetName(item.getName()); + it.setStart(it.getStart() + i * it.getLength()); + its.add(it); + } + if (item.isSubItem()) { + addSubItems(its); + } else { + addLastItemsNotSubItem(its); + this.lastRecord.addItems(its); + } + addLastItems(its); + } else { + if (item.isSubItem()) { + addSubItem(item); + } else { + addLastItemNotSubItem(item); + this.lastRecord.addItem(item); + } + addLastItem(item); + } + return true; + } + + public void addValueSet(ValueSet valueSet) { + if (valueSet != null) { + if (valueSet.getLink() != null && !valueSet.getLink().isEmpty() + && valueSets.containsKey(valueSet.getLink())) { + valueSet = valueSets.get(valueSet.getLink()).clone(); + } else { + if (valueSet.isEmpty()) { + return; + } + if (valueSet.getLink() != null && !valueSet.getLink().isEmpty()) { + this.valueSets.put(valueSet.getLink(), valueSet); + } + } + addValueSetToLastItems(valueSet); + } + } + + private void addLastItem(Item item) { + this.lastItems = new LinkedList<>(); + this.lastItems.add(item); + } + + private void addLastItems(List items) { + this.lastItems = items; + } + + private void addValueSetToLastItems(ValueSet valueSet) { + for (Item item : this.lastItems) { + if (ITEM_ALPHA.equals(item.getDataType()) && item.hasTag(TAG_MULTIPLE_RESPONSE)) { + List splits = item.splitIntoColumns(valueSet); + this.lastRecord.replaceItemWithSplit(item, splits); + } else { + if (!ITEM_ALPHA.equals(item.getDataType())) { + valueSet.removeEmptyValues(); + } + item.addValueSet(valueSet); + } + } + } + + private void addLastItemNotSubItem(Item item) { + this.lastItemsNotSubItem = new LinkedList<>(); + this.lastItemsNotSubItem.add(item); + } + + private void addLastItemsNotSubItem(List items) { + this.lastItemsNotSubItem = items; + } + + private void addSubItem(Item subItem) { + for (Item item : this.lastItemsNotSubItem) { + item.addSubItem(subItem); + } + } + + private void addSubItems(List subItems) { + for (Item item : this.lastItemsNotSubItem) { + for (Item subItem : subItems) { + item.addSubItem(subItem); + } + } + } + + public Record getMainRecord() { + return this.records.get(0); + } + + public List getRecords() { + return records; + } + + public Record getRecord(char name) { + return recordsByName.get("" + name); + } + + public Record getRecord(String name) { + return recordsByName.get(name); + } + + public void addTagged(Tag tag, Taggable o) { + if (!tags.containsKey(tag)) { + tags.put(tag, new LinkedList()); + } + tags.get(tag).add(o); + } + + public boolean hasTagged(Tag tag) { + return tags.containsKey(tag); + } + + public Record getTaggedRecord(Tag tag) { + return (Record) tags.get(tag).get(0); + } + + public Item getTaggedItem(Tag tag) { + if (tags.containsKey(tag)) { + return (Item) tags.get(tag).get(0); + } + return null; + } + + public Iterable getTaggedItems(Tag tag) { + return (Iterable) (Object) tags.get(tag); + } + + public ValueSet getTaggedValueSet(Tag tag) { + return (ValueSet) tags.get(tag).get(0); + } + + public ValueSetValue getTaggedValueSetValue(Tag tag) { + return (ValueSetValue) tags.get(tag).get(0); + } + + public Tag getTag(Tag tag) { + for (Tag t : tags.keySet()) { + if (t.equals(tag)) { + return t; + } + } + return null; + } + +} diff --git a/src/cspro2sql/sql/TemplateManager.java b/src/cspro2sql/sql/TemplateManager.java index 4ade2d6..72f0762 100644 --- a/src/cspro2sql/sql/TemplateManager.java +++ b/src/cspro2sql/sql/TemplateManager.java @@ -100,6 +100,9 @@ public TemplateManager(Dictionary dictionary) throws IOException { if (dictionary.hasTagged(Dictionary.TAG_FEMALE)) { this.params.put("@INDIVIDUAL_VALUE_SEX_FEMALE", dictionary.getTaggedValueSetValue(Dictionary.TAG_FEMALE).getKey()); } + if (dictionary.hasTagged(Dictionary.TAG_EA_COMPLETED)) { + this.params.put("@EA_STATUS_VALUE_COMPLETE", dictionary.getTaggedValueSetValue(Dictionary.TAG_EA_COMPLETED).getKey()); + } } public Dictionary getDictionary() { diff --git a/src/cspro2sql/sql/template/dashboard_info.sql b/src/cspro2sql/sql/template/dashboard_info.sql index fb68731..48c8495 100644 --- a/src/cspro2sql/sql/template/dashboard_info.sql +++ b/src/cspro2sql/sql/template/dashboard_info.sql @@ -2,8 +2,9 @@ CREATE TABLE IF NOT EXISTS @SCHEMA.DASHBOARD_INFO ( `ID` int(1) NOT NULL DEFAULT 0, `LISTING` int(1) NOT NULL DEFAULT 0, `EXPECTED` int(1) NOT NULL DEFAULT 0, + `EA_STATUS` int(1) NOT NULL DEFAULT 0, `GIS` int(1) NOT NULL DEFAULT 0, PRIMARY KEY (`ID`) ) ENGINE=INNODB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; -INSERT INTO @SCHEMA.DASHBOARD_INFO values (0, @LISTING, @EXPECTED, @GIS); +INSERT INTO @SCHEMA.DASHBOARD_INFO values (0, @LISTING, @EXPECTED, @EA_STATUS, @GIS); diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index dc73787..5d89eb2 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -48,11 +48,15 @@ public class MonitorWriter { private static int reportCount = 0; - public static boolean write(TemplateManager tm, TemplateManager tmListing, TemplateManager tmExpected, AreaNameFile areaNames, boolean gisEnabled, PrintStream out) { + public static boolean write(TemplateManager tm, TemplateManager tmListing, + TemplateManager tmExpected, TemplateManager tmEaStatus, + AreaNameFile areaNames, boolean gisEnabled, PrintStream out) { + String schema = tm.getDictionary().getSchema(); tm.addParam("@LISTING", tmListing == null ? "0" : "1"); tm.addParam("@EXPECTED", tmExpected == null ? "0" : "1"); + tm.addParam("@EA_STATUS", tmEaStatus == null ? "0" : "1"); tm.addParam("@GIS", gisEnabled ? "1" : "0"); out.println("USE " + schema + ";"); @@ -183,11 +187,24 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, Templ printAuxTable(tm, (tmListing != null) ? tmListing : tm, "aux_listing_returned", "returned", out); printAuxTable(tm, (tmExpected != null) ? tmExpected : tm, "aux_household_expected", "expected", out); - int upTo = 1; + boolean createEaTables = tmEaStatus != null && tmExpected != null && tmEaStatus.hasParam("@EA_STATUS_VALUE_COMPLETE"); + + if (createEaTables) { + printAuxEaStatusTable(tm, tmExpected, "aux_ea_expected", "expected", out); + printAuxEaStatusTable(tm, tmEaStatus, "aux_ea_completed", "completed", out); + } + for (int i = 0; i < territory.size(); i++) { + final int upTo = i + 1; + TerritoryItem territoryItem = territory.get(i); - printExpectedReport(tm, "r_household_expected_by_" + territoryItem.getName().toLowerCase(), upTo++, out); + printExpectedReport(tm, "r_household_expected_by_" + territoryItem.getName().toLowerCase(), upTo, out); printMaterialized(schema, "r_household_expected_by_" + territoryItem.getName().toLowerCase(), out); + + if (createEaTables && i < territory.size() - 1) { + printExpectedEaReport(tm, "r_ea_expected_by_" + territoryItem.getName().toLowerCase(), upTo, out); + printMaterialized(schema, "r_ea_expected_by_" + territoryItem.getName().toLowerCase(), out); + } } printTotalReport(tm, (tmListing != null) ? tmListing : tm, out); @@ -269,7 +286,7 @@ private static void printTerritoryTable(Territory territory, AreaNameFile areaNa fmt.format(formatSpecs[i], area.getCodes().get(i)); } - out.println("'" + area.getName() + "'," + area.getCodes().get(numLevels - 1) + ",'" + fullCodeBuilder.toString() + "');"); + out.println("'" + area.getName().replace("'", "''") + "'," + area.getCodes().get(numLevels - 1) + ",'" + fullCodeBuilder.toString() + "');"); } } } @@ -317,6 +334,12 @@ private static void printAuxTable(TemplateManager mainTm, TemplateManager tm, St out.println(" JOIN " + record.getFullTableName() + " ON " + record.getMainRecord().getTableName() + ".ID = " + record.getTableName() + "." + record.getMainRecord().getName()); } } + out.println(" WHERE"); + for (int i = 0; i < territory.size() - 1; i++) { + Item item = territory.get(i).getItem(); + out.println(" " + item.getColunmFullName() + " IS NOT NULL AND"); + } + out.println(" " + territory.get(territory.size() - 1).getItem().getColunmFullName() + " IS NOT NULL"); out.println(" GROUP BY"); for (int i = 0; i < territory.size() - 1; i++) { Item item = territory.get(i).getItem(); @@ -327,6 +350,59 @@ private static void printAuxTable(TemplateManager mainTm, TemplateManager tm, St out.println(); } + private static void printAuxEaStatusTable(TemplateManager mainTm, TemplateManager tm, String auxName, String columnName, PrintStream out) throws IOException { + Set records = new LinkedHashSet<>(); + Territory territory = tm.getTerritory(); + Territory mainTerritory = mainTm.getTerritory(); + if (territory.size() != mainTerritory.size()) { + throw new IOException("Number of territories in " + tm.getDictionary().getName() + + " is different from number of territories in " + + mainTm.getDictionary().getName() + + ". Fix #territory tags in dictionaries so that they match"); + } + out.println("CREATE OR REPLACE VIEW " + tm.getDictionary().getSchema() + "." + auxName + " AS"); + out.println(" SELECT "); + for (int i = 0; i < territory.size() - 1; i++) { + TerritoryItem territoryItem = territory.get(i); + TerritoryItem mainTerritoryItem = mainTerritory.get(i); + Item item = territoryItem.getItem(); + out.println(" " + item.getColunmFullName() + " AS " + mainTerritoryItem.getItemName() + ","); + records.add(item.getRecord()); + } + Item expected = tm.getDictionary().getTaggedItem(Dictionary.TAG_EXPECTED_QUESTIONNAIRES); + Item completed = tm.getDictionary().getTaggedItem(Dictionary.TAG_EA_COMPLETION_STATUS); + + if (expected != null) { + out.println(" COUNT(0) AS `" + columnName + "`"); + records.add(expected.getRecord()); + } else if (completed != null) { + out.println(" SUM(" + completed.getColunmFullName() + " = " + tm.getParam("@EA_STATUS_VALUE_COMPLETE") + ") AS `" + columnName + "`"); + records.add(completed.getRecord()); + } + Record[] recArray = records.toArray(new Record[0]); + out.println(" FROM"); + out.println(" " + recArray[0].getMainRecord().getFullTableName()); + for (Record record : recArray) { + if (!record.isMainRecord()) { + out.println(" JOIN " + record.getFullTableName() + " ON " + record.getMainRecord().getTableName() + ".ID = " + record.getTableName() + "." + record.getMainRecord().getName()); + } + } + out.println(" WHERE"); + for (int i = 0; i < territory.size() - 1; i++) { + Item item = territory.get(i).getItem(); + out.println(" " + item.getColunmFullName() + " IS NOT NULL AND"); + } + out.println(" " + territory.get(territory.size() - 1).getItem().getColunmFullName() + " IS NOT NULL"); + out.println(" GROUP BY"); + for (int i = 0; i < territory.size() - 2; i++) { + Item item = territory.get(i).getItem(); + out.println(" " + item.getColunmFullName() + ","); + } + Item item = territory.get(territory.size() - 2).getItem(); + out.println(" " + item.getColunmFullName() + ";"); + out.println(); + } + private static void printExpectedReport(TemplateManager tm, String reportName, int upTo, PrintStream out) { String schema = tm.getDictionary().getSchema(); Territory territory = tm.getTerritory(); @@ -388,13 +464,66 @@ private static void printExpectedReport(TemplateManager tm, String reportName, i out.println(" AND (`h`.`" + territoryItem.getItemName() + "` = `e`.`" + territoryItem.getItemName() + "`)"); } out.print(" GROUP BY `name`"); - for (int i = 0; i < upTo; i++) { + for (int i = 1; i < upTo; i++) { out.print(", `h`.`" + territory.get(i).getItemName() + "`"); } out.println(";"); out.println(); } + private static void printExpectedEaReport(TemplateManager tm, String reportName, int upTo, PrintStream out) { + String schema = tm.getDictionary().getSchema(); + Territory territory = tm.getTerritory(); + + out.println("CREATE OR REPLACE VIEW " + schema + ".`" + reportName + "` AS"); + out.println(" SELECT "); + out.print(" _utf8mb4 '" + territory.getFirst().getName()); + for (int i = 1; i < upTo; i++) { + out.print("#" + territory.get(i).getName()); + } + out.println("' COLLATE utf8mb4_unicode_ci AS `name`,"); + for (int i = 0; i < upTo; i++) { + out.println(" NULL AS `" + territory.get(i).getItemName() + "`,"); + } + out.println(" NULL AS `completed`,"); + out.println(" NULL AS `expected`,"); + out.println(" NULL AS `completed_expected`"); + out.println(" "); + out.println(" UNION SELECT "); + out.print(" CONCAT("); + TerritoryItem territoryItem = territory.getFirst(); + out.print(territoryItem.selectDescription()); + for (int i = 1; i < upTo; i++) { + territoryItem = territory.get(i); + out.print(",'#'," + territoryItem.selectDescription()); + } + out.println(") AS `name`,"); + for (int i = 0; i < upTo; i++) { + out.println(" `h`." + territory.get(i).getItemName() + " AS `" + territory.get(i).getItemName() + "`,"); + } + out.println(" COALESCE((`c`.`completed`), 0) AS `completed`,"); + out.println(" SUM(`h`.`expected`) AS `expected`,"); + out.println(" ((COALESCE((`c`.`completed`), 0) / SUM(`h`.`expected`)) * 100) AS `completed_expected`"); + out.println(" FROM"); + printSubTable(tm, "aux_ea_expected", "expected", upTo, out); + out.println(" `h`"); + out.println(" LEFT JOIN "); + printSubTable(tm, "aux_ea_completed", "completed", upTo, out); + out.println(" `c` ON"); + territoryItem = territory.getFirst(); + out.println(" (`h`.`" + territoryItem.getItemName() + "` = `c`.`" + territoryItem.getItemName() + "`)"); + for (int i = 1; i < upTo; i++) { + territoryItem = territory.get(i); + out.println(" AND (`h`.`" + territoryItem.getItemName() + "` = `c`.`" + territoryItem.getItemName() + "`)"); + } + out.print(" GROUP BY `name`"); + for (int i = 1; i < upTo; i++) { + out.print(", `h`.`" + territory.get(i).getItemName() + "`"); + } + out.println(";"); + out.println(); + } + private static void printSubTable(TemplateManager tm, String tableName, String columnName, int upTo, PrintStream out) { String schema = tm.getDictionary().getSchema(); Territory territory = tm.getTerritory(); @@ -409,7 +538,7 @@ private static void printSubTable(TemplateManager tm, String tableName, String c out.println(" FROM `" + schema + "`.`" + tableName + "`"); out.println(" GROUP BY"); out.print(" `" + tableName + "`." + territory.getFirst().getItemName()); - for (int i = 0; i < upTo && i < territory.size(); i++) { + for (int i = 1; i < upTo && i < territory.size(); i++) { TerritoryItem territoryItem = territory.get(i); out.print(",\n `" + tableName + "`." + territoryItem.getItemName()); } @@ -432,6 +561,7 @@ private static void printTotalReport(TemplateManager tm, TemplateManager tmListi out.println(" FROM"); printSubTable(tm, "aux_listing_returned", "returned", 1000, out); out.println(" `a`) AS `ea_freshlist`,"); + out.println(" (SELECT SUM(completed) FROM aux_ea_completed) AS `ea_completed`,"); out.println(" (SELECT "); out.println(" COUNT(0)"); out.println(" FROM"); From 438ffedad17d3e5fd4ccd02fdc54debaa0ac509c Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Wed, 18 Jul 2018 15:21:26 +0200 Subject: [PATCH 08/15] Add sex by broad age groups report --- src/cspro2sql/writer/MonitorWriter.java | 43 +++++++++++++++---------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index 5d89eb2..99f7e64 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -126,23 +126,12 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, if (ageRange != null && tm.hasParam("@INDIVIDUAL_TABLE") && tm.hasParam("@INDIVIDUAL_COLUMN_SEX") && tm.hasParam("@INDIVIDUAL_VALUE_SEX_MALE") && tm.hasParam("@INDIVIDUAL_VALUE_SEX_FEMALE") && tm.hasParam("@INDIVIDUAL_COLUMN_AGE")) { - out.println("CREATE OR REPLACE VIEW " + schema + ".`r_sex_by_age_group` AS"); - out.print(" SELECT '" + ageRange[0] + " to " + (ageRange[1] - 1) + "' as 'range', a.male, b.female FROM " - + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[0] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[1] + ") a," - + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[0] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[1] + ") b"); - for (int i = 1; i < ageRange.length - 1; i++) { - out.println(" UNION"); - out.print(" SELECT '" + ageRange[i] + " to " + (ageRange[i + 1] - 1) + "' as 'range', a.male, b.female FROM " - + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[i] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[i + 1] + ") a," - + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[i] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[i + 1] + ") b"); - } - out.println(" UNION"); - out.print(" SELECT '" + ageRange[ageRange.length - 1] + "+' as 'range', a.male, b.female FROM " - + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[ageRange.length - 1] + ") a," - + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[ageRange.length - 1] + ") b"); - out.println(); - out.println(";"); - printMaterialized(schema, "r_sex_by_age_group", out); + + printAgeBySexReport("r_sex_by_age_group", tm, ageRange, schema, out); + + int[] broadAgeRange = new int[] {0, 5, 18}; + printAgeBySexReport("r_sex_by_broad_age_group", tm, broadAgeRange, schema, out); + } if (!territory.isEmpty()) { @@ -218,6 +207,26 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, return true; } + private static void printAgeBySexReport(String reportName, TemplateManager tm, int [] ageRange, String schema, PrintStream out) { + out.println("CREATE OR REPLACE VIEW " + schema + ".`" + reportName + "` AS"); + out.print(" SELECT '" + ageRange[0] + " to " + (ageRange[1] - 1) + "' as 'range', a.male, b.female FROM " + + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[0] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[1] + ") a," + + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[0] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[1] + ") b"); + for (int i = 1; i < ageRange.length - 1; i++) { + out.println(" UNION"); + out.print(" SELECT '" + ageRange[i] + " to " + (ageRange[i + 1] - 1) + "' as 'range', a.male, b.female FROM " + + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[i] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[i + 1] + ") a," + + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[i] + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " < " + ageRange[i + 1] + ") b"); + } + out.println(" UNION"); + out.print(" SELECT '" + ageRange[ageRange.length - 1] + "+' as 'range', a.male, b.female FROM " + + "(SELECT COUNT(0) male FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_MALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[ageRange.length - 1] + ") a," + + "(SELECT COUNT(0) female FROM " + schema + "." + tm.getParam("@INDIVIDUAL_TABLE") + " WHERE " + tm.getParam("@INDIVIDUAL_COLUMN_SEX") + " = " + tm.getParam("@INDIVIDUAL_VALUE_SEX_FEMALE") + " AND " + tm.getParam("@INDIVIDUAL_COLUMN_AGE") + " >= " + ageRange[ageRange.length - 1] + ") b"); + out.println(); + out.println(";"); + printMaterialized(schema, reportName, out); + } + private static void printTerritoryTable(Territory territory, AreaNameFile areaNames, PrintStream out) throws IOException { out.println("CREATE TABLE IF NOT EXISTS `territory` ("); String idx = ""; From 676f48943af955af8b6685ee8bc70168bbc024ed Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Thu, 19 Jul 2018 20:35:12 +0200 Subject: [PATCH 09/15] Left join in exp hh rep to show EAs w. no hhs and also various SQL fixes to exp hh rep queries. --- src/cspro2sql/writer/MonitorWriter.java | 32 ++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index 99f7e64..eec1a93 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -445,17 +445,17 @@ private static void printExpectedReport(TemplateManager tm, String reportName, i for (int i = 0; i < upTo; i++) { out.println(" `h`." + territory.get(i).getItemName() + " AS `" + territory.get(i).getItemName() + "`,"); } - out.println(" SUM(`h`.`returned`) AS `returned`,"); + out.println(" SUM(`e`.`returned`) AS `returned`,"); out.println(" SUM(`l`.`returned`) AS `returned`,"); - out.println(" SUM(`e`.`expected`) AS `expected`,"); - out.println(" ((SUM(`h`.`returned`) / SUM(`l`.`returned`)) * 100) AS `field_freshlist`,"); - out.println(" ((SUM(`h`.`returned`) / SUM(`e`.`expected`)) * 100) AS `field_expected`,"); - out.println(" ((SUM(`l`.`returned`) / SUM(`e`.`expected`)) * 100) AS `freshlist_expected`"); + out.println(" SUM(`h`.`expected`) AS `expected`,"); + out.println(" ((SUM(`e`.`returned`) / SUM(`l`.`returned`)) * 100) AS `field_freshlist`,"); + out.println(" ((SUM(`e`.`returned`) / SUM(`h`.`expected`)) * 100) AS `field_expected`,"); + out.println(" ((SUM(`l`.`returned`) / SUM(`h`.`expected`)) * 100) AS `freshlist_expected`"); out.println(" FROM"); - printSubTable(tm, "aux_household_returned", "returned", upTo, out); + printSubTable(tm, "aux_household_expected", "expected", upTo, out); out.println(" `h`"); - out.println(" JOIN "); - printSubTable(tm, "aux_listing_returned", "returned", upTo, out); + out.println(" LEFT JOIN "); + printSubTable(tm, "aux_household_returned", "returned", upTo, out); out.println(" `l` ON"); territoryItem = territory.getFirst(); out.println(" (`h`.`" + territoryItem.getItemName() + "` = `l`.`" + territoryItem.getItemName() + "`)"); @@ -463,8 +463,8 @@ private static void printExpectedReport(TemplateManager tm, String reportName, i territoryItem = territory.get(i); out.println(" AND (`h`.`" + territoryItem.getItemName() + "` = `l`.`" + territoryItem.getItemName() + "`)"); } - out.println(" JOIN"); - printSubTable(tm, "aux_household_expected", "expected", upTo, out); + out.println(" LEFT JOIN"); + printSubTable(tm, "aux_listing_returned", "returned", upTo, out); out.println(" `e` ON"); territoryItem = territory.getFirst(); out.println(" (`h`.`" + territoryItem.getItemName() + "` = `e`.`" + territoryItem.getItemName() + "`)"); @@ -473,7 +473,7 @@ private static void printExpectedReport(TemplateManager tm, String reportName, i out.println(" AND (`h`.`" + territoryItem.getItemName() + "` = `e`.`" + territoryItem.getItemName() + "`)"); } out.print(" GROUP BY `name`"); - for (int i = 1; i < upTo; i++) { + for (int i = 0; i < upTo; i++) { out.print(", `h`.`" + territory.get(i).getItemName() + "`"); } out.println(";"); @@ -510,9 +510,9 @@ private static void printExpectedEaReport(TemplateManager tm, String reportName, for (int i = 0; i < upTo; i++) { out.println(" `h`." + territory.get(i).getItemName() + " AS `" + territory.get(i).getItemName() + "`,"); } - out.println(" COALESCE((`c`.`completed`), 0) AS `completed`,"); + out.println(" SUM(COALESCE(`c`.`completed`, 0)) AS `completed`,"); out.println(" SUM(`h`.`expected`) AS `expected`,"); - out.println(" ((COALESCE((`c`.`completed`), 0) / SUM(`h`.`expected`)) * 100) AS `completed_expected`"); + out.println(" ((SUM(COALESCE(`c`.`completed`, 0)) / SUM(`h`.`expected`)) * 100) AS `completed_expected`"); out.println(" FROM"); printSubTable(tm, "aux_ea_expected", "expected", upTo, out); out.println(" `h`"); @@ -526,7 +526,7 @@ private static void printExpectedEaReport(TemplateManager tm, String reportName, out.println(" AND (`h`.`" + territoryItem.getItemName() + "` = `c`.`" + territoryItem.getItemName() + "`)"); } out.print(" GROUP BY `name`"); - for (int i = 1; i < upTo; i++) { + for (int i = 0; i < upTo; i++) { out.print(", `h`.`" + territory.get(i).getItemName() + "`"); } out.println(";"); @@ -570,7 +570,7 @@ private static void printTotalReport(TemplateManager tm, TemplateManager tmListi out.println(" FROM"); printSubTable(tm, "aux_listing_returned", "returned", 1000, out); out.println(" `a`) AS `ea_freshlist`,"); - out.println(" (SELECT SUM(completed) FROM aux_ea_completed) AS `ea_completed`,"); + out.println(" (SELECT COALESCE(SUM(completed),0) FROM aux_ea_completed) AS `ea_completed`,"); out.println(" (SELECT "); out.println(" COUNT(0)"); out.println(" FROM"); @@ -585,7 +585,7 @@ private static void printTotalReport(TemplateManager tm, TemplateManager tmListi out.println(" FROM"); out.println(" " + tmListing.getDictionary().getMainRecord().getFullTableName() + ") AS `household_freshlist`,"); out.println(" (SELECT "); - out.println(" SUM(`a`.`expected`)"); + out.println(" COALESCE(SUM(`a`.`expected`), 0)"); out.println(" FROM"); printSubTable(tm, "aux_household_expected", "expected", 1000, out); out.println(" `a`) AS `household_expected`;"); From 95d534be93a7fd1392a2a226793e0883ed78dc06 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Fri, 10 Aug 2018 09:16:42 -0400 Subject: [PATCH 10/15] Remove trailing spaces in territory names --- src/cspro2sql/bean/Territory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cspro2sql/bean/Territory.java b/src/cspro2sql/bean/Territory.java index ab46902..8fec90d 100644 --- a/src/cspro2sql/bean/Territory.java +++ b/src/cspro2sql/bean/Territory.java @@ -47,9 +47,9 @@ public void addItems(Iterable dictItems) throws IOException { Tag tag = dictItem.getTag(Dictionary.TAG_TERRITORY); if (tag.getValue() != null) { String[] tagValues = tag.getValue().split(","); - dictItemToName.put(dictItem, tagValues[0]); + dictItemToName.put(dictItem, tagValues[0].trim()); if (tagValues.length > 1 && !tagValues[1].isEmpty()) { - dictItemToParentName.put(dictItem, tagValues[1]); + dictItemToParentName.put(dictItem, tagValues[1].trim()); } } } From e88cc7ba0303a756204e29f953a4ac89ff660316 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Fri, 10 Aug 2018 09:30:18 -0400 Subject: [PATCH 11/15] Fix bug in SQL with listing table --- src/cspro2sql/writer/MonitorWriter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index eec1a93..cacd1b3 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -455,7 +455,7 @@ private static void printExpectedReport(TemplateManager tm, String reportName, i printSubTable(tm, "aux_household_expected", "expected", upTo, out); out.println(" `h`"); out.println(" LEFT JOIN "); - printSubTable(tm, "aux_household_returned", "returned", upTo, out); + printSubTable(tm, "aux_listing_returned", "returned", upTo, out); out.println(" `l` ON"); territoryItem = territory.getFirst(); out.println(" (`h`.`" + territoryItem.getItemName() + "` = `l`.`" + territoryItem.getItemName() + "`)"); @@ -464,7 +464,7 @@ private static void printExpectedReport(TemplateManager tm, String reportName, i out.println(" AND (`h`.`" + territoryItem.getItemName() + "` = `l`.`" + territoryItem.getItemName() + "`)"); } out.println(" LEFT JOIN"); - printSubTable(tm, "aux_listing_returned", "returned", upTo, out); + printSubTable(tm, "aux_household_returned", "returned", upTo, out); out.println(" `e` ON"); territoryItem = territory.getFirst(); out.println(" (`h`.`" + territoryItem.getItemName() + "` = `e`.`" + territoryItem.getItemName() + "`)"); From e8838cea90fac981f9f308c02191681ac8d53f08 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Wed, 12 Sep 2018 08:13:09 -0400 Subject: [PATCH 12/15] Handle dictionaries with record type in unconvential location --- src/cspro2sql/bean/BeanFactory.java | 4 ++++ src/cspro2sql/bean/Dictionary.java | 18 ++++++++++++++++++ src/cspro2sql/reader/QuestionnaireReader.java | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/cspro2sql/bean/BeanFactory.java b/src/cspro2sql/bean/BeanFactory.java index c3b258a..cfc23d4 100644 --- a/src/cspro2sql/bean/BeanFactory.java +++ b/src/cspro2sql/bean/BeanFactory.java @@ -37,6 +37,10 @@ public static void parseDictionary(BufferedReader br, Dictionary dictionary) thr dictionary.setName(getValue(line)); } else if (line.startsWith(Dictionary.DICT_NOTE)) { parseNote(dictionary, getValue(line)); + } else if (line.startsWith(Dictionary.HEADER_RECSTART)) { + dictionary.setRecordTypeStart(Integer.parseInt(getValue(line))); + } else if (line.startsWith(Dictionary.HEADER_RECLEN)) { + dictionary.setRecordTypeLength(Integer.parseInt(getValue(line))); } else if (line.isEmpty()) { break; } diff --git a/src/cspro2sql/bean/Dictionary.java b/src/cspro2sql/bean/Dictionary.java index 2184964..02e8ee5 100644 --- a/src/cspro2sql/bean/Dictionary.java +++ b/src/cspro2sql/bean/Dictionary.java @@ -119,6 +119,8 @@ public final class Dictionary extends Taggable { private final Map> tags = new HashMap<>(); private String name; + private int recordTypeStart; + private int recordTypeLength; private Record lastRecord; private List lastItems; private List lastItemsNotSubItem; @@ -144,6 +146,22 @@ public void setName(String name) { this.name = name; } + public int getRecordTypeStart() { + return recordTypeStart; + } + + public void setRecordTypeStart(int start) { + this.recordTypeStart = start; + } + + public int getRecordTypeLength() { + return recordTypeLength; + } + + public void setRecordTypeLength(int length) { + this.recordTypeLength = length; + } + public void addRecord(Record record) { if (record.hasTag(TAG_IGNORE)) { return; diff --git a/src/cspro2sql/reader/QuestionnaireReader.java b/src/cspro2sql/reader/QuestionnaireReader.java index 823f683..ec8517c 100644 --- a/src/cspro2sql/reader/QuestionnaireReader.java +++ b/src/cspro2sql/reader/QuestionnaireReader.java @@ -43,7 +43,7 @@ public static Questionnaire parse(Dictionary dictionary, String questionnaire, S parseItem(item, rows[0], values); } for (String row : rows) { - record = dictionary.getRecord(row.charAt(0)); + record = dictionary.getRecord(row.substring(dictionary.getRecordTypeStart() - 1, dictionary.getRecordTypeStart() - 1 + dictionary.getRecordTypeLength())); valuesList = result.getRecordValues(record); if (valuesList == null) { valuesList = new LinkedList<>(); From 8c0b2fe1f017a0bc0a39b6d7301b4e2889d682dc Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Mon, 17 Sep 2018 09:57:44 -0400 Subject: [PATCH 13/15] Optimize religion query --- src/cspro2sql/sql/template/r_religion.sql | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/cspro2sql/sql/template/r_religion.sql b/src/cspro2sql/sql/template/r_religion.sql index b42153c..5f6623a 100644 --- a/src/cspro2sql/sql/template/r_religion.sql +++ b/src/cspro2sql/sql/template/r_religion.sql @@ -1,8 +1,5 @@ CREATE OR REPLACE VIEW @SCHEMA.`r_religion` AS SELECT - `vs`.`VALUE` AS `RELIGION`, COUNT(0) AS `INDIVIDUALS` - FROM - (@SCHEMA.@INDIVIDUAL_TABLE `i` - JOIN @SCHEMA.@VALUESET_RELIGION `vs` ON ((`i`.@INDIVIDUAL_COLUMN_RELIGION = `vs`.`ID`))) - GROUP BY `vs`.`VALUE`; - + `vs`.`VALUE` AS `RELIGION`, `i`.`INDIVIDUALS` + FROM (SELECT @INDIVIDUAL_COLUMN_RELIGION, COUNT(0) AS `INDIVIDUALS` FROM @SCHEMA.@INDIVIDUAL_TABLE GROUP BY @INDIVIDUAL_COLUMN_RELIGION) `i` + JOIN @SCHEMA.@VALUESET_RELIGION `vs` ON `i`.@INDIVIDUAL_COLUMN_RELIGION = `vs`.`ID`; \ No newline at end of file From 0ff3fb1deca04a57fc1f597201b572d9b583ec64 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Mon, 17 Sep 2018 09:59:41 -0400 Subject: [PATCH 14/15] Fix problems when no EA status supplied. Backtick table names in cases where it was not being done --- src/cspro2sql/UpdateEngine.java | 4 ++-- src/cspro2sql/writer/MonitorWriter.java | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/cspro2sql/UpdateEngine.java b/src/cspro2sql/UpdateEngine.java index f1763bc..ab2305f 100644 --- a/src/cspro2sql/UpdateEngine.java +++ b/src/cspro2sql/UpdateEngine.java @@ -63,9 +63,9 @@ static boolean execute(Properties prop) { while (rs.next()) { String template = rs.getString(1); System.out.print("Updating " + template + "... "); - writeDst.executeUpdate("DROP TABLE IF EXISTS " + schema + ".m" + template); + writeDst.executeUpdate("DROP TABLE IF EXISTS " + schema + ".`m" + template + "`"); writeDst.executeQuery("SELECT 0 INTO @ID"); - writeDst.executeUpdate("CREATE TABLE " + schema + ".m" + template + " (PRIMARY KEY (ID)) AS SELECT @ID := @ID + 1 ID, " + template + ".* FROM " + schema + "." + template); + writeDst.executeUpdate("CREATE TABLE " + schema + ".`m" + template + "` (PRIMARY KEY (ID)) AS SELECT @ID := @ID + 1 ID, `" + template + "`.* FROM " + schema + ".`" + template + "`"); connDst.commit(); System.out.println("done"); } diff --git a/src/cspro2sql/writer/MonitorWriter.java b/src/cspro2sql/writer/MonitorWriter.java index cacd1b3..d5bafac 100644 --- a/src/cspro2sql/writer/MonitorWriter.java +++ b/src/cspro2sql/writer/MonitorWriter.java @@ -196,7 +196,7 @@ public static boolean write(TemplateManager tm, TemplateManager tmListing, } } - printTotalReport(tm, (tmListing != null) ? tmListing : tm, out); + printTotalReport(tm, (tmListing != null) ? tmListing : tm, createEaTables, out); printMaterialized(schema, "r_total", out); } catch (IOException ex) { System.err.println(ex.getMessage()); @@ -302,9 +302,9 @@ private static void printTerritoryTable(Territory territory, AreaNameFile areaNa } private static void printMaterialized(String schema, String name, PrintStream out) { - out.println("DROP TABLE IF EXISTS " + schema + ".m" + name + ";"); + out.println("DROP TABLE IF EXISTS " + schema + ".`m" + name + "`;"); out.println("SELECT 0 INTO @ID;"); - out.println("CREATE TABLE " + schema + ".m" + name + " (PRIMARY KEY (ID)) AS SELECT @ID := @ID + 1 ID, " + name + ".* FROM " + schema + "." + name + ";"); + out.println("CREATE TABLE " + schema + ".`m" + name + "` (PRIMARY KEY (ID)) AS SELECT @ID := @ID + 1 ID, `" + name + "`.* FROM " + schema + ".`" + name + "`;"); out.println("INSERT INTO " + schema + ".`cspro2sql_report` VALUES ('" + name + "', " + (reportCount++) + ");"); out.println(); } @@ -555,7 +555,7 @@ private static void printSubTable(TemplateManager tm, String tableName, String c out.println(" )"); } - private static void printTotalReport(TemplateManager tm, TemplateManager tmListing, PrintStream out) { + private static void printTotalReport(TemplateManager tm, TemplateManager tmListing, boolean hasEACompleted, PrintStream out) { String schema = tm.getDictionary().getSchema(); out.println("CREATE OR REPLACE VIEW `" + schema + "`.`r_total` AS"); @@ -570,7 +570,11 @@ private static void printTotalReport(TemplateManager tm, TemplateManager tmListi out.println(" FROM"); printSubTable(tm, "aux_listing_returned", "returned", 1000, out); out.println(" `a`) AS `ea_freshlist`,"); - out.println(" (SELECT COALESCE(SUM(completed),0) FROM aux_ea_completed) AS `ea_completed`,"); + if (hasEACompleted) { + out.println(" (SELECT COALESCE(SUM(completed),0) FROM aux_ea_completed) AS `ea_completed`,"); + } else { + out.println(" (SELECT 0 FROM aux_household_expected LIMIT 1) AS `ea_completed`,"); + } out.println(" (SELECT "); out.println(" COUNT(0)"); out.println(" FROM"); From c8cbd410ff058e7c970bb339479bb900f8ae9b92 Mon Sep 17 00:00:00 2001 From: Josh Handley Date: Fri, 29 Mar 2019 16:34:01 -0400 Subject: [PATCH 15/15] Handle dictionaries with no record type --- src/cspro2sql/reader/QuestionnaireReader.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/cspro2sql/reader/QuestionnaireReader.java b/src/cspro2sql/reader/QuestionnaireReader.java index ec8517c..27816de 100644 --- a/src/cspro2sql/reader/QuestionnaireReader.java +++ b/src/cspro2sql/reader/QuestionnaireReader.java @@ -43,7 +43,7 @@ public static Questionnaire parse(Dictionary dictionary, String questionnaire, S parseItem(item, rows[0], values); } for (String row : rows) { - record = dictionary.getRecord(row.substring(dictionary.getRecordTypeStart() - 1, dictionary.getRecordTypeStart() - 1 + dictionary.getRecordTypeLength())); + record = dictionary.getRecord(getRecordType(dictionary, row)); valuesList = result.getRecordValues(record); if (valuesList == null) { valuesList = new LinkedList<>(); @@ -58,6 +58,17 @@ record = dictionary.getRecord(row.substring(dictionary.getRecordTypeStart() - 1, return result; } + private static String getRecordType(Dictionary dictionary, String dataRow) + { + if (dictionary.getRecordTypeLength() == 0) { + // Dictionaries with only one record may omit the record type + // by setting record length to zero. + return ""; + } else { + return dataRow.substring(dictionary.getRecordTypeStart() - 1, dictionary.getRecordTypeStart() - 1 + dictionary.getRecordTypeLength()); + } + } + private static void parseItem(Item item, String row, List values) { if (row.length() < item.getStart()) { values.add(new Answer(item, null));