From a066d539ec814d2334d3eb72d738764425379cc4 Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 22 Jan 2026 15:43:29 -0800 Subject: [PATCH 1/4] Support field type conversion for Multi Choice fields --- api/src/org/labkey/api/data/TableChange.java | 17 + .../query/AbstractQueryChangeListener.java | 6 +- .../labkey/api/query/QueryChangeListener.java | 181 +- .../org/labkey/api/query/QueryService.java | 2 +- core/package-lock.json | 8 +- core/package.json | 2 +- .../core/dialect/PostgreSql92Dialect.java | 2374 +++++++------- experiment/package-lock.json | 8 +- experiment/package.json | 2 +- .../ExperimentQueryChangeListener.java | 2 +- .../PropertyQueryChangeListener.java | 2 +- .../api/property/DomainPropertyImpl.java | 2779 +++++++++-------- .../api/property/StorageProvisionerImpl.java | 29 + .../query/CustomViewQueryChangeListener.java | 66 +- .../query/QueryDefQueryChangeListener.java | 2 +- .../org/labkey/query/QueryServiceImpl.java | 8 +- .../QuerySnapshotQueryChangeListener.java | 282 +- .../labkey/query/persist/QueryManager.java | 2354 +++++++------- .../reports/ReportQueryChangeListener.java | 2 +- .../QueryDatasetQueryChangeListener.java | 2 +- 20 files changed, 4254 insertions(+), 3874 deletions(-) diff --git a/api/src/org/labkey/api/data/TableChange.java b/api/src/org/labkey/api/data/TableChange.java index 5e8c6a1245d..1ae2dd6d094 100644 --- a/api/src/org/labkey/api/data/TableChange.java +++ b/api/src/org/labkey/api/data/TableChange.java @@ -20,6 +20,7 @@ import org.labkey.api.data.PropertyStorageSpec.Index; import org.labkey.api.data.TableInfo.IndexDefinition; import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; import org.labkey.api.exp.property.Domain; import org.labkey.api.exp.property.DomainKind; import org.labkey.api.util.logging.LogHelper; @@ -58,6 +59,7 @@ public class TableChange private Collection _constraints; private Set _indicesToBeDroppedByName; private IndexSizeMode _sizeMode = IndexSizeMode.Auto; + private Map _oldPropTypes; /** In most cases, domain knows the storage table name **/ public TableChange(Domain domain, ChangeType changeType) @@ -329,6 +331,16 @@ public void setForeignKeys(Collection foreignKey _foreignKeys = foreignKeys; } + public Map getOldPropTypes() + { + return _oldPropTypes; + } + + public void setOldPropTypes(Map oldPropTypes) + { + _oldPropTypes = oldPropTypes; + } + public final List toSpecs(Collection columnNames) { final Domain domain = _domain; @@ -349,6 +361,11 @@ public final List toSpecs(Collection columnNames) .collect(Collectors.toList()); } + public void setOldPropertyTypes(Map oldPropTypes) + { + _oldPropTypes = oldPropTypes; + } + public enum ChangeType { CreateTable, diff --git a/api/src/org/labkey/api/query/AbstractQueryChangeListener.java b/api/src/org/labkey/api/query/AbstractQueryChangeListener.java index 57c76f78dd3..4e4eb2ac135 100644 --- a/api/src/org/labkey/api/query/AbstractQueryChangeListener.java +++ b/api/src/org/labkey/api/query/AbstractQueryChangeListener.java @@ -40,13 +40,13 @@ public void queryCreated(User user, Container container, ContainerFilter scope, protected abstract void queryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, String query); @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { for (QueryPropertyChange change : changes) - queryChanged(user, container, scope, schema, change); + queryChanged(user, container, scope, schema, queryName, change); } - protected abstract void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, QueryPropertyChange change); + protected abstract void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, QueryPropertyChange change); @Override public void queryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) diff --git a/api/src/org/labkey/api/query/QueryChangeListener.java b/api/src/org/labkey/api/query/QueryChangeListener.java index 59d96dc79ee..06a80b681b9 100644 --- a/api/src/org/labkey/api/query/QueryChangeListener.java +++ b/api/src/org/labkey/api/query/QueryChangeListener.java @@ -20,10 +20,14 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; import org.labkey.api.event.PropertyChange; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; import org.labkey.api.security.User; import java.util.Collection; import java.util.Collections; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Listener for table and query events that fires when the structure/schema changes, but not when individual data @@ -58,10 +62,11 @@ public interface QueryChangeListener * @param container The container the tables or queries are changed in. * @param scope The scope of containers that the tables or queries affect. * @param schema The schema of the tables or queries. + * @param queryName The query name if the change is specific to a single query. * @param property The QueryProperty that has changed. * @param changes The set of change events. Each QueryPropertyChange is associated with a single table or query. */ - void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes); + void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @Nullable String queryName, @NotNull QueryProperty property, @NotNull Collection> changes); /** * This method is called when a set of tables or queries are deleted from the given container and schema. @@ -94,7 +99,9 @@ enum QueryProperty Description(String.class), Inherit(Boolean.class), Hidden(Boolean.class), - SchemaName(String.class),; + SchemaName(String.class), + ColumnName(String.class), + ColumnType(PropertyType.class),; private final Class _klass; @@ -112,7 +119,7 @@ public Class getPropertyClass() /** * A change event for a single property of a single table or query. * If multiple properties have been changed, QueryChangeListener will - * fire {@link QueryChangeListener#queryChanged(User, Container, ContainerFilter, SchemaKey, QueryChangeListener.QueryProperty, Collection)} + * fire {@link QueryChangeListener#queryChanged(User, Container, ContainerFilter, SchemaKey, String, QueryChangeListener.QueryProperty, Collection)} * for each property that has changed. * * @param The property type. @@ -171,6 +178,22 @@ public static void handleSchemaNameChange(@NotNull String oldValue, String newVa QueryProperty.SchemaName, Collections.singleton(change)); } + public static void handleColumnTypeChange(@NotNull PropertyDescriptor oldValue, PropertyDescriptor newValue, @NotNull SchemaKey schemaPath, @NotNull String queryName, User user, Container container) + { + if (oldValue.getPropertyType() == newValue.getPropertyType()) + return; + + QueryChangeListener.QueryPropertyChange change = new QueryChangeListener.QueryPropertyChange<>( + null, + QueryChangeListener.QueryProperty.ColumnType, + oldValue, + newValue + ); + + QueryService.get().fireQueryColumnChanged(user, container, schemaPath, queryName, + QueryProperty.ColumnType, Collections.singleton(change)); + } + @Nullable public QueryDefinition getSource() { return _queryDef; } @Override @@ -185,4 +208,156 @@ public static void handleSchemaNameChange(@NotNull String oldValue, String newVa @Nullable public V getNewValue() { return _newValue; } } + + /** + * Utility to update encoded filter string when a column type changes from Multi_Choice to a non Multi_Choice. + * This method performs targeted replacements for the given column name (case-insensitive). + */ + private static String getUpdatedFilterStrFromMVTC(String filterStr, String columnName, PropertyType oldType, PropertyType newType) + { + if (filterStr == null || columnName == null || oldType == null || newType == null) + return filterStr; + + // Only act when changing away from MULTI_CHOICE + if (oldType != PropertyType.MULTI_CHOICE || newType == PropertyType.MULTI_CHOICE) + return filterStr; + + String colLower = columnName.toLowerCase(); + String sLower = filterStr.toLowerCase(); + + // No action if column doesn't match + if (!sLower.startsWith("filter." + colLower + "~")) + return filterStr; + + // drop arraycontainsall since there is no good match + if (sLower.startsWith("filter." + colLower + "~arraycontainsall")) + return ""; + + String updated = filterStr; + + if (containsOp(updated, columnName, "arrayisempty")) + { + return replaceOp(updated, columnName, "arrayisempty", "isblank"); + } + if (containsOp(updated, columnName, "arrayisnotempty")) + { + return replaceOp(updated, columnName, "arrayisnotempty", "isnonblank"); + } + if (containsOp(updated, columnName, "arraymatches")) + { + updated = replaceOp(updated, columnName, "arraymatches", "eq"); + // Replace all occurrences of %2C with %2C%20, + // "," -> ", " during array to string conversion + return updated.replace("%2C", "%2C%20"); + } + if (containsOp(updated, columnName, "arraynotmatches")) + { + updated = replaceOp(updated, columnName, "arraynotmatches", "neq"); + // Replace all occurrences of %2C with %2C%20 + return updated.replace("%2C", "%2C%20"); + } + if (containsOp(updated, columnName, "arraycontainsany")) + { + updated = replaceOp(updated, columnName, "arraycontainsany", "in"); + // Replace all occurrences of %2C with %3B + // ";" is used as the separator for "in" operator + return updated.replace("%2C", "%3B"); + } + if (containsOp(updated, columnName, "arraycontainsnone")) + { + updated = replaceOp(updated, columnName, "arraycontainsnone", "notin"); + // Replace all occurrences of %2C with %3B + // ";" is used as the separator for "notin" operator + return updated.replace("%2C", "%3B"); + } + + // No matching operator found for this column, drop the filter + return ""; + } + + /** + * Utility to update encoded filter string when a column type is changed to Multi_Choice (migrating operators to array equivalents). + */ + private static String getUpdatedMVTCFilterStr(String filterStr, String columnName, PropertyType oldType, PropertyType newType) + { + if (filterStr == null || columnName == null || oldType == null || newType == null) + return filterStr; + + // Only act when changing to MULTI_CHOICE + if (oldType == PropertyType.MULTI_CHOICE || newType != PropertyType.MULTI_CHOICE) + return filterStr; + + String colLower = columnName.toLowerCase(); + String sLower = filterStr.toLowerCase(); + + // No action if column doesn't match + if (!sLower.startsWith("filter." + colLower + "~")) + return filterStr; + + String updated = filterStr; + + // Return on first matching operator for this column + if (containsOp(updated, columnName, "eq")) + { + return replaceOp(updated, columnName, "eq", "arraymatches"); + } + if (containsOp(updated, columnName, "neq")) + { + return replaceOp(updated, columnName, "neq", "arraycontainsnone"); + } + if (containsOp(updated, columnName, "isblank")) + { + return replaceOp(updated, columnName, "isblank", "arrayisempty"); + } + if (containsOp(updated, columnName, "isnonblank")) + { + return replaceOp(updated, columnName, "isnonblank", "arrayisnotempty"); + } + if (containsOp(updated, columnName, "in")) + { + updated = replaceOp(updated, columnName, "in", "arraycontainsany"); + // update ";" to "," for separator appropriate for array operator + return updated.replace("%3B", "%2C"); + } + if (containsOp(updated, columnName, "notin")) + { + updated = replaceOp(updated, columnName, "notin", "arraycontainsnone"); + return updated.replace("%3B", "%2C"); + } + + // No matching operator found for this column, drop the filter + return ""; + } + + static String getUpdatedFilterStrOnColumnTypeUpdate(String filterStr, String columnName, PropertyType oldType, PropertyType newType) + { + if (oldType == PropertyType.MULTI_CHOICE) + return getUpdatedFilterStrFromMVTC(filterStr, columnName, oldType, newType); + else if (newType == PropertyType.MULTI_CHOICE) + return getUpdatedMVTCFilterStr(filterStr, columnName, oldType, newType); + else + return filterStr; + } + + private static boolean containsOp(String filterStr, String columnName, String op) + { + String regex = "(?i)filter\\." + Pattern.quote(columnName) + "~" + Pattern.quote(op); + return Pattern.compile(regex).matcher(filterStr).find(); + } + + private static String replaceOp(String filterStr, String columnName, String fromOp, String toOp) + { + String regex = "(?i)(filter\\.)" + Pattern.quote(columnName) + "(~)" + Pattern.quote(fromOp); + Matcher m = Pattern.compile(regex).matcher(filterStr); + StringBuffer sb = new StringBuffer(); + while (m.find()) + { + // Preserve the literal 'filter.' and '~', but use the provided columnName casing and new operator + String replacement = m.group(1) + columnName + m.group(2) + toOp; + m.appendReplacement(sb, Matcher.quoteReplacement(replacement)); + } + m.appendTail(sb); + return sb.toString(); + } + } diff --git a/api/src/org/labkey/api/query/QueryService.java b/api/src/org/labkey/api/query/QueryService.java index bb6d87b7614..942d6a79461 100644 --- a/api/src/org/labkey/api/query/QueryService.java +++ b/api/src/org/labkey/api/query/QueryService.java @@ -491,7 +491,7 @@ public String getDefaultCommentSummary() void fireQueryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries); void fireQueryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, QueryChangeListener.QueryProperty property, Collection> changes); void fireQueryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries); - + void fireQueryColumnChanged(User user, Container container, @NotNull SchemaKey schemaPath, @NotNull String queryName, QueryChangeListener.QueryProperty property, Collection> changes); /** OLAP **/ // could make this a separate service diff --git a/core/package-lock.json b/core/package-lock.json index c9c708e80a2..ef72ba6e0e2 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.13.0", + "@labkey/components": "7.14.0-fb-mvtc-convert.1", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.13.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.13.0.tgz", - "integrity": "sha512-+2o42no7q9IInKbvSd5XHDrnmLKucgudQ+7C2FD6ya+Da8mRu76GWG6L168iwbtMaguQZzFQmMGpD5VScWZiyQ==", + "version": "7.14.0-fb-mvtc-convert.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.14.0-fb-mvtc-convert.1.tgz", + "integrity": "sha512-LDwQkXH1oAsDTn0C+Ep6JvIl4sqXE2c5v+a/veezi96ltzWgXd7gOPrUubFRbLO0xXG3nn8QzGWdu0G7Zd0kWg==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index 2c1202f3221..7107e778af0 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.13.0", + "@labkey/components": "7.14.0-fb-mvtc-convert.1", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java index c5fbcad586b..01f833d58f5 100644 --- a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java +++ b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java @@ -1,1151 +1,1223 @@ -/* - * Copyright (c) 2012-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.core.dialect; - -import jakarta.servlet.ServletException; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.Constraint; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DatabaseIdentifier; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.InClauseGenerator; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.ParameterMarkerInClauseGenerator; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.Selector; -import org.labkey.api.data.Selector.ForEachBlock; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableChange; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TempTableInClauseGenerator; -import org.labkey.api.data.TempTableTracker; -import org.labkey.api.data.dialect.BasePostgreSqlDialect; -import org.labkey.api.data.dialect.DialectStringHandler; -import org.labkey.api.data.dialect.JdbcHelper; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.data.dialect.StandardJdbcHelper; -import org.labkey.api.query.AliasManager; -import org.labkey.api.util.ConfigurationException; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.view.template.Warnings; -import org.labkey.core.admin.sql.ScriptReorderer; -import org.springframework.jdbc.BadSqlGrammarException; - -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.Driver; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Level; -import java.util.logging.LogManager; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/* - * This is the base class defining PostgreSQL-specific (i.e., not Redshift) behavior. PostgreSQL 9.2 is no longer - * supported; however, we keep this class to track changes we implemented specifically for this version. - */ -abstract class PostgreSql92Dialect extends BasePostgreSqlDialect -{ - public static final String PRODUCT_NAME = "PostgreSQL"; - - // This has been the standard PostgreSQL identifier max byte length for many years. However, this could change in - // the future plus servers can be compiled with a different limit, so we query this setting on first connection to - // each database. - private int _maxIdentifierByteLength = 63; - private InClauseGenerator _inClauseGenerator; - - private final TempTableInClauseGenerator _tempTableInClauseGenerator = new TempTableInClauseGenerator(); - private final AtomicBoolean _arraySortFunctionExists = new AtomicBoolean(false); - - @Override - public void handleCreateDatabaseException(SQLException e) throws ServletException - { - if ("55006".equals(e.getSQLState())) - { - LOG.error("You must close down pgAdmin III and all other applications accessing PostgreSQL."); - throw (new ServletException("Close down or disconnect pgAdmin III and all other applications accessing PostgreSQL", e)); - } - else - { - super.handleCreateDatabaseException(e); - } - } - - @Override - public void prepareDriver(Class driverClass) - { - // PostgreSQL driver 42.0.0 added logging via the Java Logging API (java.util.logging). This caused the driver to - // start logging SQLExceptions (such as the initial connection failure on bootstrap) to the console... harmless - // but annoying. This code suppresses the driver logging. - Logger pgjdbcLogger = LogManager.getLogManager().getLogger("org.postgresql"); - - if (null != pgjdbcLogger) - pgjdbcLogger.setLevel(Level.OFF); - } - - // Make sure that the PL/pgSQL language is enabled in the associated database. If not, throw. Since 9.0, PostgreSQL has - // shipped with PL/pgSQL enabled by default, so the check is no longer critical, but continue to verify just to be safe. - @Override - public void prepareNewLabKeyDatabase(DbScope scope) - { - if (new SqlSelector(scope, "SELECT * FROM pg_language WHERE lanname = 'plpgsql'").exists()) - return; - - String dbName = scope.getDatabaseName(); - String message = "PL/pgSQL is not enabled in the \"" + dbName + "\" database because it is not enabled in your Template1 master database."; - String advice = "Use PostgreSQL's 'createlang' command line utility to enable PL/pgSQL in the \"" + dbName + "\" database then restart Tomcat."; - - throw new ConfigurationException(message, advice); - } - - @Override - public String prepare(DbScope scope) - { - initializeInClauseGenerator(scope); - determineIfArraySortFunctionExists(scope); - return super.prepare(scope); - } - - @NotNull - @Override - protected Set getReservedWords() - { - Set words = super.getReservedWords(); - words.add("collation"); - - return words; - } - - /* - These override method implementations were migrated from PostgreSql91Dialect when that class was promoted to api: - getProductName() - createStringHandler() - getJdbcHelper() - getScriptWarnings() - initializeInClauseGenerator() - */ - - @Override - public String getProductName() - { - return PRODUCT_NAME; - } - - // Query PostgreSQL-specific settings - @Override - protected void determineSettings(DbScope scope) - { - if (getServerType().supportsSpecialMetadataQueries()) - { - super.determineSettings(scope); - - String value = new SqlSelector(scope, "SELECT setting FROM pg_settings WHERE name = 'max_identifier_length'").getObject(String.class); - try - { - _maxIdentifierByteLength = Integer.valueOf(value); - } - catch (NumberFormatException e) - { - LOG.error("Couldn't parse max_identifier_length; continuing with default value of {}", _maxIdentifierByteLength, e); - } - } - } - - @Override - protected DialectStringHandler createStringHandler() - { - // TODO: Isn't this the wrong setting? Should we be looking at the "backslash_quote" setting instead? - if (getStandardConformingStrings()) - return super.createStringHandler(); - else - return new PostgreSqlNonConformingStringHandler(); - } - - /* - PostgreSQL example connection URLs we need to parse: - - jdbc:postgresql:database - jdbc:postgresql://host/database - jdbc:postgresql://host:port/database - jdbc:postgresql:database?user=fred&password=secret&ssl=true - jdbc:postgresql://host/database?user=fred&password=secret&ssl=true - jdbc:postgresql://host:port/database?user=fred&password=secret&ssl=true - */ - @Override - public JdbcHelper getJdbcHelper() - { - return new StandardJdbcHelper(PostgreSqlDialectFactory.JDBC_PREFIX); - } - - @Override - public String getDefaultDatabaseName() - { - return "template1"; - } - - @Override - public boolean canExecuteUpgradeScripts() - { - return true; - } - - @Override - public Collection getScriptWarnings(String name, String sql) - { - // Strip out all block- and single-line comments - Pattern commentPattern = Pattern.compile(ScriptReorderer.COMMENT_REGEX, Pattern.DOTALL + Pattern.MULTILINE); - Matcher matcher = commentPattern.matcher(sql); - String noComments = matcher.replaceAll(""); - - List warnings = new LinkedList<>(); - - // Split statements by semicolon and CRLF - for (String statement : noComments.split(";[\\n\\r]+")) - { - if (Strings.CI.startsWith(statement.trim(), "SET ")) - warnings.add(statement); - } - - return warnings; - } - - @Override - public String getSQLScriptPath() - { - return "postgresql"; - } - - @Override - public String getUniqueIdentType() - { - return "SERIAL"; - } - - @Override - public boolean supportsGroupConcat() - { - return getServerType().supportsGroupConcat(); - } - - @Override - public boolean supportsSelectConcat() - { - return true; - } - - @Override - public SQLFragment getSelectConcat(SQLFragment selectSql, String delimiter) - { - SQLFragment result = new SQLFragment("array_to_string(array("); - result.append(selectSql); - result.append("), "); - result.append(getStringHandler().quoteStringLiteral(delimiter)); - result.append(")"); - return result; - } - - // Does this datasource include our sort array function? The LabKey datasource should always have it, but external datasources might not - private void determineIfArraySortFunctionExists(DbScope scope) - { - if (getServerType().supportsSpecialMetadataQueries()) - { - Selector selector = new SqlSelector(scope, "SELECT * FROM pg_catalog.pg_namespace n INNER JOIN pg_catalog.pg_proc p ON pronamespace = n.oid WHERE nspname = 'core' AND proname = 'sort'"); - _arraySortFunctionExists.set(selector.exists()); - } - - // Array sort function should always exist in LabKey scope (for now) - assert !scope.isLabKeyScope() || _arraySortFunctionExists.get(); - } - - @Override - public SQLFragment getGroupConcat(SQLFragment sql, boolean distinct, boolean sorted, @NotNull SQLFragment delimiterSQL, boolean includeNulls) - { - // Sort function might not exist in external datasource; skip that syntax if not - boolean useSortFunction = sorted && _arraySortFunctionExists.get(); - SQLFragment result = new SQLFragment(); - - if (useSortFunction) - { - result.append("array_to_string("); - result.append("core.sort("); // TODO: Switch to use ORDER BY option inside array aggregate instead of our custom function - result.append("array_agg("); - if (distinct) - { - result.append("DISTINCT "); - } - - if (includeNulls) - { - result.append("COALESCE(CAST("); - result.append(sql); - result.append(" AS VARCHAR), '')"); - } - else - { - result.append(sql); - } - - result.append(")"); // array_agg - result.append(")"); // core.sort - } - else - { - result.append("string_agg("); - if (distinct) - { - result.append("DISTINCT "); - } - - if (includeNulls) - { - result.append("COALESCE("); - result.append(sql); - result.append("::text, '')"); - } - else - { - result.append(sql); - result.append("::text"); - } - } - - result.append(", "); - result.append(delimiterSQL); - result.append(")"); // array_to_string | string_agg - - return result; - } - - @Override - public SQLFragment getAnalyzeCommandForTable(String tableName) - { - return new SQLFragment("ANALYZE ").appendIdentifier(tableName); - } - - private void initializeInClauseGenerator(DbScope scope) - { - _inClauseGenerator = getJdbcVersion(scope) >= 4 ? new ArrayParameterInClauseGenerator(scope) : new ParameterMarkerInClauseGenerator(); - } - - @Override - public InClauseGenerator getDefaultInClauseGenerator() - { - return _inClauseGenerator; - } - - @Override - public TempTableInClauseGenerator getTempTableInClauseGenerator() - { - return _tempTableInClauseGenerator; - } - - @Override - public void addAdminWarningMessages(Warnings warnings, boolean showAllWarnings) - { - super.addAdminWarningMessages(warnings, showAllWarnings); - if (showAllWarnings) - warnings.add(HtmlString.of(PostgreSqlDialectFactory.getStandardWarningMessage("has not been tested against", getMajorVersion() + ".x"))); - } - - private int getIdentifierMaxByteLength() - { - return _maxIdentifierByteLength; - } - - @Override - public boolean isIdentifierTooLong(String identifier) - { - return identifier.getBytes(StandardCharsets.UTF_8).length > getIdentifierMaxByteLength(); - } - - @Override - public String truncateAndJoin(String... parts) - { - String ret = String.join("$", parts); - - if (isIdentifierTooLong(ret)) - { - int maxBytes = getIdentifierMaxByteLength(); - StringBuilder sb = new StringBuilder(maxBytes); - int partsLength = parts.length; - int remainingBytes = maxBytes - partsLength + 1; // Make room for dollar signs - for (int i = 0; i < partsLength; i++) - { - String truncated = truncateBytes(parts[i], remainingBytes / (partsLength - i)); - if (i > 0) - sb.append("$"); - sb.append(truncated); - remainingBytes -= truncated.getBytes(StandardCharsets.UTF_8).length; - } - ret = sb.toString(); - assert ret.getBytes(StandardCharsets.UTF_8).length <= maxBytes; - } - - return ret; - } - - @Override - public String truncate(String str, int reserved) - { - return truncateBytes(str, getIdentifierMaxByteLength() - reserved); - } - - // Truncates based on UTF-8 bytes - private static String truncateBytes(String str, int maxBytes) - { - if (maxBytes < 13) - throw new IllegalStateException("maxBytes for legal name too small: " + maxBytes); - int len = str.getBytes(StandardCharsets.UTF_8).length; - if (len > maxBytes) - { - String prefix = generateIdentifierPrefix(str); - str = prefix + StringUtilsLabKey.rightUtf8Bytes(str, maxBytes - prefix.getBytes(StandardCharsets.UTF_8).length); - } - assert str.getBytes(StandardCharsets.UTF_8).length <= maxBytes; - assert !StringUtilsLabKey.hasBrokenSurrogate(str); - return str; - } - - @Override - public boolean canShowExecutionPlan(ExecutionPlanType type) - { - return true; - } - - @Override - protected Collection getQueryExecutionPlan(Connection conn, DbScope scope, SQLFragment sql, ExecutionPlanType type) - { - SQLFragment copy = new SQLFragment(sql); - copy.insert(0, type == ExecutionPlanType.Estimated ? "EXPLAIN " : "EXPLAIN ANALYZE "); - - return new SqlSelector(scope, conn, copy).getCollection(String.class); - } - - @Override - // No need to split up PostgreSQL scripts; execute all statements in a single block (unless we have a special stored proc call). - protected Pattern getSQLScriptSplitPattern() - { - return null; - } - - private static final Pattern PROC_PATTERN = Pattern.compile("^\\s*SELECT\\s+core\\.(executeJava(?:Upgrade|Initialization)Code\\s*\\(\\s*'(.+)'\\s*\\))\\s*;\\s*$", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE); - - @NotNull - @Override - protected Pattern getSQLScriptProcPattern() - { - return PROC_PATTERN; - } - - @Override - protected void checkSqlScript(String lowerNoComments, String lowerNoCommentsNoWhiteSpace, Collection errors) - { - if (lowerNoCommentsNoWhiteSpace.contains("setsearch_pathto")) - errors.add("Do not use \"SET search_path TO \". Instead, schema-qualify references to all objects."); - - if (!lowerNoCommentsNoWhiteSpace.endsWith(";")) - errors.add("Script must end with a semicolon"); - } - - @Override - public @NotNull Collection getAutoIncrementSequences(TableInfo table) - { - SQLFragment sql = new SQLFragment(""" - SELECT SchemaName, TableName, ColumnName, LastValue FROM ( - SELECT - s.relname AS SequenceName, -- Not used - tns.nspname AS SchemaName, - t.relname AS TableName, - a.attname AS ColumnName, - seq.last_value AS LastValue, - sns.nspname AS SequenceSchema -- Not used. In theory, sequence could live in a different schema, but not our practice - FROM - pg_depend d - JOIN - pg_class s ON d.objid = s.oid -- The sequence - JOIN - pg_namespace sns ON s.relnamespace = sns.oid - JOIN - pg_class t ON d.refobjid = t.oid -- The table - JOIN - pg_namespace tns ON t.relnamespace = tns.oid - JOIN - pg_attribute a ON d.refobjid = a.attrelid AND d.refobjsubid = a.attnum - JOIN - pg_sequences seq ON s.relname = seq.SequenceName AND tns.nspname = seq.SchemaName -- maybe sns.nspname instead? but that is slower... - WHERE - s.relkind = 'S' -- Sequence - AND t.relkind IN ('r', 'P') -- Table (regular table or partitioned table) - AND d.deptype IN ('a', 'i') -- Automatic dependency for DEFAULT or index-related for PK - ) AS x - WHERE SchemaName ILIKE ? AND TableName ILIKE ? - """, - table.getSchema().getName(), - table.getName() - ); - return new SqlSelector(table.getSchema(), sql).getCollection(Sequence.class); - } - - @Override - public String getBinaryDataType() - { - return "BYTEA"; - } - - @Override - public String getGlobalTempTablePrefix() - { - return DbSchema.TEMP_SCHEMA_NAME + "."; - } - - @Override - public String getDropIndexCommand(String tableName, String indexName) - { - return "DROP INDEX " + indexName; - } - - @Override - public String getCreateDatabaseSql(String dbName) - { - // This will handle both mixed case and special characters on PostgreSQL - var legal = makeIdentifierFromMetaDataName(dbName); - return new SQLFragment("CREATE DATABASE ").appendIdentifier(legal).append(" WITH ENCODING 'UTF8'").getRawSQL(); - } - - @Override - public String getCreateSchemaSql(String schemaName) - { - if (!isLegalName(schemaName) || isReserved(schemaName)) - throw new IllegalArgumentException("Not a legal schema name: " + schemaName); - - //Quoted schema names are bad news - return "CREATE SCHEMA " + schemaName; - } - - @Override - public String getTruncateSql(String tableName) - { - // To be consistent with MS SQL server, always restart the sequence. Note that the default for postgres - // is to continue the sequence but we don't have this option with MS SQL Server - return "TRUNCATE TABLE " + tableName + " RESTART IDENTITY"; - } - - @Override - public List getChangeStatements(TableChange change) - { - List result = new ArrayList<>(); - switch (change.getType()) - { - case CreateTable -> result.addAll(getCreateTableStatements(change)); - case DropTable -> { - SQLFragment f = new SQLFragment("DROP TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - result.add(f); - } - case AddColumns -> result.addAll(getAddColumnsStatements(change)); - case DropColumns -> result.add(getDropColumnsStatement(change)); - case RenameColumns -> result.addAll(getRenameColumnsStatement(change)); - case DropIndicesByName -> result.addAll(getDropIndexByNameStatements(change)); - case AddIndices -> result.addAll(getCreateIndexStatements(change)); - case ResizeColumns, ChangeColumnTypes -> result.addAll(getChangeColumnTypeStatement(change)); - case DropConstraints -> result.addAll(getDropConstraintsStatement(change)); - case AddConstraints -> result.addAll(getAddConstraintsStatement(change)); - default -> throw new IllegalArgumentException("Unsupported change type: " + change.getType()); - } - - return result; - } - - private Collection getDropIndexByNameStatements(TableChange change) - { - List statements = new ArrayList<>(); - for (String indexName : change.getIndicesToBeDroppedByName()) - { - statements.add(getDropIndexCommand(change, indexName)); - } - return statements; - } - - private SQLFragment getDropIndexCommand(TableChange change, String indexName) - { - SQLFragment f = new SQLFragment("DROP INDEX "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(indexName); - return f; - } - - /** - * We've historically created lower-cased column names in provisioned tables in Postgres. Keep doing that - * for consistency, though ideally we'd stop doing this and update all existing provisioned tables. - */ - private DatabaseIdentifier makePropertyIdentifier(String name) - { - if (isIdentifierTooLong(name)) - throw new UnsupportedOperationException("Name is too long: " + name); - return new _DatabaseIdentifier(name, quoteIdentifier(name.toLowerCase()), this); - } - - /** - * Generate the Alter Table statement to change the size or type of the column - *

- * NOTE: expects data size check to be done prior, - * will throw a SQL exception if not able to change size due to existing data - */ - private List getChangeColumnTypeStatement(TableChange change) - { - List statements = new ArrayList<>(); - - // Postgres allows executing multiple ALTER COLUMN statements under one ALTER TABLE - List nonDateTimeClauses = new ArrayList<>(); - - for (PropertyStorageSpec column : change.getColumns()) - { - DatabaseIdentifier columnIdent = makePropertyIdentifier(column.getName()); - if (column.getJdbcType().isDateOrTime()) - { - String tempColumnName = column.getName() + "~~temp~~"; - DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); - - // 1) ADD temp column - SQLFragment addTemp = new SQLFragment("ALTER TABLE "); - addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); - statements.add(addTemp); - - // 2) UPDATE: copy casted value to temp column - SQLFragment update = new SQLFragment("UPDATE "); - update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - update.append(" SET ").appendIdentifier(tempColumnIdent); - update.append(" = CAST(").appendIdentifier(columnIdent).append(" AS ").append(getSqlTypeName(column)).append(")"); - statements.add(update); - - // 3) DROP original column - SQLFragment drop = new SQLFragment("ALTER TABLE "); - drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); - statements.add(drop); - - // 4) RENAME temp column to original column name - SQLFragment rename = new SQLFragment("ALTER TABLE "); - rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); - statements.add(rename); - } - else - { - String dbType; - if (column.getJdbcType().isText()) - { - // Using the common default max size to make type change to text - dbType = column.getSize() == -1 || column.getSize() > SqlDialect.MAX_VARCHAR_SIZE ? - getSqlTypeName(JdbcType.LONGVARCHAR) : - getSqlTypeName(column.getJdbcType()) + "(" + column.getSize().toString() + ")"; - } - else if (column.getJdbcType().isDecimal()) - { - dbType = getSqlTypeName(column.getJdbcType()) + DEFAULT_DECIMAL_SCALE_PRECISION; - } - else - { - dbType = getSqlTypeName(column.getJdbcType()); - } - - SQLFragment clause = new SQLFragment(); - clause.append("ALTER COLUMN ").appendIdentifier(columnIdent).append(" TYPE ").append(dbType); - nonDateTimeClauses.add(clause); - } - } - - if (!nonDateTimeClauses.isEmpty()) - { - SQLFragment alter = new SQLFragment("ALTER TABLE "); - alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - alter.append(" "); - String sep = ""; - for (SQLFragment c : nonDateTimeClauses) - { - alter.append(sep).append(c); - sep = ", "; - } - statements.add(alter); - } - - return statements; - } - - private List getRenameColumnsStatement(TableChange change) - { - List statements = new ArrayList<>(); - for (Map.Entry oldToNew : change.getColumnRenames().entrySet()) - { - DatabaseIdentifier oldIdentifier = makePropertyIdentifier(oldToNew.getKey()); - DatabaseIdentifier newIdentifier = makePropertyIdentifier(oldToNew.getValue()); - if (!oldIdentifier.equals(newIdentifier)) - { - SQLFragment f = new SQLFragment("ALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" RENAME COLUMN ").appendIdentifier(oldIdentifier).append(" TO ").appendIdentifier(newIdentifier); - statements.add(f); - } - } - - // TODO: This loop should not guess the name of the old indices; instead, it should look them up. - // TableChange.setIndexedColumns() could set _indexRenames providing the name, and then this code uses that info. - // Or maybe schemaTableInfo.getAllIndices() and then use Index.isSameIndex() to find names. Issue 53838. - for (Map.Entry oldToNew : change.getIndexRenames().entrySet()) - { - PropertyStorageSpec.Index oldIndex = oldToNew.getKey(); - PropertyStorageSpec.Index newIndex = oldToNew.getValue(); - String oldName = nameIndex(change.getTableName(), oldIndex.columnNames); // TODO: Look up name - String newName = nameIndex(change.getTableName(), newIndex.columnNames); - if (!oldName.equals(newName)) - { - SQLFragment f = new SQLFragment("ALTER INDEX "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(oldName); - f.append(" RENAME TO ").appendIdentifier(newName); - statements.add(f); - } - } - - return statements; - } - - private SQLFragment getDropColumnsStatement(TableChange change) - { - List sqlParts = new ArrayList<>(); - for (PropertyStorageSpec prop : change.getColumns()) - { - SQLFragment sql = new SQLFragment("DROP COLUMN "); - if (prop.getExactName()) - { - sql.append(quoteIdentifier(prop.getName())); - } - else - { - sql.appendIdentifier(makePropertyIdentifier(prop.getName())); - } - sqlParts.add(sql); - } - - SQLFragment f = new SQLFragment("ALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" ").append(sqlParts, ", "); - return f; - } - - // TODO if there are cases where user-defined columns need indices, this method will need to support - // creating indices like getCreateTableStatement does. - private List getAddColumnsStatements(TableChange change) - { - List statements = new ArrayList<>(); - String pkColumn = null; - Constraint constraint = null; - - List columnSpecs = new ArrayList<>(); - for (PropertyStorageSpec prop : change.getColumns()) - { - columnSpecs.add(getSqlColumnSpec(prop)); - if (prop.isPrimaryKey()) - { - assert null == pkColumn : "no more than one primary key defined"; - pkColumn = prop.getName(); - constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); - } - } - - SQLFragment alter = new SQLFragment("ALTER TABLE "); - alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - alter.append(" "); - String sep = ""; - for (SQLFragment col : columnSpecs) - { - alter.append(sep); - alter.append("ADD COLUMN "); - alter.append(col); - sep = ", "; - } - statements.add(alter); - if (null != pkColumn) - { - SQLFragment addPk = new SQLFragment("ALTER TABLE "); - addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) - .append(" ").append(constraint.getType().toString()).append(" (") - .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); - statements.add(addPk); - } - - return statements; - } - - private List getDropConstraintsStatement(TableChange change) - { - return change.getConstraints().stream().map(constraint -> { - SQLFragment f = new SQLFragment("ALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" DROP CONSTRAINT ").appendIdentifier(constraint.getName()); - return f; - }).collect(Collectors.toList()); - } - - private List getAddConstraintsStatement(TableChange change) - { - List statements = new ArrayList<>(); - Collection constraints = change.getConstraints(); - - if (null!=constraints && !constraints.isEmpty()) - { - statements = constraints.stream().map(constraint -> { - List columns = new ArrayList<>(); - for (String col : constraint.getColumns()) - { - columns.add(new SQLFragment().appendIdentifier(col)); - } - - SQLFragment f = new SQLFragment(); - f.append("DO $$\nBEGIN\nIF NOT EXISTS\n(SELECT 1 FROM information_schema.constraint_column_usage\nWHERE table_name = ") - .appendStringLiteral(change.getSchemaName() + "." + change.getTableName(), this) - .append(" and constraint_name = ") - .appendStringLiteral(constraint.getName(), this) - .append(") THEN\nALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()).append(" ") - .append(constraint.getType().toString()).append(" (") - .append(columns, ",") - .append(")").appendEOS().append("\nEND IF)").appendEOS().append("\nEND$$").appendEOS(); - return f; - }).collect(Collectors.toList()); - } - - return statements; - } - - private List getCreateTableStatements(TableChange change) - { - List statements = new ArrayList<>(); - List createTableSqlParts = new ArrayList<>(); - String pkColumn = null; - for (PropertyStorageSpec prop : change.getColumns()) - { - createTableSqlParts.add(getSqlColumnSpec(prop)); - if (prop.isPrimaryKey()) - { - assert null == pkColumn : "no more than one primary key defined"; - pkColumn = prop.getName(); - } - } - - for (PropertyStorageSpec.ForeignKey foreignKey : change.getForeignKeys()) - { - DbSchema schema = DbSchema.get(foreignKey.getSchemaName(), DbSchemaType.Module); - TableInfo tableInfo = foreignKey.isProvisioned() ? - foreignKey.getTableInfoProvisioned() : - schema.getTable(foreignKey.getTableName()); - String constraintName = "fk_" + foreignKey.getColumnName() + "_" + change.getTableName() + "_" + tableInfo.getName(); - SQLFragment fkFrag = new SQLFragment("CONSTRAINT "); - fkFrag.appendIdentifier(constraintName) - .append(" FOREIGN KEY (") - .appendIdentifier(makePropertyIdentifier(foreignKey.getColumnName())) - .append(") REFERENCES ") - .appendIdentifier(tableInfo.getSchema().getName()).append(".").appendIdentifier(tableInfo.getName()) - .append(" (") - .appendIdentifier(makePropertyIdentifier(foreignKey.getForeignColumnName())) - .append(")"); - createTableSqlParts.add(fkFrag); - } - - SQLFragment create = new SQLFragment("CREATE TABLE "); - create.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - create.append(" (").append(createTableSqlParts, ", ").append(")"); - statements.add(create); - if (null != pkColumn) - { - // Making this just for consistent naming - Constraint constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); - - SQLFragment addPk = new SQLFragment("ALTER TABLE "); - addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) - .append(" ").append(constraint.getType().toString()).append(" (") - .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); - statements.add(addPk); - } - - statements.addAll(getCreateIndexStatements(change)); - statements.addAll(getAddConstraintsStatement(change)); - return statements; - } - - private List getCreateIndexStatements(TableChange change) - { - List statements = new ArrayList<>(); - for (PropertyStorageSpec.Index index : change.getIndexedColumns()) - { - String newIndexName = nameIndex(change.getTableName(), index.columnNames); - SQLFragment f = new SQLFragment("CREATE "); - if (index.isUnique) - f.append("UNIQUE "); - f.append("INDEX ").appendIdentifier(newIndexName).append(" ON "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" ("); - String separator = ""; - for (String columnName : index.columnNames) - { - f.append(separator).appendIdentifier(makePropertyIdentifier(columnName)); - separator = ", "; - } - f.append(")"); - f.appendEOS(); - statements.add(f); - - if (index.isClustered) - { - SQLFragment c = new SQLFragment(); - c.append(PropertyStorageSpec.CLUSTER_TYPE.CLUSTER.toString()).append(" "); - c.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - c.append(" USING ").appendIdentifier(newIndexName); - statements.add(c); - } - } - return statements; - } - - @Override - public String nameIndex(String tableName, String[] indexedColumns) - { - return AliasManager.makeLegalName(tableName + '_' + StringUtils.join(indexedColumns, "_"), this); - } - - private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop) - { - return getSqlColumnSpec(prop, prop.getName()); - } - - private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop, String columnName) - { - SQLFragment colSpec = new SQLFragment(); - colSpec.appendIdentifier(makePropertyIdentifier(columnName)).append(" "); - colSpec.append(getSqlTypeName(prop)); - - // Apply size and precision to varchar and Decimal types - if (prop.getJdbcType() == JdbcType.VARCHAR && prop.getSize() != -1 && prop.getSize() <= SqlDialect.MAX_VARCHAR_SIZE) - { - colSpec.append("(").append(prop.getSize().toString()).append(")"); - } - else if (prop.getJdbcType() == JdbcType.DECIMAL) - { - colSpec.append(DEFAULT_DECIMAL_SCALE_PRECISION); - } - - if (prop.isPrimaryKey() || !prop.isNullable()) - colSpec.append(" NOT NULL"); - - if (null != prop.getDefaultValue()) - { - if (prop.getJdbcType() == JdbcType.BOOLEAN) - { - colSpec.append(" DEFAULT "); - colSpec.append((Boolean)prop.getDefaultValue() ? getBooleanTRUE() : getBooleanFALSE()); - } - else if (prop.getJdbcType() == JdbcType.VARCHAR) - { - colSpec.append(" DEFAULT "); - colSpec.append(getStringHandler().quoteStringLiteral(prop.getDefaultValue().toString())); - } - else - { - throw new IllegalArgumentException("Default value on type " + prop.getJdbcType().name() + " is not supported."); - } - } - return colSpec; - } - - @Override - public void purgeTempSchema(Map createdTableNames) - { - try - { - trackTempTables(createdTableNames); - } - catch (SQLException e) - { - LOG.warn("error cleaning up temp schema", e); - } - - DbSchema coreSchema = CoreSchema.getInstance().getSchema(); - SqlExecutor executor = new SqlExecutor(coreSchema); - - //rs = conn.getMetaData().getFunctions(dbName, tempSchemaName, "%"); - - new SqlSelector(coreSchema, "SELECT proname AS SPECIFIC_NAME, CAST(proargtypes AS VARCHAR) FROM pg_proc WHERE pronamespace=(select oid from pg_namespace where nspname = ?)", DbSchema.getTemp().getName()).forEach( - new ForEachBlock<>() - { - private Map _types = null; - - @Override - public void exec(ResultSet rs) throws SQLException - { - if (null == _types) - { - _types = new HashMap<>(); - new SqlSelector(coreSchema, "SELECT CAST(oid AS VARCHAR) as oid, typname, (select nspname from pg_namespace where oid = typnamespace) as nspname FROM pg_type").forEach(type -> - _types.put(type.getString(1), quoteIdentifier(type.getString(3)) + "." + quoteIdentifier(type.getString(2)))); - } - - String name = rs.getString(1); - String[] oids = StringUtils.split(rs.getString(2), ' '); - SQLFragment drop = new SQLFragment("DROP FUNCTION temp.").append(name); - drop.append("("); - String comma = ""; - for (String oid : oids) - { - drop.append(comma).append(_types.get(oid)); - comma = ","; - } - drop.append(")"); - - try - { - executor.execute(drop); - } - catch (BadSqlGrammarException x) - { - LOG.warn("could not clean up postgres function : temp." + name, x); - } - } - }); - - // TODO delete types in temp schema as well! search for "CREATE TYPE" in StatementUtils.java - } - - // - // ARRAY and SET syntax - // - - // NOTE LabKey currently does not support ARRAY[VARCHAR], use ARRAY[text] instead - // - // Postgres string literals can be auto-cast to both VARCHAR and TEXT. These all work - // 'color' = 'color'::varchar - // 'color' = 'color'::text - // ARRAY['color'] = ARRAY['color'::text]; - // However, ARRAY[text] cannot be auto cast to ARRAY[varchar] - // ARRAY['color'] = ARRAY['color'::varchar]; -- ERROR! - // - - - @Override - public boolean supportsArrays() - { - return true; - } - - @Override - public SQLFragment array_construct(SQLFragment[] elements) - { - SQLFragment ret = new SQLFragment(); - ret.append("ARRAY["); - String separator = ""; - for (SQLFragment element : elements) - { - ret.append(separator); - ret.append(element); - separator = ", "; - } - ret.append("]"); - return ret; - } - - @Override - public SQLFragment array_all_in_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append("(").append(a).append(") <@ (").append(b).append(")"); - return ret; - } - - @Override - public SQLFragment array_some_in_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append("(").append(a).append(") && (").append(b).append(")"); - return ret; - } - - @Override - public SQLFragment array_none_in_array(SQLFragment a, SQLFragment b) - { - return new SQLFragment(" NOT (").append(array_some_in_array(a, b)).append(")"); - } - - @Override - public SQLFragment array_same_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append(array_all_in_array(a, b)).append(" AND ").append(array_all_in_array(b, a)); - return ret; - } - - @Override - public SQLFragment array_not_same_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append("NOT (").append(array_all_in_array(a, b)).append(") OR NOT (").append(array_all_in_array(b, a)).append(")"); - return ret; - } - - @Override - public SQLFragment element_in_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append("(").append(a).append(")"); - // DOCs imply that IS NOT DISTINCT FROM ANY should work, but it doesn't??? - // ret.append(" IS NOT DISTINCT FROM ANY("); - ret.append(" = ANY("); - ret.append(b); - ret.append(")"); - return ret; - } - - @Override - public SQLFragment element_not_in_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append("(").append(a).append(")"); - // DOCs imply that IS NOT DISTINCT FROM ANY should work, but it doesn't??? - // ret.append(" IS DISTINCT FROM ALL("); - ret.append(" <> ALL("); - ret.append(b); - ret.append(")"); - return ret; - } -} +/* + * Copyright (c) 2012-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.core.dialect; + +import jakarta.servlet.ServletException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Constraint; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DatabaseIdentifier; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.InClauseGenerator; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.ParameterMarkerInClauseGenerator; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.Selector; +import org.labkey.api.data.Selector.ForEachBlock; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableChange; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TempTableInClauseGenerator; +import org.labkey.api.data.TempTableTracker; +import org.labkey.api.data.dialect.BasePostgreSqlDialect; +import org.labkey.api.data.dialect.DialectStringHandler; +import org.labkey.api.data.dialect.JdbcHelper; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.data.dialect.StandardJdbcHelper; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.query.AliasManager; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.view.template.Warnings; +import org.labkey.core.admin.sql.ScriptReorderer; +import org.springframework.jdbc.BadSqlGrammarException; + +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/* + * This is the base class defining PostgreSQL-specific (i.e., not Redshift) behavior. PostgreSQL 9.2 is no longer + * supported; however, we keep this class to track changes we implemented specifically for this version. + */ +abstract class PostgreSql92Dialect extends BasePostgreSqlDialect +{ + public static final String PRODUCT_NAME = "PostgreSQL"; + + // This has been the standard PostgreSQL identifier max byte length for many years. However, this could change in + // the future plus servers can be compiled with a different limit, so we query this setting on first connection to + // each database. + private int _maxIdentifierByteLength = 63; + private InClauseGenerator _inClauseGenerator; + + private final TempTableInClauseGenerator _tempTableInClauseGenerator = new TempTableInClauseGenerator(); + private final AtomicBoolean _arraySortFunctionExists = new AtomicBoolean(false); + + @Override + public void handleCreateDatabaseException(SQLException e) throws ServletException + { + if ("55006".equals(e.getSQLState())) + { + LOG.error("You must close down pgAdmin III and all other applications accessing PostgreSQL."); + throw (new ServletException("Close down or disconnect pgAdmin III and all other applications accessing PostgreSQL", e)); + } + else + { + super.handleCreateDatabaseException(e); + } + } + + @Override + public void prepareDriver(Class driverClass) + { + // PostgreSQL driver 42.0.0 added logging via the Java Logging API (java.util.logging). This caused the driver to + // start logging SQLExceptions (such as the initial connection failure on bootstrap) to the console... harmless + // but annoying. This code suppresses the driver logging. + Logger pgjdbcLogger = LogManager.getLogManager().getLogger("org.postgresql"); + + if (null != pgjdbcLogger) + pgjdbcLogger.setLevel(Level.OFF); + } + + // Make sure that the PL/pgSQL language is enabled in the associated database. If not, throw. Since 9.0, PostgreSQL has + // shipped with PL/pgSQL enabled by default, so the check is no longer critical, but continue to verify just to be safe. + @Override + public void prepareNewLabKeyDatabase(DbScope scope) + { + if (new SqlSelector(scope, "SELECT * FROM pg_language WHERE lanname = 'plpgsql'").exists()) + return; + + String dbName = scope.getDatabaseName(); + String message = "PL/pgSQL is not enabled in the \"" + dbName + "\" database because it is not enabled in your Template1 master database."; + String advice = "Use PostgreSQL's 'createlang' command line utility to enable PL/pgSQL in the \"" + dbName + "\" database then restart Tomcat."; + + throw new ConfigurationException(message, advice); + } + + @Override + public String prepare(DbScope scope) + { + initializeInClauseGenerator(scope); + determineIfArraySortFunctionExists(scope); + return super.prepare(scope); + } + + @NotNull + @Override + protected Set getReservedWords() + { + Set words = super.getReservedWords(); + words.add("collation"); + + return words; + } + + /* + These override method implementations were migrated from PostgreSql91Dialect when that class was promoted to api: + getProductName() + createStringHandler() + getJdbcHelper() + getScriptWarnings() + initializeInClauseGenerator() + */ + + @Override + public String getProductName() + { + return PRODUCT_NAME; + } + + // Query PostgreSQL-specific settings + @Override + protected void determineSettings(DbScope scope) + { + if (getServerType().supportsSpecialMetadataQueries()) + { + super.determineSettings(scope); + + String value = new SqlSelector(scope, "SELECT setting FROM pg_settings WHERE name = 'max_identifier_length'").getObject(String.class); + try + { + _maxIdentifierByteLength = Integer.valueOf(value); + } + catch (NumberFormatException e) + { + LOG.error("Couldn't parse max_identifier_length; continuing with default value of {}", _maxIdentifierByteLength, e); + } + } + } + + @Override + protected DialectStringHandler createStringHandler() + { + // TODO: Isn't this the wrong setting? Should we be looking at the "backslash_quote" setting instead? + if (getStandardConformingStrings()) + return super.createStringHandler(); + else + return new PostgreSqlNonConformingStringHandler(); + } + + /* + PostgreSQL example connection URLs we need to parse: + + jdbc:postgresql:database + jdbc:postgresql://host/database + jdbc:postgresql://host:port/database + jdbc:postgresql:database?user=fred&password=secret&ssl=true + jdbc:postgresql://host/database?user=fred&password=secret&ssl=true + jdbc:postgresql://host:port/database?user=fred&password=secret&ssl=true + */ + @Override + public JdbcHelper getJdbcHelper() + { + return new StandardJdbcHelper(PostgreSqlDialectFactory.JDBC_PREFIX); + } + + @Override + public String getDefaultDatabaseName() + { + return "template1"; + } + + @Override + public boolean canExecuteUpgradeScripts() + { + return true; + } + + @Override + public Collection getScriptWarnings(String name, String sql) + { + // Strip out all block- and single-line comments + Pattern commentPattern = Pattern.compile(ScriptReorderer.COMMENT_REGEX, Pattern.DOTALL + Pattern.MULTILINE); + Matcher matcher = commentPattern.matcher(sql); + String noComments = matcher.replaceAll(""); + + List warnings = new LinkedList<>(); + + // Split statements by semicolon and CRLF + for (String statement : noComments.split(";[\\n\\r]+")) + { + if (Strings.CI.startsWith(statement.trim(), "SET ")) + warnings.add(statement); + } + + return warnings; + } + + @Override + public String getSQLScriptPath() + { + return "postgresql"; + } + + @Override + public String getUniqueIdentType() + { + return "SERIAL"; + } + + @Override + public boolean supportsGroupConcat() + { + return getServerType().supportsGroupConcat(); + } + + @Override + public boolean supportsSelectConcat() + { + return true; + } + + @Override + public SQLFragment getSelectConcat(SQLFragment selectSql, String delimiter) + { + SQLFragment result = new SQLFragment("array_to_string(array("); + result.append(selectSql); + result.append("), "); + result.append(getStringHandler().quoteStringLiteral(delimiter)); + result.append(")"); + return result; + } + + // Does this datasource include our sort array function? The LabKey datasource should always have it, but external datasources might not + private void determineIfArraySortFunctionExists(DbScope scope) + { + if (getServerType().supportsSpecialMetadataQueries()) + { + Selector selector = new SqlSelector(scope, "SELECT * FROM pg_catalog.pg_namespace n INNER JOIN pg_catalog.pg_proc p ON pronamespace = n.oid WHERE nspname = 'core' AND proname = 'sort'"); + _arraySortFunctionExists.set(selector.exists()); + } + + // Array sort function should always exist in LabKey scope (for now) + assert !scope.isLabKeyScope() || _arraySortFunctionExists.get(); + } + + @Override + public SQLFragment getGroupConcat(SQLFragment sql, boolean distinct, boolean sorted, @NotNull SQLFragment delimiterSQL, boolean includeNulls) + { + // Sort function might not exist in external datasource; skip that syntax if not + boolean useSortFunction = sorted && _arraySortFunctionExists.get(); + SQLFragment result = new SQLFragment(); + + if (useSortFunction) + { + result.append("array_to_string("); + result.append("core.sort("); // TODO: Switch to use ORDER BY option inside array aggregate instead of our custom function + result.append("array_agg("); + if (distinct) + { + result.append("DISTINCT "); + } + + if (includeNulls) + { + result.append("COALESCE(CAST("); + result.append(sql); + result.append(" AS VARCHAR), '')"); + } + else + { + result.append(sql); + } + + result.append(")"); // array_agg + result.append(")"); // core.sort + } + else + { + result.append("string_agg("); + if (distinct) + { + result.append("DISTINCT "); + } + + if (includeNulls) + { + result.append("COALESCE("); + result.append(sql); + result.append("::text, '')"); + } + else + { + result.append(sql); + result.append("::text"); + } + } + + result.append(", "); + result.append(delimiterSQL); + result.append(")"); // array_to_string | string_agg + + return result; + } + + @Override + public SQLFragment getAnalyzeCommandForTable(String tableName) + { + return new SQLFragment("ANALYZE ").appendIdentifier(tableName); + } + + private void initializeInClauseGenerator(DbScope scope) + { + _inClauseGenerator = getJdbcVersion(scope) >= 4 ? new ArrayParameterInClauseGenerator(scope) : new ParameterMarkerInClauseGenerator(); + } + + @Override + public InClauseGenerator getDefaultInClauseGenerator() + { + return _inClauseGenerator; + } + + @Override + public TempTableInClauseGenerator getTempTableInClauseGenerator() + { + return _tempTableInClauseGenerator; + } + + @Override + public void addAdminWarningMessages(Warnings warnings, boolean showAllWarnings) + { + super.addAdminWarningMessages(warnings, showAllWarnings); + if (showAllWarnings) + warnings.add(HtmlString.of(PostgreSqlDialectFactory.getStandardWarningMessage("has not been tested against", getMajorVersion() + ".x"))); + } + + private int getIdentifierMaxByteLength() + { + return _maxIdentifierByteLength; + } + + @Override + public boolean isIdentifierTooLong(String identifier) + { + return identifier.getBytes(StandardCharsets.UTF_8).length > getIdentifierMaxByteLength(); + } + + @Override + public String truncateAndJoin(String... parts) + { + String ret = String.join("$", parts); + + if (isIdentifierTooLong(ret)) + { + int maxBytes = getIdentifierMaxByteLength(); + StringBuilder sb = new StringBuilder(maxBytes); + int partsLength = parts.length; + int remainingBytes = maxBytes - partsLength + 1; // Make room for dollar signs + for (int i = 0; i < partsLength; i++) + { + String truncated = truncateBytes(parts[i], remainingBytes / (partsLength - i)); + if (i > 0) + sb.append("$"); + sb.append(truncated); + remainingBytes -= truncated.getBytes(StandardCharsets.UTF_8).length; + } + ret = sb.toString(); + assert ret.getBytes(StandardCharsets.UTF_8).length <= maxBytes; + } + + return ret; + } + + @Override + public String truncate(String str, int reserved) + { + return truncateBytes(str, getIdentifierMaxByteLength() - reserved); + } + + // Truncates based on UTF-8 bytes + private static String truncateBytes(String str, int maxBytes) + { + if (maxBytes < 13) + throw new IllegalStateException("maxBytes for legal name too small: " + maxBytes); + int len = str.getBytes(StandardCharsets.UTF_8).length; + if (len > maxBytes) + { + String prefix = generateIdentifierPrefix(str); + str = prefix + StringUtilsLabKey.rightUtf8Bytes(str, maxBytes - prefix.getBytes(StandardCharsets.UTF_8).length); + } + assert str.getBytes(StandardCharsets.UTF_8).length <= maxBytes; + assert !StringUtilsLabKey.hasBrokenSurrogate(str); + return str; + } + + @Override + public boolean canShowExecutionPlan(ExecutionPlanType type) + { + return true; + } + + @Override + protected Collection getQueryExecutionPlan(Connection conn, DbScope scope, SQLFragment sql, ExecutionPlanType type) + { + SQLFragment copy = new SQLFragment(sql); + copy.insert(0, type == ExecutionPlanType.Estimated ? "EXPLAIN " : "EXPLAIN ANALYZE "); + + return new SqlSelector(scope, conn, copy).getCollection(String.class); + } + + @Override + // No need to split up PostgreSQL scripts; execute all statements in a single block (unless we have a special stored proc call). + protected Pattern getSQLScriptSplitPattern() + { + return null; + } + + private static final Pattern PROC_PATTERN = Pattern.compile("^\\s*SELECT\\s+core\\.(executeJava(?:Upgrade|Initialization)Code\\s*\\(\\s*'(.+)'\\s*\\))\\s*;\\s*$", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE); + + @NotNull + @Override + protected Pattern getSQLScriptProcPattern() + { + return PROC_PATTERN; + } + + @Override + protected void checkSqlScript(String lowerNoComments, String lowerNoCommentsNoWhiteSpace, Collection errors) + { + if (lowerNoCommentsNoWhiteSpace.contains("setsearch_pathto")) + errors.add("Do not use \"SET search_path TO \". Instead, schema-qualify references to all objects."); + + if (!lowerNoCommentsNoWhiteSpace.endsWith(";")) + errors.add("Script must end with a semicolon"); + } + + @Override + public @NotNull Collection getAutoIncrementSequences(TableInfo table) + { + SQLFragment sql = new SQLFragment(""" + SELECT SchemaName, TableName, ColumnName, LastValue FROM ( + SELECT + s.relname AS SequenceName, -- Not used + tns.nspname AS SchemaName, + t.relname AS TableName, + a.attname AS ColumnName, + seq.last_value AS LastValue, + sns.nspname AS SequenceSchema -- Not used. In theory, sequence could live in a different schema, but not our practice + FROM + pg_depend d + JOIN + pg_class s ON d.objid = s.oid -- The sequence + JOIN + pg_namespace sns ON s.relnamespace = sns.oid + JOIN + pg_class t ON d.refobjid = t.oid -- The table + JOIN + pg_namespace tns ON t.relnamespace = tns.oid + JOIN + pg_attribute a ON d.refobjid = a.attrelid AND d.refobjsubid = a.attnum + JOIN + pg_sequences seq ON s.relname = seq.SequenceName AND tns.nspname = seq.SchemaName -- maybe sns.nspname instead? but that is slower... + WHERE + s.relkind = 'S' -- Sequence + AND t.relkind IN ('r', 'P') -- Table (regular table or partitioned table) + AND d.deptype IN ('a', 'i') -- Automatic dependency for DEFAULT or index-related for PK + ) AS x + WHERE SchemaName ILIKE ? AND TableName ILIKE ? + """, + table.getSchema().getName(), + table.getName() + ); + return new SqlSelector(table.getSchema(), sql).getCollection(Sequence.class); + } + + @Override + public String getBinaryDataType() + { + return "BYTEA"; + } + + @Override + public String getGlobalTempTablePrefix() + { + return DbSchema.TEMP_SCHEMA_NAME + "."; + } + + @Override + public String getDropIndexCommand(String tableName, String indexName) + { + return "DROP INDEX " + indexName; + } + + @Override + public String getCreateDatabaseSql(String dbName) + { + // This will handle both mixed case and special characters on PostgreSQL + var legal = makeIdentifierFromMetaDataName(dbName); + return new SQLFragment("CREATE DATABASE ").appendIdentifier(legal).append(" WITH ENCODING 'UTF8'").getRawSQL(); + } + + @Override + public String getCreateSchemaSql(String schemaName) + { + if (!isLegalName(schemaName) || isReserved(schemaName)) + throw new IllegalArgumentException("Not a legal schema name: " + schemaName); + + //Quoted schema names are bad news + return "CREATE SCHEMA " + schemaName; + } + + @Override + public String getTruncateSql(String tableName) + { + // To be consistent with MS SQL server, always restart the sequence. Note that the default for postgres + // is to continue the sequence but we don't have this option with MS SQL Server + return "TRUNCATE TABLE " + tableName + " RESTART IDENTITY"; + } + + @Override + public List getChangeStatements(TableChange change) + { + List result = new ArrayList<>(); + switch (change.getType()) + { + case CreateTable -> result.addAll(getCreateTableStatements(change)); + case DropTable -> { + SQLFragment f = new SQLFragment("DROP TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + result.add(f); + } + case AddColumns -> result.addAll(getAddColumnsStatements(change)); + case DropColumns -> result.add(getDropColumnsStatement(change)); + case RenameColumns -> result.addAll(getRenameColumnsStatement(change)); + case DropIndicesByName -> result.addAll(getDropIndexByNameStatements(change)); + case AddIndices -> result.addAll(getCreateIndexStatements(change)); + case ResizeColumns, ChangeColumnTypes -> result.addAll(getChangeColumnTypeStatement(change)); + case DropConstraints -> result.addAll(getDropConstraintsStatement(change)); + case AddConstraints -> result.addAll(getAddConstraintsStatement(change)); + default -> throw new IllegalArgumentException("Unsupported change type: " + change.getType()); + } + + return result; + } + + private Collection getDropIndexByNameStatements(TableChange change) + { + List statements = new ArrayList<>(); + for (String indexName : change.getIndicesToBeDroppedByName()) + { + statements.add(getDropIndexCommand(change, indexName)); + } + return statements; + } + + private SQLFragment getDropIndexCommand(TableChange change, String indexName) + { + SQLFragment f = new SQLFragment("DROP INDEX "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(indexName); + return f; + } + + /** + * We've historically created lower-cased column names in provisioned tables in Postgres. Keep doing that + * for consistency, though ideally we'd stop doing this and update all existing provisioned tables. + */ + private DatabaseIdentifier makePropertyIdentifier(String name) + { + if (isIdentifierTooLong(name)) + throw new UnsupportedOperationException("Name is too long: " + name); + return new _DatabaseIdentifier(name, quoteIdentifier(name.toLowerCase()), this); + } + + /** + * Generate the Alter Table statement to change the size or type of the column + *

+ * NOTE: expects data size check to be done prior, + * will throw a SQL exception if not able to change size due to existing data + */ + private List getChangeColumnTypeStatement(TableChange change) + { + List statements = new ArrayList<>(); + + // Postgres allows executing multiple ALTER COLUMN statements under one ALTER TABLE + List nonDateTimeClauses = new ArrayList<>(); + + for (PropertyStorageSpec column : change.getColumns()) + { + PropertyType oldPropertyType = change.getOldPropTypes().get(column.getName()); + DatabaseIdentifier columnIdent = makePropertyIdentifier(column.getName()); + if (column.getJdbcType().isDateOrTime()) + { + String tempColumnName = column.getName() + "~~temp~~"; + DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); + + // 1) ADD temp column + SQLFragment addTemp = new SQLFragment("ALTER TABLE "); + addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); + statements.add(addTemp); + + // 2) UPDATE: copy casted value to temp column + SQLFragment update = new SQLFragment("UPDATE "); + update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + update.append(" SET ").appendIdentifier(tempColumnIdent); + update.append(" = CAST(").appendIdentifier(columnIdent).append(" AS ").append(getSqlTypeName(column)).append(")"); + statements.add(update); + + // 3) DROP original column + SQLFragment drop = new SQLFragment("ALTER TABLE "); + drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); + statements.add(drop); + + // 4) RENAME temp column to original column name + SQLFragment rename = new SQLFragment("ALTER TABLE "); + rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); + statements.add(rename); + } + else if (oldPropertyType == PropertyType.MULTI_CHOICE && column.getJdbcType().isText()) + { + // Converting from text[] (array) to text requires an intermediate column and transformation + String tempColumnName = column.getName() + "~~temp~~"; + DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); + + // 1) ADD temp column of text type + SQLFragment addTemp = new SQLFragment("ALTER TABLE "); + addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); + statements.add(addTemp); + + // 2) UPDATE: convert and copy value to temp column + // - NULL array -> NULL + // - empty array -> NULL + // - non-empty array -> concatenate array elements with comma (', ') + SQLFragment update = new SQLFragment("UPDATE "); + update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + update.append(" SET ").appendIdentifier(tempColumnIdent).append(" = CASE "); + update.append(" WHEN ").appendIdentifier(columnIdent).append(" IS NULL THEN NULL "); + update.append(" WHEN COALESCE(array_length(").appendIdentifier(columnIdent).append(", 1), 0) = 0 THEN NULL "); + update.append(" ELSE array_to_string(").appendIdentifier(columnIdent).append(", ', ') END"); + statements.add(update); + + // 3) DROP original column + SQLFragment drop = new SQLFragment("ALTER TABLE "); + drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); + statements.add(drop); + + // 4) RENAME temp column to original column name + SQLFragment rename = new SQLFragment("ALTER TABLE "); + rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); + statements.add(rename); + } + else if (column.getJdbcType() == JdbcType.ARRAY) + { + // Converting from text to text[] requires an intermediate column and transformation + String tempColumnName = column.getName() + "~~temp~~"; + DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); + + // 1) ADD temp column of array type (e.g., text[]) + SQLFragment addTemp = new SQLFragment("ALTER TABLE "); + addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); + statements.add(addTemp); + + // 2) UPDATE: copy converted value to temp column as single-element array + // - NULL or blank ('') -> empty array [] + // - otherwise -> single-element array [text] + SQLFragment update = new SQLFragment("UPDATE "); + update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + update.append(" SET ").appendIdentifier(tempColumnIdent); + update.append(" = CASE WHEN ").appendIdentifier(columnIdent).append(" IS NULL OR ").appendIdentifier(columnIdent).append(" = '' THEN ARRAY[]::text[] ELSE ARRAY["); + update.appendIdentifier(columnIdent).append("]::text[] END"); + statements.add(update); + + // 3) DROP original column + SQLFragment drop = new SQLFragment("ALTER TABLE "); + drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); + statements.add(drop); + + // 4) RENAME temp column to original column name + SQLFragment rename = new SQLFragment("ALTER TABLE "); + rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); + statements.add(rename); + } + else + { + String dbType; + if (column.getJdbcType().isText()) + { + // Using the common default max size to make type change to text + dbType = column.getSize() == -1 || column.getSize() > SqlDialect.MAX_VARCHAR_SIZE ? + getSqlTypeName(JdbcType.LONGVARCHAR) : + getSqlTypeName(column.getJdbcType()) + "(" + column.getSize().toString() + ")"; + } + else if (column.getJdbcType().isDecimal()) + { + dbType = getSqlTypeName(column.getJdbcType()) + DEFAULT_DECIMAL_SCALE_PRECISION; + } + else + { + dbType = getSqlTypeName(column.getJdbcType()); + } + + SQLFragment clause = new SQLFragment(); + clause.append("ALTER COLUMN ").appendIdentifier(columnIdent).append(" TYPE ").append(dbType); + nonDateTimeClauses.add(clause); + } + } + + if (!nonDateTimeClauses.isEmpty()) + { + SQLFragment alter = new SQLFragment("ALTER TABLE "); + alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + alter.append(" "); + String sep = ""; + for (SQLFragment c : nonDateTimeClauses) + { + alter.append(sep).append(c); + sep = ", "; + } + statements.add(alter); + } + + return statements; + } + + private List getRenameColumnsStatement(TableChange change) + { + List statements = new ArrayList<>(); + for (Map.Entry oldToNew : change.getColumnRenames().entrySet()) + { + DatabaseIdentifier oldIdentifier = makePropertyIdentifier(oldToNew.getKey()); + DatabaseIdentifier newIdentifier = makePropertyIdentifier(oldToNew.getValue()); + if (!oldIdentifier.equals(newIdentifier)) + { + SQLFragment f = new SQLFragment("ALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" RENAME COLUMN ").appendIdentifier(oldIdentifier).append(" TO ").appendIdentifier(newIdentifier); + statements.add(f); + } + } + + // TODO: This loop should not guess the name of the old indices; instead, it should look them up. + // TableChange.setIndexedColumns() could set _indexRenames providing the name, and then this code uses that info. + // Or maybe schemaTableInfo.getAllIndices() and then use Index.isSameIndex() to find names. Issue 53838. + for (Map.Entry oldToNew : change.getIndexRenames().entrySet()) + { + PropertyStorageSpec.Index oldIndex = oldToNew.getKey(); + PropertyStorageSpec.Index newIndex = oldToNew.getValue(); + String oldName = nameIndex(change.getTableName(), oldIndex.columnNames); // TODO: Look up name + String newName = nameIndex(change.getTableName(), newIndex.columnNames); + if (!oldName.equals(newName)) + { + SQLFragment f = new SQLFragment("ALTER INDEX "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(oldName); + f.append(" RENAME TO ").appendIdentifier(newName); + statements.add(f); + } + } + + return statements; + } + + private SQLFragment getDropColumnsStatement(TableChange change) + { + List sqlParts = new ArrayList<>(); + for (PropertyStorageSpec prop : change.getColumns()) + { + SQLFragment sql = new SQLFragment("DROP COLUMN "); + if (prop.getExactName()) + { + sql.append(quoteIdentifier(prop.getName())); + } + else + { + sql.appendIdentifier(makePropertyIdentifier(prop.getName())); + } + sqlParts.add(sql); + } + + SQLFragment f = new SQLFragment("ALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" ").append(sqlParts, ", "); + return f; + } + + // TODO if there are cases where user-defined columns need indices, this method will need to support + // creating indices like getCreateTableStatement does. + private List getAddColumnsStatements(TableChange change) + { + List statements = new ArrayList<>(); + String pkColumn = null; + Constraint constraint = null; + + List columnSpecs = new ArrayList<>(); + for (PropertyStorageSpec prop : change.getColumns()) + { + columnSpecs.add(getSqlColumnSpec(prop)); + if (prop.isPrimaryKey()) + { + assert null == pkColumn : "no more than one primary key defined"; + pkColumn = prop.getName(); + constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); + } + } + + SQLFragment alter = new SQLFragment("ALTER TABLE "); + alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + alter.append(" "); + String sep = ""; + for (SQLFragment col : columnSpecs) + { + alter.append(sep); + alter.append("ADD COLUMN "); + alter.append(col); + sep = ", "; + } + statements.add(alter); + if (null != pkColumn) + { + SQLFragment addPk = new SQLFragment("ALTER TABLE "); + addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) + .append(" ").append(constraint.getType().toString()).append(" (") + .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); + statements.add(addPk); + } + + return statements; + } + + private List getDropConstraintsStatement(TableChange change) + { + return change.getConstraints().stream().map(constraint -> { + SQLFragment f = new SQLFragment("ALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" DROP CONSTRAINT ").appendIdentifier(constraint.getName()); + return f; + }).collect(Collectors.toList()); + } + + private List getAddConstraintsStatement(TableChange change) + { + List statements = new ArrayList<>(); + Collection constraints = change.getConstraints(); + + if (null!=constraints && !constraints.isEmpty()) + { + statements = constraints.stream().map(constraint -> { + List columns = new ArrayList<>(); + for (String col : constraint.getColumns()) + { + columns.add(new SQLFragment().appendIdentifier(col)); + } + + SQLFragment f = new SQLFragment(); + f.append("DO $$\nBEGIN\nIF NOT EXISTS\n(SELECT 1 FROM information_schema.constraint_column_usage\nWHERE table_name = ") + .appendStringLiteral(change.getSchemaName() + "." + change.getTableName(), this) + .append(" and constraint_name = ") + .appendStringLiteral(constraint.getName(), this) + .append(") THEN\nALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()).append(" ") + .append(constraint.getType().toString()).append(" (") + .append(columns, ",") + .append(")").appendEOS().append("\nEND IF)").appendEOS().append("\nEND$$").appendEOS(); + return f; + }).collect(Collectors.toList()); + } + + return statements; + } + + private List getCreateTableStatements(TableChange change) + { + List statements = new ArrayList<>(); + List createTableSqlParts = new ArrayList<>(); + String pkColumn = null; + for (PropertyStorageSpec prop : change.getColumns()) + { + createTableSqlParts.add(getSqlColumnSpec(prop)); + if (prop.isPrimaryKey()) + { + assert null == pkColumn : "no more than one primary key defined"; + pkColumn = prop.getName(); + } + } + + for (PropertyStorageSpec.ForeignKey foreignKey : change.getForeignKeys()) + { + DbSchema schema = DbSchema.get(foreignKey.getSchemaName(), DbSchemaType.Module); + TableInfo tableInfo = foreignKey.isProvisioned() ? + foreignKey.getTableInfoProvisioned() : + schema.getTable(foreignKey.getTableName()); + String constraintName = "fk_" + foreignKey.getColumnName() + "_" + change.getTableName() + "_" + tableInfo.getName(); + SQLFragment fkFrag = new SQLFragment("CONSTRAINT "); + fkFrag.appendIdentifier(constraintName) + .append(" FOREIGN KEY (") + .appendIdentifier(makePropertyIdentifier(foreignKey.getColumnName())) + .append(") REFERENCES ") + .appendIdentifier(tableInfo.getSchema().getName()).append(".").appendIdentifier(tableInfo.getName()) + .append(" (") + .appendIdentifier(makePropertyIdentifier(foreignKey.getForeignColumnName())) + .append(")"); + createTableSqlParts.add(fkFrag); + } + + SQLFragment create = new SQLFragment("CREATE TABLE "); + create.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + create.append(" (").append(createTableSqlParts, ", ").append(")"); + statements.add(create); + if (null != pkColumn) + { + // Making this just for consistent naming + Constraint constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); + + SQLFragment addPk = new SQLFragment("ALTER TABLE "); + addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) + .append(" ").append(constraint.getType().toString()).append(" (") + .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); + statements.add(addPk); + } + + statements.addAll(getCreateIndexStatements(change)); + statements.addAll(getAddConstraintsStatement(change)); + return statements; + } + + private List getCreateIndexStatements(TableChange change) + { + List statements = new ArrayList<>(); + for (PropertyStorageSpec.Index index : change.getIndexedColumns()) + { + String newIndexName = nameIndex(change.getTableName(), index.columnNames); + SQLFragment f = new SQLFragment("CREATE "); + if (index.isUnique) + f.append("UNIQUE "); + f.append("INDEX ").appendIdentifier(newIndexName).append(" ON "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" ("); + String separator = ""; + for (String columnName : index.columnNames) + { + f.append(separator).appendIdentifier(makePropertyIdentifier(columnName)); + separator = ", "; + } + f.append(")"); + f.appendEOS(); + statements.add(f); + + if (index.isClustered) + { + SQLFragment c = new SQLFragment(); + c.append(PropertyStorageSpec.CLUSTER_TYPE.CLUSTER.toString()).append(" "); + c.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + c.append(" USING ").appendIdentifier(newIndexName); + statements.add(c); + } + } + return statements; + } + + @Override + public String nameIndex(String tableName, String[] indexedColumns) + { + return AliasManager.makeLegalName(tableName + '_' + StringUtils.join(indexedColumns, "_"), this); + } + + private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop) + { + return getSqlColumnSpec(prop, prop.getName()); + } + + private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop, String columnName) + { + SQLFragment colSpec = new SQLFragment(); + colSpec.appendIdentifier(makePropertyIdentifier(columnName)).append(" "); + colSpec.append(getSqlTypeName(prop)); + + // Apply size and precision to varchar and Decimal types + if (prop.getJdbcType() == JdbcType.VARCHAR && prop.getSize() != -1 && prop.getSize() <= SqlDialect.MAX_VARCHAR_SIZE) + { + colSpec.append("(").append(prop.getSize().toString()).append(")"); + } + else if (prop.getJdbcType() == JdbcType.DECIMAL) + { + colSpec.append(DEFAULT_DECIMAL_SCALE_PRECISION); + } + + if (prop.isPrimaryKey() || !prop.isNullable()) + colSpec.append(" NOT NULL"); + + if (null != prop.getDefaultValue()) + { + if (prop.getJdbcType() == JdbcType.BOOLEAN) + { + colSpec.append(" DEFAULT "); + colSpec.append((Boolean)prop.getDefaultValue() ? getBooleanTRUE() : getBooleanFALSE()); + } + else if (prop.getJdbcType() == JdbcType.VARCHAR) + { + colSpec.append(" DEFAULT "); + colSpec.append(getStringHandler().quoteStringLiteral(prop.getDefaultValue().toString())); + } + else + { + throw new IllegalArgumentException("Default value on type " + prop.getJdbcType().name() + " is not supported."); + } + } + return colSpec; + } + + @Override + public void purgeTempSchema(Map createdTableNames) + { + try + { + trackTempTables(createdTableNames); + } + catch (SQLException e) + { + LOG.warn("error cleaning up temp schema", e); + } + + DbSchema coreSchema = CoreSchema.getInstance().getSchema(); + SqlExecutor executor = new SqlExecutor(coreSchema); + + //rs = conn.getMetaData().getFunctions(dbName, tempSchemaName, "%"); + + new SqlSelector(coreSchema, "SELECT proname AS SPECIFIC_NAME, CAST(proargtypes AS VARCHAR) FROM pg_proc WHERE pronamespace=(select oid from pg_namespace where nspname = ?)", DbSchema.getTemp().getName()).forEach( + new ForEachBlock<>() + { + private Map _types = null; + + @Override + public void exec(ResultSet rs) throws SQLException + { + if (null == _types) + { + _types = new HashMap<>(); + new SqlSelector(coreSchema, "SELECT CAST(oid AS VARCHAR) as oid, typname, (select nspname from pg_namespace where oid = typnamespace) as nspname FROM pg_type").forEach(type -> + _types.put(type.getString(1), quoteIdentifier(type.getString(3)) + "." + quoteIdentifier(type.getString(2)))); + } + + String name = rs.getString(1); + String[] oids = StringUtils.split(rs.getString(2), ' '); + SQLFragment drop = new SQLFragment("DROP FUNCTION temp.").append(name); + drop.append("("); + String comma = ""; + for (String oid : oids) + { + drop.append(comma).append(_types.get(oid)); + comma = ","; + } + drop.append(")"); + + try + { + executor.execute(drop); + } + catch (BadSqlGrammarException x) + { + LOG.warn("could not clean up postgres function : temp." + name, x); + } + } + }); + + // TODO delete types in temp schema as well! search for "CREATE TYPE" in StatementUtils.java + } + + // + // ARRAY and SET syntax + // + + // NOTE LabKey currently does not support ARRAY[VARCHAR], use ARRAY[text] instead + // + // Postgres string literals can be auto-cast to both VARCHAR and TEXT. These all work + // 'color' = 'color'::varchar + // 'color' = 'color'::text + // ARRAY['color'] = ARRAY['color'::text]; + // However, ARRAY[text] cannot be auto cast to ARRAY[varchar] + // ARRAY['color'] = ARRAY['color'::varchar]; -- ERROR! + // + + + @Override + public boolean supportsArrays() + { + return true; + } + + @Override + public SQLFragment array_construct(SQLFragment[] elements) + { + SQLFragment ret = new SQLFragment(); + ret.append("ARRAY["); + String separator = ""; + for (SQLFragment element : elements) + { + ret.append(separator); + ret.append(element); + separator = ", "; + } + ret.append("]"); + return ret; + } + + @Override + public SQLFragment array_all_in_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append("(").append(a).append(") <@ (").append(b).append(")"); + return ret; + } + + @Override + public SQLFragment array_some_in_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append("(").append(a).append(") && (").append(b).append(")"); + return ret; + } + + @Override + public SQLFragment array_none_in_array(SQLFragment a, SQLFragment b) + { + return new SQLFragment(" NOT (").append(array_some_in_array(a, b)).append(")"); + } + + @Override + public SQLFragment array_same_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append(array_all_in_array(a, b)).append(" AND ").append(array_all_in_array(b, a)); + return ret; + } + + @Override + public SQLFragment array_not_same_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append("NOT (").append(array_all_in_array(a, b)).append(") OR NOT (").append(array_all_in_array(b, a)).append(")"); + return ret; + } + + @Override + public SQLFragment element_in_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append("(").append(a).append(")"); + // DOCs imply that IS NOT DISTINCT FROM ANY should work, but it doesn't??? + // ret.append(" IS NOT DISTINCT FROM ANY("); + ret.append(" = ANY("); + ret.append(b); + ret.append(")"); + return ret; + } + + @Override + public SQLFragment element_not_in_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append("(").append(a).append(")"); + // DOCs imply that IS NOT DISTINCT FROM ANY should work, but it doesn't??? + // ret.append(" IS DISTINCT FROM ALL("); + ret.append(" <> ALL("); + ret.append(b); + ret.append(")"); + return ret; + } +} diff --git a/experiment/package-lock.json b/experiment/package-lock.json index c3ed56aedca..7a889cd4ac9 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.13.0" + "@labkey/components": "7.14.0-fb-mvtc-convert.1" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.13.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.13.0.tgz", - "integrity": "sha512-+2o42no7q9IInKbvSd5XHDrnmLKucgudQ+7C2FD6ya+Da8mRu76GWG6L168iwbtMaguQZzFQmMGpD5VScWZiyQ==", + "version": "7.14.0-fb-mvtc-convert.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.14.0-fb-mvtc-convert.1.tgz", + "integrity": "sha512-LDwQkXH1oAsDTn0C+Ep6JvIl4sqXE2c5v+a/veezi96ltzWgXd7gOPrUubFRbLO0xXG3nn8QzGWdu0G7Zd0kWg==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index 6601245661c..52420d77e1c 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.13.0" + "@labkey/components": "7.14.0-fb-mvtc-convert.1" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/experiment/src/org/labkey/experiment/ExperimentQueryChangeListener.java b/experiment/src/org/labkey/experiment/ExperimentQueryChangeListener.java index f0a6c6d5201..177dc34ab45 100644 --- a/experiment/src/org/labkey/experiment/ExperimentQueryChangeListener.java +++ b/experiment/src/org/labkey/experiment/ExperimentQueryChangeListener.java @@ -63,7 +63,7 @@ private List getRenamedDataClasses(Container container, String } @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { boolean isSamples = schema.toString().equalsIgnoreCase("samples"); boolean isData = schema.toString().equalsIgnoreCase("exp.data"); diff --git a/experiment/src/org/labkey/experiment/PropertyQueryChangeListener.java b/experiment/src/org/labkey/experiment/PropertyQueryChangeListener.java index 65e879b53bd..e29ccc60340 100644 --- a/experiment/src/org/labkey/experiment/PropertyQueryChangeListener.java +++ b/experiment/src/org/labkey/experiment/PropertyQueryChangeListener.java @@ -61,7 +61,7 @@ private void updateLookupSchema(String newValue, String oldSchema, Container con } @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { if (!property.equals(QueryProperty.SchemaName) && !property.equals(QueryProperty.Name)) // Issue 53846 return; diff --git a/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java b/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java index 2b10c8c8718..40e036aff70 100644 --- a/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java +++ b/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java @@ -1,1381 +1,1398 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api.property; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.data.BooleanFormat; -import org.labkey.api.data.ColumnRenderPropertiesImpl; -import org.labkey.api.data.ConditionalFormat; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DatabaseIdentifier; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.PHI; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.Table; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.exp.ChangePropertyDescriptorException; -import org.labkey.api.exp.DomainDescriptor; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.IPropertyType; -import org.labkey.api.exp.property.IPropertyValidator; -import org.labkey.api.exp.property.Lookup; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.gwt.client.DefaultScaleType; -import org.labkey.api.gwt.client.DefaultValueType; -import org.labkey.api.gwt.client.FacetingBehaviorType; -import org.labkey.api.security.User; -import org.labkey.api.util.StringExpressionFactory; -import org.labkey.api.util.TestContext; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -public class DomainPropertyImpl implements DomainProperty -{ - private final DomainImpl _domain; - - PropertyDescriptor _pd; - PropertyDescriptor _pdOld; - boolean _deleted; - - private boolean _schemaChanged; - private boolean _schemaImport; - private List _validators; - private List _formats; - private String _defaultValue; - - public DomainPropertyImpl(DomainImpl type, PropertyDescriptor pd) - { - this(type, pd, null); - } - - public DomainPropertyImpl(DomainImpl type, PropertyDescriptor pd, List formats) - { - _domain = type; - _pd = pd.clone(); - _formats = formats; - } - - @Override - public int getPropertyId() - { - return _pd.getPropertyId(); - } - - @Override - public Container getContainer() - { - return _pd.getContainer(); - } - - @Override - public String getPropertyURI() - { - return _pd.getPropertyURI(); - } - - @Override - public String getName() - { - return _pd.getName(); - } - - @Override - public String getDescription() - { - return _pd.getDescription(); - } - - @Override - public String getFormat() - { - return _pd.getFormat(); - } - - @Override - public String getLabel() - { - return _pd.getLabel(); - } - - @Override - public String getConceptURI() - { - return _pd.getConceptURI(); - } - - @Override - public Domain getDomain() - { - return _domain; - } - - @Override - public IPropertyType getType() - { - return PropertyService.get().getType(getContainer(), _pd.getRangeURI()); - } - - @Override - public boolean isRequired() - { - return _pd.isRequired(); - } - - @Override - public boolean isHidden() - { - return _pd.isHidden(); - } - - @Override - public boolean isDeleted() - { - return _deleted; - } - - @Override - public boolean isShownInInsertView() - { - return _pd.isShownInInsertView(); - } - - @Override - public boolean isShownInDetailsView() - { - return _pd.isShownInDetailsView(); - } - - @Override - public boolean isShownInUpdateView() - { - return _pd.isShownInUpdateView(); - } - - @Override - public boolean isShownInLookupView() - { - return _pd.isShownInLookupView(); - } - - @Override - public boolean isMeasure() - { - return _pd.isMeasure(); - } - - @Override - public boolean isDimension() - { - return _pd.isDimension(); - } - - @Override - public boolean isRecommendedVariable() - { - return _pd.isRecommendedVariable(); - } - - @Override - public DefaultScaleType getDefaultScale() - { - return _pd.getDefaultScale(); - } - - @Override - public PHI getPHI() - { - return _pd.getPHI(); - } - - @Override - public String getRedactedText() { return _pd.getRedactedText(); } - - @Override - public boolean isExcludeFromShifting() - { - return _pd.isExcludeFromShifting(); - } - - @Override - public boolean isMvEnabled() - { - return _pd.isMvEnabled(); - } - - @Override - public boolean isMvEnabledForDrop() - { - if (null != _pdOld) - return _pdOld.isMvEnabled(); // if we need to drop/recreate we care about the old one - return _pd.isMvEnabled(); - } - - @Override - public void delete() - { - _deleted = true; - } - - @Override - public void setSchemaImport(boolean isSchemaImport) - { - // if this flag is set True then the column is dropped and recreated by its Domain if there is a type change - _schemaImport = isSchemaImport; - } - - @Override - public void setName(String name) - { - if (Strings.CS.equals(name, getName())) - return; - edit().setName(name); - } - - @Override - public void setDescription(String description) - { - if (Strings.CS.equals(description, getDescription())) - return; - edit().setDescription(description); - } - - @Override - public void setType(IPropertyType domain) - { - edit().setRangeURI(domain.getTypeURI()); - } - - @Override - public void setPropertyURI(String uri) - { - if (Strings.CS.equals(uri, getPropertyURI())) - return; - edit().setPropertyURI(uri); - } - - @Override - public void setRangeURI(String rangeURI) - { - if (Strings.CS.equals(rangeURI, getRangeURI())) - return; - editSchema().setRangeURI(rangeURI); - } - - @Override - public String getRangeURI() - { - return _pd.getRangeURI(); - } - - @Override - public void setFormat(String s) - { - if (Strings.CS.equals(s, getFormat())) - return; - edit().setFormat(s); - } - - @Override - public void setLabel(String caption) - { - if (Strings.CS.equals(caption, getLabel())) - return; - edit().setLabel(caption); - } - - @Override - public void setConceptURI(String conceptURI) - { - if (Strings.CS.equals(conceptURI, getConceptURI())) - return; - edit().setConceptURI(conceptURI); - } - - @Override - public void setRequired(boolean required) - { - if (required == isRequired()) - return; - edit().setRequired(required); - } - - @Override - public void setHidden(boolean hidden) - { - if (hidden == isHidden()) - return; - edit().setHidden(hidden); - } - - @Override - public void setShownInDetailsView(boolean shown) - { - if (shown == isShownInDetailsView()) - return; - edit().setShownInDetailsView(shown); - } - - @Override - public void setShownInInsertView(boolean shown) - { - if (shown == isShownInInsertView()) - return; - edit().setShownInInsertView(shown); - } - - @Override - public void setShownInUpdateView(boolean shown) - { - if (shown == isShownInUpdateView()) - return; - edit().setShownInUpdateView(shown); - } - - @Override - public void setShownInLookupView(boolean shown) - { - if (shown == isShownInLookupView()) - return; - edit().setShownInLookupView(shown); - } - - @Override - public void setMeasure(boolean isMeasure) - { - // UNDONE: isMeasure() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. - if (!isEdited() && isMeasure == isMeasure()) - return; - edit().setMeasure(isMeasure); - } - - @Override - public void setDimension(boolean isDimension) - { - // UNDONE: isDimension() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. - if (!isEdited() && isDimension == isDimension()) - return; - edit().setDimension(isDimension); - } - - @Override - public void setRecommendedVariable(boolean isRecommendedVariable) - { - if (!isEdited() && isRecommendedVariable == isRecommendedVariable()) - return; - edit().setRecommendedVariable(isRecommendedVariable); - } - - @Override - public void setDefaultScale(DefaultScaleType defaultScale) - { - if (!isEdited() && getDefaultScale() == defaultScale) - return; - - edit().setDefaultScale(defaultScale); - } - - @Override - public void setPhi(PHI phi) - { - if (!isEdited() && getPHI() == phi) - return; - edit().setPHI(phi); - } - - @Override - public void setRedactedText(String redactedText) - { - if (!isEdited() && ((getRedactedText() != null && getRedactedText().equals(redactedText)) - || (getRedactedText() == null && redactedText == null))) - return; - edit().setRedactedText(redactedText); - } - - @Override - public void setExcludeFromShifting(boolean isExcludeFromShifting) - { - // UNDONE: isExcludeFromShifting() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. - if (!isEdited() && isExcludeFromShifting == isExcludeFromShifting()) - return; - edit().setExcludeFromShifting(isExcludeFromShifting); - } - - @Override - public void setMvEnabled(boolean mv) - { - if (mv == isMvEnabled()) - return; - edit().setMvEnabled(mv); - } - - @Override - public void setScale(int scale) - { - if (scale == getScale()) - return; - edit().setScale(scale); - } - - /** Need the string version of this method because it's called by reflection and must match by name */ - public void setImportAliases(String aliases) - { - if (Strings.CS.equals(aliases, getImportAliases())) - return; - edit().setImportAliases(aliases); - } - - /** Need the string version of this method because it's called by reflection and must match by name */ - public String getImportAliases() - { - return _pd.getImportAliases(); - } - - @Override - public void setImportAliasSet(Set aliases) - { - String current = getImportAliases(); - String newAliases = ColumnRenderPropertiesImpl.convertToString(aliases); - if (Strings.CS.equals(current, newAliases)) - return; - edit().setImportAliasesSet(aliases); - } - - @Override - public Set getImportAliasSet() - { - return _pd.getImportAliasSet(); - } - - @Override - public void setURL(String url) - { - if (Strings.CS.equals(getURL(), url)) - return; - - if (null == url) - edit().setURL(null); - else - edit().setURL(StringExpressionFactory.createURL(url)); - } - - @Override - public String getURL() - { - return _pd.getURL() == null ? null : _pd.getURL().toString(); - } - - @Override - public void setURLTarget(String urlTarget) - { - if (Strings.CS.equals(getURLTarget(), urlTarget)) - return; - edit().setURLTarget(urlTarget); - } - - @Override - public String getURLTarget() - { - return _pd.getURLTarget(); - } - - private boolean isEdited() - { - return null != _pdOld; - } - - private PropertyDescriptor editSchema() - { - PropertyDescriptor pd = edit(); - _schemaChanged = true; - _pd.clearPropertyType(); - return pd; - } - - public boolean isRecreateRequired() - { - return _schemaChanged && _schemaImport; - } - - public void markAsNew() - { - assert isRecreateRequired() && !isNew(); - _pd.setPropertyId(0); - } - - private PropertyDescriptor edit() - { - if (_pdOld == null) - { - _pdOld = _pd; - _pd = _pdOld.clone(); - } - return _pd; - } - - @Override - public PropertyType getPropertyType() - { - return _pd.getPropertyType(); - } - - @Override - public JdbcType getJdbcType() - { - return _pd.getPropertyType().getJdbcType(); - } - - @Override - public int getScale() - { - return _pd.getScale(); - } - - @Override - public String getInputType() - { - return _pd.getPropertyType().getInputType(); - } - - @Override - public DefaultValueType getDefaultValueTypeEnum() - { - return _pd.getDefaultValueTypeEnum(); - } - - @Override - public void setDefaultValueTypeEnum(DefaultValueType defaultValueType) - { - _pd.setDefaultValueTypeEnum(defaultValueType); - } - - public String getDefaultValueType() - { - return _pd.getDefaultValueType(); - } - - @Override - public void setDefaultValueType(String defaultValueTypeName) - { - if (getDefaultValueType() != null && getDefaultValueType().equals(defaultValueTypeName)) - return; - - if (getDefaultValueType() == null && defaultValueTypeName == null) - return; // if both are null, don't call edit(), with marks property as dirty - - edit().setDefaultValueType(defaultValueTypeName); - } - - @Override - public void setDefaultValue(String value) - { - _defaultValue = value; - } - - public String getDefaultValue() - { - return _defaultValue; - } - - @Override - public Lookup getLookup() - { - return _pd.getLookup(); - } - - @Override - public void setLookup(Lookup lookup) - { - Lookup current = getLookup(); - - if (current == lookup) - return; - - // current will return null if the schema or query is null so check - // for this case in the passed in lookup - if (current == null) - if (lookup.getQueryName() == null || lookup.getSchemaKey() == null) - return; - - if (current != null && current.equals(lookup)) - return; - - if (lookup == null) - { - edit().setLookupContainer(null); - edit().setLookupSchema(null); - edit().setLookupQuery(null); - return; - } - if (lookup.getContainer() == null) - { - edit().setLookupContainer(null); - } - else - { - edit().setLookupContainer(lookup.getContainer().getId()); - } - edit().setLookupQuery(lookup.getQueryName()); - edit().setLookupSchema(Objects.toString(lookup.getSchemaKey(),null)); - } - - @Override - public void setScannable(boolean scannable) - { - if (scannable != isScannable()) - edit().setScannable(scannable); - } - - @Override - public void setOldPropertyDescriptor(PropertyDescriptor oldPropertyDescriptor) - { - if (isEdited()) - return; - - _pdOld = oldPropertyDescriptor.clone(); - } - - @Override - public boolean isScannable() - { - return _pd.isScannable(); - } - - @Override - public void setPrincipalConceptCode(String code) - { - if (!Strings.CS.equals(code, getPrincipalConceptCode())) - edit().setPrincipalConceptCode(code); - } - - @Override - public String getPrincipalConceptCode() - { - return _pd.getPrincipalConceptCode(); - } - - @Override - public String getSourceOntology() - { - return _pd.getSourceOntology(); - } - - @Override - public void setSourceOntology(String sourceOntology) - { - if (!Strings.CS.equals(sourceOntology, getSourceOntology())) - edit().setSourceOntology(sourceOntology); - } - - @Override - public String getConceptSubtree() - { - return _pd.getConceptSubtree(); - } - - @Override - public void setConceptSubtree(String path) - { - if (!Strings.CS.equals(path, getConceptSubtree())) - edit().setConceptSubtree(path); - } - - @Override - public String getConceptImportColumn() - { - return _pd.getConceptImportColumn(); - } - - @Override - public void setConceptImportColumn(String conceptImportColumn) - { - if (!Strings.CS.equals(conceptImportColumn, getConceptImportColumn())) - edit().setConceptImportColumn(conceptImportColumn); - } - - @Override - public String getConceptLabelColumn() - { - return _pd.getConceptLabelColumn(); - } - - @Override - public void setConceptLabelColumn(String conceptLabelColumn) - { - if (!Strings.CS.equals(conceptLabelColumn, getConceptLabelColumn())) - edit().setConceptLabelColumn(conceptLabelColumn); - } - - @Override - public void setDerivationDataScope(String scope) - { - if (!Strings.CS.equals(scope, getDerivationDataScope())) - edit().setDerivationDataScope(scope); - } - - @Override - public String getDerivationDataScope() - { - return _pd.getDerivationDataScope(); - } - - @Override - public PropertyDescriptor getPropertyDescriptor() - { - return _pd; - } - - @Override - public List getConditionalFormats() - { - return ensureConditionalFormats(); - } - - public boolean isNew() - { - return _pd.getPropertyId() == 0; - } - - // Scenario to swap property descriptors on study upload to or from a system property, instead of updating the - // current property descriptor. Avoids overwriting a system property. - public boolean isSystemPropertySwap() - { - if (_pd.getPropertyId() == 0 && _pd.getPropertyURI() != null && _pdOld != null && _pdOld.getPropertyURI() != null - && !_pd.getPropertyURI().equals(_pdOld.getPropertyURI())) - { - return SystemProperty.getProperties().stream().anyMatch(sp -> - sp.getPropertyURI().equals(_pd.getPropertyURI()) || sp.getPropertyURI().equals(_pdOld.getPropertyURI())); - } - - return false; - } - - public boolean isDirty() - { - if (_pdOld != null) return true; - - for (PropertyValidatorImpl v : ensureValidators()) - { - if (v.isDirty() || v.isNew()) - return true; - } - return false; - } - - public void delete(User user) - { - DomainPropertyManager.get().removeValidatorsForPropertyDescriptor(getContainer(), getPropertyId()); - DomainPropertyManager.get().deleteConditionalFormats(getPropertyId()); - - DomainKind kind = getDomain().getDomainKind(); - if (null != kind) - kind.deletePropertyDescriptor(getDomain(), user, _pd); - OntologyManager.removePropertyDescriptorFromDomain(this); - } - - public void save(User user, DomainDescriptor dd, int sortOrder) throws ChangePropertyDescriptorException - { - if (isSystemPropertySwap()) - { - _pd = OntologyManager.insertOrUpdatePropertyDescriptor(_pd, dd, sortOrder); - OntologyManager.removePropertyDescriptorFromDomain(new DomainPropertyImpl((DomainImpl) getDomain(), _pdOld)); - } - else if (isNew()) - { - _pd = OntologyManager.insertOrUpdatePropertyDescriptor(_pd, dd, sortOrder); - } - else if (_pdOld != null) - { - PropertyType oldType = _pdOld.getPropertyType(); - PropertyType newType = _pd.getPropertyType(); - boolean changedType = false; - if (oldType.getJdbcType() != newType.getJdbcType()) - { - if (newType.getJdbcType().isText() || - (oldType.getJdbcType().isInteger() && newType.getJdbcType().isNumeric())) - { - changedType = true; - if (newType.getJdbcType().isText()) - { - // Remove any previously set formatting string as it won't apply to a text field - _pd.setFormat(null); - } - } - else if (newType.getJdbcType().isDateOrTime() && oldType.getJdbcType().isDateOrTime()) - { - changedType = true; - _pd.setFormat(null); - } - else - { - throw new ChangePropertyDescriptorException("Cannot convert an instance of " + oldType.getJdbcType() + " to " + newType.getJdbcType() + "."); - } - } - - // Issue 44711: Prevent attachment and file field types from being converted to a different type - if (PropertyType.FILE_LINK.getInputType().equalsIgnoreCase(oldType.getInputType()) && oldType != newType) - throw new ChangePropertyDescriptorException("Cannot convert an instance of " + oldType.name() + " to " + newType.name() + "."); - - OntologyManager.validatePropertyDescriptor(_pd); - Table.update(user, OntologyManager.getTinfoPropertyDescriptor(), _pd, _pdOld.getPropertyId()); - OntologyManager.ensurePropertyDomain(_pd, dd, sortOrder); - - boolean hasProvisioner = null != getDomain().getDomainKind() && null != getDomain().getDomainKind().getStorageSchemaName() && dd.getStorageTableName() != null; - SqlDialect dialect = OntologyManager.getExpSchema().getSqlDialect(); - - if (hasProvisioner) - { - boolean mvAdded = !_pdOld.isMvEnabled() && _pd.isMvEnabled(); - boolean mvDropped = _pdOld.isMvEnabled() && !_pd.isMvEnabled(); - boolean propRenamed = !_pdOld.getName().equals(_pd.getName()); - boolean propResized = _pd.isStringType() && _pdOld.getScale() != _pd.getScale(); - - // Drop first, so rename doesn't have to worry about it - if (mvDropped) - ((StorageProvisionerImpl)StorageProvisioner.get()).dropMvIndicator(this, _pdOld); - - if (propRenamed) - StorageProvisionerImpl.get().renameProperty(this.getDomain(), this, _pdOld, mvDropped); - - if (changedType) - { - StorageProvisionerImpl.get().changePropertyType(this.getDomain(), this); - if (_pdOld.getJdbcType() == JdbcType.BOOLEAN && _pd.getJdbcType().isText()) - { - updateBooleanValue( - new SQLFragment().appendIdentifier(_domain.getDomainKind().getStorageSchemaName()).append(".").appendIdentifier(_domain.getStorageTableName()), - _pd.getLegalSelectName(dialect), _pdOld.getFormat(), null); // GitHub Issue #647 - } - } - else if (propResized) - StorageProvisionerImpl.get().resizeProperty(this.getDomain(), this, _pdOld.getScale()); - - if (mvAdded) - StorageProvisionerImpl.get().addMvIndicator(this); - } - else if (changedType) - { - if (oldType.getJdbcType().isDateOrTime() && newType.getJdbcType().isText()) - { - new SqlExecutor(OntologyManager.getExpSchema()).execute( - new SQLFragment("UPDATE "). - append(OntologyManager.getTinfoObjectProperty()). - append(" SET StringValue = DateTimeValue, DateTimeValue = NULL WHERE PropertyId = ?"). - add(_pdOld.getPropertyId())); - } - else if (!oldType.getJdbcType().isText() && newType.getJdbcType().isText()) - { - new SqlExecutor(OntologyManager.getExpSchema()).execute( - new SQLFragment("UPDATE "). - append(OntologyManager.getTinfoObjectProperty()). - append(" SET StringValue = FloatValue, FloatValue = NULL WHERE PropertyId = ?"). - add(_pdOld.getPropertyId())); - } - else if (oldType.getJdbcType().isDateOrTime() && newType.getJdbcType().isDateOrTime()) - { - String sqlTypeName = dialect.getSqlTypeName(newType.getJdbcType()); - String update = String.format("CAST(DateTimeValue AS %s)", sqlTypeName); - if (newType.getJdbcType() == JdbcType.TIME) - update = dialect.getDateTimeToTimeCast("DateTimeValue"); - SQLFragment sqlFragment = new SQLFragment("UPDATE ") - .append(OntologyManager.getTinfoObjectProperty()) - .append(" SET DateTimeValue = ") - .append(update) - .append(" WHERE PropertyId = ?") - .add(_pdOld.getPropertyId()); - new SqlExecutor(OntologyManager.getExpSchema()).execute(sqlFragment); - } - else //noinspection StatementWithEmptyBody - if (oldType.getJdbcType().isInteger() && newType.getJdbcType().isReal()) - { - // Since exp.ObjectProperty stores these types in the same column, there's nothing for us to do - } - else - { - throw new ChangePropertyDescriptorException("Cannot convert from " + oldType.getJdbcType() + " to " + newType.getJdbcType() + " for non-provisioned table"); - } - } - - if (changedType && _pdOld.getJdbcType() == JdbcType.BOOLEAN && _pd.getJdbcType().isText()) - { - updateBooleanValue(OntologyManager.getTinfoObjectProperty().getSQLName(), dialect.makeDatabaseIdentifier("StringValue"), _pdOld.getFormat(), new SQLFragment("PropertyId = ?", _pdOld.getPropertyId())); - } - } - else - { - OntologyManager.ensurePropertyDomain(_pd, _domain._dd, sortOrder); - } - - _pdOld = null; - _schemaChanged = false; - _schemaImport = false; - - for (PropertyValidatorImpl validator : ensureValidators()) - { - if (validator.isDeleted()) - DomainPropertyManager.get().removePropertyValidator(this, validator); - else - DomainPropertyManager.get().savePropertyValidator(user, this, validator); - } - - DomainPropertyManager.get().saveConditionalFormats(user, getPropertyDescriptor(), ensureConditionalFormats()); - } - - /** - * Format values in columns that were just converted from booleans to strings with the DB's default type conversion. - * Postgres will now have 'true' and 'false', and SQLServer will have '0' and '1'. Use the format string to use the - * preferred format, and standardize on 'true' and 'false' in the absence of an explicitly configured format. - */ - private void updateBooleanValue(SQLFragment schemaTable, DatabaseIdentifier column, String formatString, @Nullable SQLFragment whereClause) - { - BooleanFormat f = BooleanFormat.getInstance(formatString); - String trueValue = StringUtils.trimToNull(f.format(true)); - String falseValue = StringUtils.trimToNull(f.format(false)); - String nullValue = StringUtils.trimToNull(f.format(null)); - SQLFragment sql = new SQLFragment("UPDATE ").append(schemaTable).append(" SET "). - appendIdentifier(column).append(" = CASE WHEN "). - appendIdentifier(column).append(" IN ('1', 'true') THEN ? WHEN "). - appendIdentifier(column).append(" IN ('0', 'false') THEN ? ELSE ? END"); - sql.add(trueValue); - sql.add(falseValue); - sql.add(nullValue); - if (whereClause != null) - { - sql.append(" WHERE "); - sql.append(whereClause); - } - new SqlExecutor(OntologyManager.getExpSchema()).execute(sql); - } - - @Override - @NotNull - public List getValidators() - { - return Collections.unmodifiableList(ensureValidators()); - } - - @Override - public void addValidator(IPropertyValidator validator) - { - if (validator != null) - { - if (0 != validator.getPropertyId() && getPropertyId() != validator.getPropertyId()) - throw new IllegalStateException(); - - // Ensure validator is a valid kind (ex. urn:lsid:labkey.com:PropertyValidator:length is no longer valid) - if ( null != PropertyService.get().getValidatorKind(validator.getTypeURI()) ) - { - PropertyValidator impl = new PropertyValidator(); - impl.copy(validator); - impl.setPropertyId(getPropertyId()); - ensureValidators().add(new PropertyValidatorImpl(impl)); - } - } - } - - @Override - public void removeValidator(IPropertyValidator validator) - { - int idx = ensureValidators().indexOf(validator); - if (idx != -1) - { - PropertyValidatorImpl impl = ensureValidators().get(idx); - impl.delete(); - } - } - - @Override - public void removeValidator(long validatorId) - { - if (validatorId == 0) return; - - for (PropertyValidatorImpl imp : ensureValidators()) - { - if (imp.getRowId() == validatorId) - { - imp.delete(); - break; - } - } - } - - @Override - public void copyFrom(DomainProperty propSrc, Container targetContainer) - { - setDescription(propSrc.getDescription()); - setFormat(propSrc.getFormat()); - setLabel(propSrc.getLabel()); - setName(propSrc.getName()); - setDescription(propSrc.getDescription()); - setConceptURI(propSrc.getConceptURI()); - setType(propSrc.getType()); - setDimension(propSrc.isDimension()); - setMeasure(propSrc.isMeasure()); - setRecommendedVariable(propSrc.isRecommendedVariable()); - setDefaultScale(propSrc.getDefaultScale()); - setRequired(propSrc.isRequired()); - setExcludeFromShifting(propSrc.isExcludeFromShifting()); - setFacetingBehavior(propSrc.getFacetingBehavior()); - setImportAliasSet(propSrc.getImportAliasSet()); - setPhi(propSrc.getPHI()); - setURL(propSrc.getURL()); - setURLTarget(propSrc.getURLTarget()); - setHidden(propSrc.isHidden()); - setShownInDetailsView(propSrc.isShownInDetailsView()); - setShownInInsertView(propSrc.isShownInInsertView()); - setShownInUpdateView(propSrc.isShownInUpdateView()); - setShownInLookupView(propSrc.isShownInLookupView()); - setMvEnabled(propSrc.isMvEnabled()); - setDefaultValueTypeEnum(propSrc.getDefaultValueTypeEnum()); - setScale(propSrc.getScale()); - setScannable(propSrc.isScannable()); - - setPrincipalConceptCode(propSrc.getPrincipalConceptCode()); - setSourceOntology(propSrc.getSourceOntology()); - setConceptSubtree(propSrc.getConceptSubtree()); - setConceptImportColumn(propSrc.getConceptImportColumn()); - setConceptLabelColumn(propSrc.getConceptLabelColumn()); - setDerivationDataScope(propSrc.getDerivationDataScope()); - - // check to see if we're moving a lookup column to another container: - Lookup lookup = propSrc.getLookup(); - if (lookup != null && !getContainer().equals(targetContainer)) - { - // we need to update the lookup properties if the lookup container is either the source or the destination container - if (lookup.getContainer() == null) - lookup.setContainer(propSrc.getContainer()); - else if (lookup.getContainer().equals(targetContainer)) - lookup.setContainer(null); - } - setLookup(lookup); - } - - @Override - public void setConditionalFormats(List formats) - { - String newVal = ConditionalFormat.toStringVal(formats); - String oldVal = ConditionalFormat.toStringVal(getConditionalFormats()); - - if (!Objects.equals(newVal, oldVal)) - edit(); - - _formats = formats; - } - - private List ensureValidators() - { - if (_validators == null) - { - _validators = new ArrayList<>(); - for (PropertyValidator validator : DomainPropertyManager.get().getValidators(this)) - { - _validators.add(new PropertyValidatorImpl(validator)); - } - } - return _validators; - } - - private List ensureConditionalFormats() - { - if (_formats == null) - { - _formats = new ArrayList<>(); - _formats.addAll(DomainPropertyManager.get().getConditionalFormats(this)); - } - return _formats; - } - - public PropertyDescriptor getOldProperty() - { - return _pdOld; - } - - @Override - public FacetingBehaviorType getFacetingBehavior() - { - return _pd.getFacetingBehaviorType(); - } - - @Override - public void setFacetingBehavior(FacetingBehaviorType type) - { - if (getFacetingBehavior() == type) - return; - - edit().setFacetingBehaviorType(type); - } - - @Override - public int hashCode() - { - return _pd.hashCode(); - } - - @Override - public boolean equals(Object obj) - { - if (obj == this) - return true; - if (!(obj instanceof DomainPropertyImpl)) - return false; - // once a domain property has been edited, it no longer equals any other domain property: - if (_pdOld != null || ((DomainPropertyImpl) obj)._pdOld != null) - return false; - return (_pd.equals(((DomainPropertyImpl) obj)._pd)); - } - - @Override - public String toString() - { - return super.toString() + _pd.getPropertyURI(); - } - - public Map getAuditRecordMap(@Nullable String validatorStr, @Nullable String conditionalFormatStr) - { - Map map = new LinkedHashMap<>(); - if (!StringUtils.isEmpty(getName())) - map.put("Name", getName()); - if (!StringUtils.isEmpty(getLabel())) - map.put("Label", getLabel()); - if (null != getPropertyType()) - { - if (org.labkey.api.gwt.client.ui.PropertyType.expFlag.getURI().equals(getConceptURI())) - map.put("Type", "Flag"); - else - map.put("Type", getPropertyType().getXarName()); - } - if (getPropertyType().getJdbcType().isText()) - map.put("Scale", getScale()); - if (!StringUtils.isEmpty(getDescription())) - map.put("Description", getDescription()); - if (!StringUtils.isEmpty(getFormat())) - map.put("Format", getFormat()); - if (!StringUtils.isEmpty(getURL())) - map.put("URL", getURL()); - if (!StringUtils.isEmpty(getURLTarget())) - map.put("URLTarget", getURLTarget()); - if (getPHI() != null) - map.put("PHI", getPHI().getLabel()); - if (getDefaultScale() != null) - map.put("DefaultScale", getDefaultScale().getLabel()); - map.put("Required", isRequired()); - map.put("Hidden", isHidden()); - map.put("MvEnabled", isMvEnabled()); - map.put("Measure", isMeasure()); - map.put("Dimension", isDimension()); - map.put("ShownInInsert", isShownInInsertView()); - map.put("ShownInDetails", isShownInDetailsView()); - map.put("ShownInUpdate", isShownInUpdateView()); - map.put("ShownInLookupView", isShownInLookupView()); - map.put("RecommendedVariable", isRecommendedVariable()); - map.put("ExcludedFromShifting", isExcludeFromShifting()); - map.put("Scannable", isScannable()); - if (!StringUtils.isEmpty(getDerivationDataScope())) - map.put("DerivationDataScope", getDerivationDataScope()); - String importAliasStr = StringUtils.join(getImportAliasSet(), ","); - if (!StringUtils.isEmpty(importAliasStr)) - map.put("ImportAliases", importAliasStr); - if (getDefaultValueTypeEnum() != null) - map.put("DefaultValueType", getDefaultValueTypeEnum().getLabel()); - if (getLookup() != null) - map.put("Lookup", getLookup().toJSONString()); - - if (!StringUtils.isEmpty(validatorStr)) - map.put("Validator", validatorStr); - if (!StringUtils.isEmpty(conditionalFormatStr)) - map.put("ConditionalFormat", conditionalFormatStr); - - return map; - } - - public static class TestCase extends Assert - { - private PropertyDescriptor _pd; - private DomainPropertyImpl _dp; - - @Test - public void testUpdateDomainPropertyFromDescriptor() - { - Container c = ContainerManager.ensureContainer("/_DomainPropertyImplTest", TestContext.get().getUser()); - String domainURI = new Lsid("Junit", "DD", "Domain1").toString(); - Domain d = PropertyService.get().createDomain(c, domainURI, "Domain1"); - - resetProperties(d, domainURI, c); - - // verify no change - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertFalse(_dp.isDirty()); - assertFalse(_dp._schemaChanged); - - // change a property - _pd.setPHI(PHI.Restricted); - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertTrue(_dp.isDirty()); - assertFalse(_dp._schemaChanged); - assertTrue(_dp.getPHI() == _pd.getPHI()); - - // Issue #18738 change the schema outside of a schema reload and verify that the column - // change the schema but don't mark the property as "Schema Import" - // this will allow whatever type changes the UI allows (text -> multiline, for example) - resetProperties(d, domainURI, c); - _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#double"); - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertTrue(_dp.isDirty()); - assertTrue(_dp._schemaChanged); - assertFalse(_dp.isRecreateRequired()); - assertTrue(Strings.CS.equals(_dp.getRangeURI(), _pd.getRangeURI())); - - // setting schema import to true will enable the _schemaChanged flag to toggle - // so it should be set true here - resetProperties(d, domainURI, c); - _dp.setSchemaImport(true); - _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#double"); - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertTrue(_dp.isDirty()); - assertTrue(_dp._schemaChanged); - assertTrue(_dp.isRecreateRequired()); - assertTrue(Strings.CS.equals(_dp.getRangeURI(), _pd.getRangeURI())); - - // verify no change when setting value to the same value as it was - resetProperties(d, domainURI, c); - _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#int"); - _pd.setPHI(PHI.NotPHI); - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertFalse(_dp.isDirty()); - assertFalse(_dp._schemaChanged); - assertFalse(_dp.isRecreateRequired()); - - // verify Lookup is set to null with null schema - resetProperties(d, domainURI, c); - verifyLookup(null, "lkSchema", null, true); - - // verify Lookup is set to null with null query - resetProperties(d, domainURI, c); - verifyLookup(null, null, "lkQuery",true); - - // verify Lookup is set to null with invalid container - resetProperties(d, domainURI, c); - verifyLookup("bogus", null, "lkQuery",true); - - // verify Lookup is set with valid schema and query - resetProperties(d, domainURI, c); - verifyLookup(null, "lkSchema", "lkQuery",true); - - // verify Lookup is set with valid container, schema and query - resetProperties(d, domainURI, c); - verifyLookup(c.getId(), "lkSchema1", "lkQuery2",true); - - // no cleanup as we never persisted anything - } - - private void verifyLookup(String containerId, String schema, String query, Boolean expectedDirty) - { - _pd.setLookupContainer(containerId); - _pd.setLookupQuery(query); - _pd.setLookupSchema(schema); - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertTrue(_dp.isDirty() == expectedDirty); - assertFalse(_dp._schemaChanged); - - // verify the lookup object returned - Lookup l = _dp.getLookup(); - - if (l == null) - { - // lookup can be null if we specified a containerId that is invalid or - // we specified a valid containerId (including null) but schema or query is null - if (containerId != null && null == ContainerManager.getForId(containerId)) - assertTrue(true); - else if (query == null || schema == null) - assertTrue(true); - else - assertTrue(false); - } - else - { - if (containerId != null) - assertTrue(Strings.CS.equals(l.getContainer().getId(), _pd.getLookupContainer())); - - assertTrue(Strings.CS.equals(l.getQueryName(), _pd.getLookupQuery())); - assertTrue(Strings.CS.equals(l.getSchemaKey().toString(), _pd.getLookupSchema())); - } - } - - private void resetProperties(Domain d, String domainUri, Container c) - { - _pd = getPropertyDescriptor(c, domainUri); - _dp = (DomainPropertyImpl) d.addProperty(); - _pd.copyTo(_dp.getPropertyDescriptor()); - } - - - private PropertyDescriptor getPropertyDescriptor(Container c, String domainURI) - { - PropertyDescriptor pd = new PropertyDescriptor(); - pd.setPropertyURI(domainURI + ":column"); - pd.setName("column"); - pd.setLabel("label"); - pd.setConceptURI(null); - pd.setRangeURI("http://www.w3.org/2001/XMLSchema#int"); - pd.setContainer(c); - pd.setDescription("description"); - pd.setURL(StringExpressionFactory.createURL((String)null)); - pd.setURLTarget(null); - pd.setImportAliases(null); - pd.setRequired(false); - pd.setHidden(false); - pd.setShownInInsertView(true); - pd.setShownInUpdateView(true); - pd.setShownInDetailsView(true); - pd.setDimension(false); - pd.setMeasure(true); - pd.setRecommendedVariable(false); - pd.setDefaultScale(DefaultScaleType.LINEAR); - pd.setFormat(null); - pd.setMvEnabled(false); - pd.setLookupContainer(c.getId()); - pd.setLookupSchema("lkSchema"); - pd.setLookupQuery("lkQuery"); - pd.setFacetingBehaviorType(FacetingBehaviorType.AUTOMATIC); - pd.setPHI(PHI.NotPHI); - pd.setExcludeFromShifting(false); - return pd; - } - } - - -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api.property; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.data.BooleanFormat; +import org.labkey.api.data.ColumnRenderPropertiesImpl; +import org.labkey.api.data.ConditionalFormat; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DatabaseIdentifier; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.PHI; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.exp.ChangePropertyDescriptorException; +import org.labkey.api.exp.DomainDescriptor; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.IPropertyType; +import org.labkey.api.exp.property.IPropertyValidator; +import org.labkey.api.exp.property.Lookup; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.gwt.client.DefaultScaleType; +import org.labkey.api.gwt.client.DefaultValueType; +import org.labkey.api.gwt.client.FacetingBehaviorType; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.security.User; +import org.labkey.api.util.StringExpressionFactory; +import org.labkey.api.util.TestContext; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class DomainPropertyImpl implements DomainProperty +{ + private final DomainImpl _domain; + + PropertyDescriptor _pd; + PropertyDescriptor _pdOld; + boolean _deleted; + + private boolean _schemaChanged; + private boolean _schemaImport; + private List _validators; + private List _formats; + private String _defaultValue; + + public DomainPropertyImpl(DomainImpl type, PropertyDescriptor pd) + { + this(type, pd, null); + } + + public DomainPropertyImpl(DomainImpl type, PropertyDescriptor pd, List formats) + { + _domain = type; + _pd = pd.clone(); + _formats = formats; + } + + @Override + public int getPropertyId() + { + return _pd.getPropertyId(); + } + + @Override + public Container getContainer() + { + return _pd.getContainer(); + } + + @Override + public String getPropertyURI() + { + return _pd.getPropertyURI(); + } + + @Override + public String getName() + { + return _pd.getName(); + } + + @Override + public String getDescription() + { + return _pd.getDescription(); + } + + @Override + public String getFormat() + { + return _pd.getFormat(); + } + + @Override + public String getLabel() + { + return _pd.getLabel(); + } + + @Override + public String getConceptURI() + { + return _pd.getConceptURI(); + } + + @Override + public Domain getDomain() + { + return _domain; + } + + @Override + public IPropertyType getType() + { + return PropertyService.get().getType(getContainer(), _pd.getRangeURI()); + } + + @Override + public boolean isRequired() + { + return _pd.isRequired(); + } + + @Override + public boolean isHidden() + { + return _pd.isHidden(); + } + + @Override + public boolean isDeleted() + { + return _deleted; + } + + @Override + public boolean isShownInInsertView() + { + return _pd.isShownInInsertView(); + } + + @Override + public boolean isShownInDetailsView() + { + return _pd.isShownInDetailsView(); + } + + @Override + public boolean isShownInUpdateView() + { + return _pd.isShownInUpdateView(); + } + + @Override + public boolean isShownInLookupView() + { + return _pd.isShownInLookupView(); + } + + @Override + public boolean isMeasure() + { + return _pd.isMeasure(); + } + + @Override + public boolean isDimension() + { + return _pd.isDimension(); + } + + @Override + public boolean isRecommendedVariable() + { + return _pd.isRecommendedVariable(); + } + + @Override + public DefaultScaleType getDefaultScale() + { + return _pd.getDefaultScale(); + } + + @Override + public PHI getPHI() + { + return _pd.getPHI(); + } + + @Override + public String getRedactedText() { return _pd.getRedactedText(); } + + @Override + public boolean isExcludeFromShifting() + { + return _pd.isExcludeFromShifting(); + } + + @Override + public boolean isMvEnabled() + { + return _pd.isMvEnabled(); + } + + @Override + public boolean isMvEnabledForDrop() + { + if (null != _pdOld) + return _pdOld.isMvEnabled(); // if we need to drop/recreate we care about the old one + return _pd.isMvEnabled(); + } + + @Override + public void delete() + { + _deleted = true; + } + + @Override + public void setSchemaImport(boolean isSchemaImport) + { + // if this flag is set True then the column is dropped and recreated by its Domain if there is a type change + _schemaImport = isSchemaImport; + } + + @Override + public void setName(String name) + { + if (Strings.CS.equals(name, getName())) + return; + edit().setName(name); + } + + @Override + public void setDescription(String description) + { + if (Strings.CS.equals(description, getDescription())) + return; + edit().setDescription(description); + } + + @Override + public void setType(IPropertyType domain) + { + edit().setRangeURI(domain.getTypeURI()); + } + + @Override + public void setPropertyURI(String uri) + { + if (Strings.CS.equals(uri, getPropertyURI())) + return; + edit().setPropertyURI(uri); + } + + @Override + public void setRangeURI(String rangeURI) + { + if (Strings.CS.equals(rangeURI, getRangeURI())) + return; + editSchema().setRangeURI(rangeURI); + } + + @Override + public String getRangeURI() + { + return _pd.getRangeURI(); + } + + @Override + public void setFormat(String s) + { + if (Strings.CS.equals(s, getFormat())) + return; + edit().setFormat(s); + } + + @Override + public void setLabel(String caption) + { + if (Strings.CS.equals(caption, getLabel())) + return; + edit().setLabel(caption); + } + + @Override + public void setConceptURI(String conceptURI) + { + if (Strings.CS.equals(conceptURI, getConceptURI())) + return; + edit().setConceptURI(conceptURI); + } + + @Override + public void setRequired(boolean required) + { + if (required == isRequired()) + return; + edit().setRequired(required); + } + + @Override + public void setHidden(boolean hidden) + { + if (hidden == isHidden()) + return; + edit().setHidden(hidden); + } + + @Override + public void setShownInDetailsView(boolean shown) + { + if (shown == isShownInDetailsView()) + return; + edit().setShownInDetailsView(shown); + } + + @Override + public void setShownInInsertView(boolean shown) + { + if (shown == isShownInInsertView()) + return; + edit().setShownInInsertView(shown); + } + + @Override + public void setShownInUpdateView(boolean shown) + { + if (shown == isShownInUpdateView()) + return; + edit().setShownInUpdateView(shown); + } + + @Override + public void setShownInLookupView(boolean shown) + { + if (shown == isShownInLookupView()) + return; + edit().setShownInLookupView(shown); + } + + @Override + public void setMeasure(boolean isMeasure) + { + // UNDONE: isMeasure() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. + if (!isEdited() && isMeasure == isMeasure()) + return; + edit().setMeasure(isMeasure); + } + + @Override + public void setDimension(boolean isDimension) + { + // UNDONE: isDimension() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. + if (!isEdited() && isDimension == isDimension()) + return; + edit().setDimension(isDimension); + } + + @Override + public void setRecommendedVariable(boolean isRecommendedVariable) + { + if (!isEdited() && isRecommendedVariable == isRecommendedVariable()) + return; + edit().setRecommendedVariable(isRecommendedVariable); + } + + @Override + public void setDefaultScale(DefaultScaleType defaultScale) + { + if (!isEdited() && getDefaultScale() == defaultScale) + return; + + edit().setDefaultScale(defaultScale); + } + + @Override + public void setPhi(PHI phi) + { + if (!isEdited() && getPHI() == phi) + return; + edit().setPHI(phi); + } + + @Override + public void setRedactedText(String redactedText) + { + if (!isEdited() && ((getRedactedText() != null && getRedactedText().equals(redactedText)) + || (getRedactedText() == null && redactedText == null))) + return; + edit().setRedactedText(redactedText); + } + + @Override + public void setExcludeFromShifting(boolean isExcludeFromShifting) + { + // UNDONE: isExcludeFromShifting() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. + if (!isEdited() && isExcludeFromShifting == isExcludeFromShifting()) + return; + edit().setExcludeFromShifting(isExcludeFromShifting); + } + + @Override + public void setMvEnabled(boolean mv) + { + if (mv == isMvEnabled()) + return; + edit().setMvEnabled(mv); + } + + @Override + public void setScale(int scale) + { + if (scale == getScale()) + return; + edit().setScale(scale); + } + + /** Need the string version of this method because it's called by reflection and must match by name */ + public void setImportAliases(String aliases) + { + if (Strings.CS.equals(aliases, getImportAliases())) + return; + edit().setImportAliases(aliases); + } + + /** Need the string version of this method because it's called by reflection and must match by name */ + public String getImportAliases() + { + return _pd.getImportAliases(); + } + + @Override + public void setImportAliasSet(Set aliases) + { + String current = getImportAliases(); + String newAliases = ColumnRenderPropertiesImpl.convertToString(aliases); + if (Strings.CS.equals(current, newAliases)) + return; + edit().setImportAliasesSet(aliases); + } + + @Override + public Set getImportAliasSet() + { + return _pd.getImportAliasSet(); + } + + @Override + public void setURL(String url) + { + if (Strings.CS.equals(getURL(), url)) + return; + + if (null == url) + edit().setURL(null); + else + edit().setURL(StringExpressionFactory.createURL(url)); + } + + @Override + public String getURL() + { + return _pd.getURL() == null ? null : _pd.getURL().toString(); + } + + @Override + public void setURLTarget(String urlTarget) + { + if (Strings.CS.equals(getURLTarget(), urlTarget)) + return; + edit().setURLTarget(urlTarget); + } + + @Override + public String getURLTarget() + { + return _pd.getURLTarget(); + } + + private boolean isEdited() + { + return null != _pdOld; + } + + private PropertyDescriptor editSchema() + { + PropertyDescriptor pd = edit(); + _schemaChanged = true; + _pd.clearPropertyType(); + return pd; + } + + public boolean isRecreateRequired() + { + return _schemaChanged && _schemaImport; + } + + public void markAsNew() + { + assert isRecreateRequired() && !isNew(); + _pd.setPropertyId(0); + } + + private PropertyDescriptor edit() + { + if (_pdOld == null) + { + _pdOld = _pd; + _pd = _pdOld.clone(); + } + return _pd; + } + + @Override + public PropertyType getPropertyType() + { + return _pd.getPropertyType(); + } + + @Override + public JdbcType getJdbcType() + { + return _pd.getPropertyType().getJdbcType(); + } + + @Override + public int getScale() + { + return _pd.getScale(); + } + + @Override + public String getInputType() + { + return _pd.getPropertyType().getInputType(); + } + + @Override + public DefaultValueType getDefaultValueTypeEnum() + { + return _pd.getDefaultValueTypeEnum(); + } + + @Override + public void setDefaultValueTypeEnum(DefaultValueType defaultValueType) + { + _pd.setDefaultValueTypeEnum(defaultValueType); + } + + public String getDefaultValueType() + { + return _pd.getDefaultValueType(); + } + + @Override + public void setDefaultValueType(String defaultValueTypeName) + { + if (getDefaultValueType() != null && getDefaultValueType().equals(defaultValueTypeName)) + return; + + if (getDefaultValueType() == null && defaultValueTypeName == null) + return; // if both are null, don't call edit(), with marks property as dirty + + edit().setDefaultValueType(defaultValueTypeName); + } + + @Override + public void setDefaultValue(String value) + { + _defaultValue = value; + } + + public String getDefaultValue() + { + return _defaultValue; + } + + @Override + public Lookup getLookup() + { + return _pd.getLookup(); + } + + @Override + public void setLookup(Lookup lookup) + { + Lookup current = getLookup(); + + if (current == lookup) + return; + + // current will return null if the schema or query is null so check + // for this case in the passed in lookup + if (current == null) + if (lookup.getQueryName() == null || lookup.getSchemaKey() == null) + return; + + if (current != null && current.equals(lookup)) + return; + + if (lookup == null) + { + edit().setLookupContainer(null); + edit().setLookupSchema(null); + edit().setLookupQuery(null); + return; + } + if (lookup.getContainer() == null) + { + edit().setLookupContainer(null); + } + else + { + edit().setLookupContainer(lookup.getContainer().getId()); + } + edit().setLookupQuery(lookup.getQueryName()); + edit().setLookupSchema(Objects.toString(lookup.getSchemaKey(),null)); + } + + @Override + public void setScannable(boolean scannable) + { + if (scannable != isScannable()) + edit().setScannable(scannable); + } + + @Override + public void setOldPropertyDescriptor(PropertyDescriptor oldPropertyDescriptor) + { + if (isEdited()) + return; + + _pdOld = oldPropertyDescriptor.clone(); + } + + @Override + public boolean isScannable() + { + return _pd.isScannable(); + } + + @Override + public void setPrincipalConceptCode(String code) + { + if (!Strings.CS.equals(code, getPrincipalConceptCode())) + edit().setPrincipalConceptCode(code); + } + + @Override + public String getPrincipalConceptCode() + { + return _pd.getPrincipalConceptCode(); + } + + @Override + public String getSourceOntology() + { + return _pd.getSourceOntology(); + } + + @Override + public void setSourceOntology(String sourceOntology) + { + if (!Strings.CS.equals(sourceOntology, getSourceOntology())) + edit().setSourceOntology(sourceOntology); + } + + @Override + public String getConceptSubtree() + { + return _pd.getConceptSubtree(); + } + + @Override + public void setConceptSubtree(String path) + { + if (!Strings.CS.equals(path, getConceptSubtree())) + edit().setConceptSubtree(path); + } + + @Override + public String getConceptImportColumn() + { + return _pd.getConceptImportColumn(); + } + + @Override + public void setConceptImportColumn(String conceptImportColumn) + { + if (!Strings.CS.equals(conceptImportColumn, getConceptImportColumn())) + edit().setConceptImportColumn(conceptImportColumn); + } + + @Override + public String getConceptLabelColumn() + { + return _pd.getConceptLabelColumn(); + } + + @Override + public void setConceptLabelColumn(String conceptLabelColumn) + { + if (!Strings.CS.equals(conceptLabelColumn, getConceptLabelColumn())) + edit().setConceptLabelColumn(conceptLabelColumn); + } + + @Override + public void setDerivationDataScope(String scope) + { + if (!Strings.CS.equals(scope, getDerivationDataScope())) + edit().setDerivationDataScope(scope); + } + + @Override + public String getDerivationDataScope() + { + return _pd.getDerivationDataScope(); + } + + @Override + public PropertyDescriptor getPropertyDescriptor() + { + return _pd; + } + + @Override + public List getConditionalFormats() + { + return ensureConditionalFormats(); + } + + public boolean isNew() + { + return _pd.getPropertyId() == 0; + } + + // Scenario to swap property descriptors on study upload to or from a system property, instead of updating the + // current property descriptor. Avoids overwriting a system property. + public boolean isSystemPropertySwap() + { + if (_pd.getPropertyId() == 0 && _pd.getPropertyURI() != null && _pdOld != null && _pdOld.getPropertyURI() != null + && !_pd.getPropertyURI().equals(_pdOld.getPropertyURI())) + { + return SystemProperty.getProperties().stream().anyMatch(sp -> + sp.getPropertyURI().equals(_pd.getPropertyURI()) || sp.getPropertyURI().equals(_pdOld.getPropertyURI())); + } + + return false; + } + + public boolean isDirty() + { + if (_pdOld != null) return true; + + for (PropertyValidatorImpl v : ensureValidators()) + { + if (v.isDirty() || v.isNew()) + return true; + } + return false; + } + + public void delete(User user) + { + DomainPropertyManager.get().removeValidatorsForPropertyDescriptor(getContainer(), getPropertyId()); + DomainPropertyManager.get().deleteConditionalFormats(getPropertyId()); + + DomainKind kind = getDomain().getDomainKind(); + if (null != kind) + kind.deletePropertyDescriptor(getDomain(), user, _pd); + OntologyManager.removePropertyDescriptorFromDomain(this); + } + + public void save(User user, DomainDescriptor dd, int sortOrder) throws ChangePropertyDescriptorException + { + if (isSystemPropertySwap()) + { + _pd = OntologyManager.insertOrUpdatePropertyDescriptor(_pd, dd, sortOrder); + OntologyManager.removePropertyDescriptorFromDomain(new DomainPropertyImpl((DomainImpl) getDomain(), _pdOld)); + } + else if (isNew()) + { + _pd = OntologyManager.insertOrUpdatePropertyDescriptor(_pd, dd, sortOrder); + } + else if (_pdOld != null) + { + PropertyType oldType = _pdOld.getPropertyType(); + PropertyType newType = _pd.getPropertyType(); + boolean changedType = false; + if (oldType.getJdbcType() != newType.getJdbcType()) + { + if (newType.getJdbcType().isText() || + (oldType.getJdbcType().isInteger() && newType.getJdbcType().isNumeric())) + { + changedType = true; + if (newType.getJdbcType().isText()) + { + // Remove any previously set formatting string as it won't apply to a text field + _pd.setFormat(null); + } + } + else if (newType.getJdbcType().isDateOrTime() && oldType.getJdbcType().isDateOrTime()) + { + changedType = true; + _pd.setFormat(null); + } + else if (newType == PropertyType.MULTI_CHOICE || oldType == PropertyType.MULTI_CHOICE) + { + changedType = true; + _pd.setFormat(null); + } + else + { + throw new ChangePropertyDescriptorException("Cannot convert an instance of " + oldType.getJdbcType() + " to " + newType.getJdbcType() + "."); + } + } + + // Issue 44711: Prevent attachment and file field types from being converted to a different type + if (PropertyType.FILE_LINK.getInputType().equalsIgnoreCase(oldType.getInputType()) && oldType != newType) + throw new ChangePropertyDescriptorException("Cannot convert an instance of " + oldType.name() + " to " + newType.name() + "."); + + OntologyManager.validatePropertyDescriptor(_pd); + Table.update(user, OntologyManager.getTinfoPropertyDescriptor(), _pd, _pdOld.getPropertyId()); + OntologyManager.ensurePropertyDomain(_pd, dd, sortOrder); + + boolean hasProvisioner = null != getDomain().getDomainKind() && null != getDomain().getDomainKind().getStorageSchemaName() && dd.getStorageTableName() != null; + SqlDialect dialect = OntologyManager.getExpSchema().getSqlDialect(); + + if (hasProvisioner) + { + boolean mvAdded = !_pdOld.isMvEnabled() && _pd.isMvEnabled(); + boolean mvDropped = _pdOld.isMvEnabled() && !_pd.isMvEnabled(); + boolean propRenamed = !_pdOld.getName().equals(_pd.getName()); + boolean propResized = _pd.isStringType() && _pdOld.getScale() != _pd.getScale(); + + // Drop first, so rename doesn't have to worry about it + if (mvDropped) + ((StorageProvisionerImpl)StorageProvisioner.get()).dropMvIndicator(this, _pdOld); + + if (propRenamed) + StorageProvisionerImpl.get().renameProperty(this.getDomain(), this, _pdOld, mvDropped); + + if (changedType) + { + var domainKind = _domain.getDomainKind(); + if (domainKind == null) + throw new ChangePropertyDescriptorException("Cannot change property type for domain, unknown domain kind."); + + StorageProvisionerImpl.get().changePropertyType(this.getDomain(), this); + if (_pdOld.getJdbcType() == JdbcType.BOOLEAN && _pd.getJdbcType().isText()) + { + updateBooleanValue( + new SQLFragment().appendIdentifier(domainKind.getStorageSchemaName()).append(".").appendIdentifier(_domain.getStorageTableName()), + _pd.getLegalSelectName(dialect), _pdOld.getFormat(), null); // GitHub Issue #647 + } + + TableInfo table = domainKind.getTableInfo(user, getContainer(), _domain, ContainerFilter.getUnsafeEverythingFilter()); + if (table != null && _pdOld.getPropertyType() != null) + QueryChangeListener.QueryPropertyChange.handleColumnTypeChange(_pdOld, _pd, SchemaKey.fromString(table.getUserSchema().getName()), table.getName(), user, getContainer()); + } + else if (propResized) + StorageProvisionerImpl.get().resizeProperty(this.getDomain(), this, _pdOld.getScale()); + + if (mvAdded) + StorageProvisionerImpl.get().addMvIndicator(this); + } + else if (changedType) + { + if (oldType.getJdbcType().isDateOrTime() && newType.getJdbcType().isText()) + { + new SqlExecutor(OntologyManager.getExpSchema()).execute( + new SQLFragment("UPDATE "). + append(OntologyManager.getTinfoObjectProperty()). + append(" SET StringValue = DateTimeValue, DateTimeValue = NULL WHERE PropertyId = ?"). + add(_pdOld.getPropertyId())); + } + else if (!oldType.getJdbcType().isText() && newType.getJdbcType().isText()) + { + new SqlExecutor(OntologyManager.getExpSchema()).execute( + new SQLFragment("UPDATE "). + append(OntologyManager.getTinfoObjectProperty()). + append(" SET StringValue = FloatValue, FloatValue = NULL WHERE PropertyId = ?"). + add(_pdOld.getPropertyId())); + } + else if (oldType.getJdbcType().isDateOrTime() && newType.getJdbcType().isDateOrTime()) + { + String sqlTypeName = dialect.getSqlTypeName(newType.getJdbcType()); + String update = String.format("CAST(DateTimeValue AS %s)", sqlTypeName); + if (newType.getJdbcType() == JdbcType.TIME) + update = dialect.getDateTimeToTimeCast("DateTimeValue"); + SQLFragment sqlFragment = new SQLFragment("UPDATE ") + .append(OntologyManager.getTinfoObjectProperty()) + .append(" SET DateTimeValue = ") + .append(update) + .append(" WHERE PropertyId = ?") + .add(_pdOld.getPropertyId()); + new SqlExecutor(OntologyManager.getExpSchema()).execute(sqlFragment); + } + else //noinspection StatementWithEmptyBody + if (oldType.getJdbcType().isInteger() && newType.getJdbcType().isReal()) + { + // Since exp.ObjectProperty stores these types in the same column, there's nothing for us to do + } + else + { + throw new ChangePropertyDescriptorException("Cannot convert from " + oldType.getJdbcType() + " to " + newType.getJdbcType() + " for non-provisioned table"); + } + } + + if (changedType && _pdOld.getJdbcType() == JdbcType.BOOLEAN && _pd.getJdbcType().isText()) + { + updateBooleanValue(OntologyManager.getTinfoObjectProperty().getSQLName(), dialect.makeDatabaseIdentifier("StringValue"), _pdOld.getFormat(), new SQLFragment("PropertyId = ?", _pdOld.getPropertyId())); + } + } + else + { + OntologyManager.ensurePropertyDomain(_pd, _domain._dd, sortOrder); + } + + _pdOld = null; + _schemaChanged = false; + _schemaImport = false; + + for (PropertyValidatorImpl validator : ensureValidators()) + { + if (validator.isDeleted()) + DomainPropertyManager.get().removePropertyValidator(this, validator); + else + DomainPropertyManager.get().savePropertyValidator(user, this, validator); + } + + DomainPropertyManager.get().saveConditionalFormats(user, getPropertyDescriptor(), ensureConditionalFormats()); + } + + /** + * Format values in columns that were just converted from booleans to strings with the DB's default type conversion. + * Postgres will now have 'true' and 'false', and SQLServer will have '0' and '1'. Use the format string to use the + * preferred format, and standardize on 'true' and 'false' in the absence of an explicitly configured format. + */ + private void updateBooleanValue(SQLFragment schemaTable, DatabaseIdentifier column, String formatString, @Nullable SQLFragment whereClause) + { + BooleanFormat f = BooleanFormat.getInstance(formatString); + String trueValue = StringUtils.trimToNull(f.format(true)); + String falseValue = StringUtils.trimToNull(f.format(false)); + String nullValue = StringUtils.trimToNull(f.format(null)); + SQLFragment sql = new SQLFragment("UPDATE ").append(schemaTable).append(" SET "). + appendIdentifier(column).append(" = CASE WHEN "). + appendIdentifier(column).append(" IN ('1', 'true') THEN ? WHEN "). + appendIdentifier(column).append(" IN ('0', 'false') THEN ? ELSE ? END"); + sql.add(trueValue); + sql.add(falseValue); + sql.add(nullValue); + if (whereClause != null) + { + sql.append(" WHERE "); + sql.append(whereClause); + } + new SqlExecutor(OntologyManager.getExpSchema()).execute(sql); + } + + @Override + @NotNull + public List getValidators() + { + return Collections.unmodifiableList(ensureValidators()); + } + + @Override + public void addValidator(IPropertyValidator validator) + { + if (validator != null) + { + if (0 != validator.getPropertyId() && getPropertyId() != validator.getPropertyId()) + throw new IllegalStateException(); + + // Ensure validator is a valid kind (ex. urn:lsid:labkey.com:PropertyValidator:length is no longer valid) + if ( null != PropertyService.get().getValidatorKind(validator.getTypeURI()) ) + { + PropertyValidator impl = new PropertyValidator(); + impl.copy(validator); + impl.setPropertyId(getPropertyId()); + ensureValidators().add(new PropertyValidatorImpl(impl)); + } + } + } + + @Override + public void removeValidator(IPropertyValidator validator) + { + int idx = ensureValidators().indexOf(validator); + if (idx != -1) + { + PropertyValidatorImpl impl = ensureValidators().get(idx); + impl.delete(); + } + } + + @Override + public void removeValidator(long validatorId) + { + if (validatorId == 0) return; + + for (PropertyValidatorImpl imp : ensureValidators()) + { + if (imp.getRowId() == validatorId) + { + imp.delete(); + break; + } + } + } + + @Override + public void copyFrom(DomainProperty propSrc, Container targetContainer) + { + setDescription(propSrc.getDescription()); + setFormat(propSrc.getFormat()); + setLabel(propSrc.getLabel()); + setName(propSrc.getName()); + setDescription(propSrc.getDescription()); + setConceptURI(propSrc.getConceptURI()); + setType(propSrc.getType()); + setDimension(propSrc.isDimension()); + setMeasure(propSrc.isMeasure()); + setRecommendedVariable(propSrc.isRecommendedVariable()); + setDefaultScale(propSrc.getDefaultScale()); + setRequired(propSrc.isRequired()); + setExcludeFromShifting(propSrc.isExcludeFromShifting()); + setFacetingBehavior(propSrc.getFacetingBehavior()); + setImportAliasSet(propSrc.getImportAliasSet()); + setPhi(propSrc.getPHI()); + setURL(propSrc.getURL()); + setURLTarget(propSrc.getURLTarget()); + setHidden(propSrc.isHidden()); + setShownInDetailsView(propSrc.isShownInDetailsView()); + setShownInInsertView(propSrc.isShownInInsertView()); + setShownInUpdateView(propSrc.isShownInUpdateView()); + setShownInLookupView(propSrc.isShownInLookupView()); + setMvEnabled(propSrc.isMvEnabled()); + setDefaultValueTypeEnum(propSrc.getDefaultValueTypeEnum()); + setScale(propSrc.getScale()); + setScannable(propSrc.isScannable()); + + setPrincipalConceptCode(propSrc.getPrincipalConceptCode()); + setSourceOntology(propSrc.getSourceOntology()); + setConceptSubtree(propSrc.getConceptSubtree()); + setConceptImportColumn(propSrc.getConceptImportColumn()); + setConceptLabelColumn(propSrc.getConceptLabelColumn()); + setDerivationDataScope(propSrc.getDerivationDataScope()); + + // check to see if we're moving a lookup column to another container: + Lookup lookup = propSrc.getLookup(); + if (lookup != null && !getContainer().equals(targetContainer)) + { + // we need to update the lookup properties if the lookup container is either the source or the destination container + if (lookup.getContainer() == null) + lookup.setContainer(propSrc.getContainer()); + else if (lookup.getContainer().equals(targetContainer)) + lookup.setContainer(null); + } + setLookup(lookup); + } + + @Override + public void setConditionalFormats(List formats) + { + String newVal = ConditionalFormat.toStringVal(formats); + String oldVal = ConditionalFormat.toStringVal(getConditionalFormats()); + + if (!Objects.equals(newVal, oldVal)) + edit(); + + _formats = formats; + } + + private List ensureValidators() + { + if (_validators == null) + { + _validators = new ArrayList<>(); + for (PropertyValidator validator : DomainPropertyManager.get().getValidators(this)) + { + _validators.add(new PropertyValidatorImpl(validator)); + } + } + return _validators; + } + + private List ensureConditionalFormats() + { + if (_formats == null) + { + _formats = new ArrayList<>(); + _formats.addAll(DomainPropertyManager.get().getConditionalFormats(this)); + } + return _formats; + } + + public PropertyDescriptor getOldProperty() + { + return _pdOld; + } + + @Override + public FacetingBehaviorType getFacetingBehavior() + { + return _pd.getFacetingBehaviorType(); + } + + @Override + public void setFacetingBehavior(FacetingBehaviorType type) + { + if (getFacetingBehavior() == type) + return; + + edit().setFacetingBehaviorType(type); + } + + @Override + public int hashCode() + { + return _pd.hashCode(); + } + + @Override + public boolean equals(Object obj) + { + if (obj == this) + return true; + if (!(obj instanceof DomainPropertyImpl)) + return false; + // once a domain property has been edited, it no longer equals any other domain property: + if (_pdOld != null || ((DomainPropertyImpl) obj)._pdOld != null) + return false; + return (_pd.equals(((DomainPropertyImpl) obj)._pd)); + } + + @Override + public String toString() + { + return super.toString() + _pd.getPropertyURI(); + } + + public Map getAuditRecordMap(@Nullable String validatorStr, @Nullable String conditionalFormatStr) + { + Map map = new LinkedHashMap<>(); + if (!StringUtils.isEmpty(getName())) + map.put("Name", getName()); + if (!StringUtils.isEmpty(getLabel())) + map.put("Label", getLabel()); + if (null != getPropertyType()) + { + if (org.labkey.api.gwt.client.ui.PropertyType.expFlag.getURI().equals(getConceptURI())) + map.put("Type", "Flag"); + else + map.put("Type", getPropertyType().getXarName()); + } + if (getPropertyType().getJdbcType().isText()) + map.put("Scale", getScale()); + if (!StringUtils.isEmpty(getDescription())) + map.put("Description", getDescription()); + if (!StringUtils.isEmpty(getFormat())) + map.put("Format", getFormat()); + if (!StringUtils.isEmpty(getURL())) + map.put("URL", getURL()); + if (!StringUtils.isEmpty(getURLTarget())) + map.put("URLTarget", getURLTarget()); + if (getPHI() != null) + map.put("PHI", getPHI().getLabel()); + if (getDefaultScale() != null) + map.put("DefaultScale", getDefaultScale().getLabel()); + map.put("Required", isRequired()); + map.put("Hidden", isHidden()); + map.put("MvEnabled", isMvEnabled()); + map.put("Measure", isMeasure()); + map.put("Dimension", isDimension()); + map.put("ShownInInsert", isShownInInsertView()); + map.put("ShownInDetails", isShownInDetailsView()); + map.put("ShownInUpdate", isShownInUpdateView()); + map.put("ShownInLookupView", isShownInLookupView()); + map.put("RecommendedVariable", isRecommendedVariable()); + map.put("ExcludedFromShifting", isExcludeFromShifting()); + map.put("Scannable", isScannable()); + if (!StringUtils.isEmpty(getDerivationDataScope())) + map.put("DerivationDataScope", getDerivationDataScope()); + String importAliasStr = StringUtils.join(getImportAliasSet(), ","); + if (!StringUtils.isEmpty(importAliasStr)) + map.put("ImportAliases", importAliasStr); + if (getDefaultValueTypeEnum() != null) + map.put("DefaultValueType", getDefaultValueTypeEnum().getLabel()); + if (getLookup() != null) + map.put("Lookup", getLookup().toJSONString()); + + if (!StringUtils.isEmpty(validatorStr)) + map.put("Validator", validatorStr); + if (!StringUtils.isEmpty(conditionalFormatStr)) + map.put("ConditionalFormat", conditionalFormatStr); + + return map; + } + + public static class TestCase extends Assert + { + private PropertyDescriptor _pd; + private DomainPropertyImpl _dp; + + @Test + public void testUpdateDomainPropertyFromDescriptor() + { + Container c = ContainerManager.ensureContainer("/_DomainPropertyImplTest", TestContext.get().getUser()); + String domainURI = new Lsid("Junit", "DD", "Domain1").toString(); + Domain d = PropertyService.get().createDomain(c, domainURI, "Domain1"); + + resetProperties(d, domainURI, c); + + // verify no change + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertFalse(_dp.isDirty()); + assertFalse(_dp._schemaChanged); + + // change a property + _pd.setPHI(PHI.Restricted); + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertTrue(_dp.isDirty()); + assertFalse(_dp._schemaChanged); + assertTrue(_dp.getPHI() == _pd.getPHI()); + + // Issue #18738 change the schema outside of a schema reload and verify that the column + // change the schema but don't mark the property as "Schema Import" + // this will allow whatever type changes the UI allows (text -> multiline, for example) + resetProperties(d, domainURI, c); + _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#double"); + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertTrue(_dp.isDirty()); + assertTrue(_dp._schemaChanged); + assertFalse(_dp.isRecreateRequired()); + assertTrue(Strings.CS.equals(_dp.getRangeURI(), _pd.getRangeURI())); + + // setting schema import to true will enable the _schemaChanged flag to toggle + // so it should be set true here + resetProperties(d, domainURI, c); + _dp.setSchemaImport(true); + _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#double"); + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertTrue(_dp.isDirty()); + assertTrue(_dp._schemaChanged); + assertTrue(_dp.isRecreateRequired()); + assertTrue(Strings.CS.equals(_dp.getRangeURI(), _pd.getRangeURI())); + + // verify no change when setting value to the same value as it was + resetProperties(d, domainURI, c); + _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#int"); + _pd.setPHI(PHI.NotPHI); + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertFalse(_dp.isDirty()); + assertFalse(_dp._schemaChanged); + assertFalse(_dp.isRecreateRequired()); + + // verify Lookup is set to null with null schema + resetProperties(d, domainURI, c); + verifyLookup(null, "lkSchema", null, true); + + // verify Lookup is set to null with null query + resetProperties(d, domainURI, c); + verifyLookup(null, null, "lkQuery",true); + + // verify Lookup is set to null with invalid container + resetProperties(d, domainURI, c); + verifyLookup("bogus", null, "lkQuery",true); + + // verify Lookup is set with valid schema and query + resetProperties(d, domainURI, c); + verifyLookup(null, "lkSchema", "lkQuery",true); + + // verify Lookup is set with valid container, schema and query + resetProperties(d, domainURI, c); + verifyLookup(c.getId(), "lkSchema1", "lkQuery2",true); + + // no cleanup as we never persisted anything + } + + private void verifyLookup(String containerId, String schema, String query, Boolean expectedDirty) + { + _pd.setLookupContainer(containerId); + _pd.setLookupQuery(query); + _pd.setLookupSchema(schema); + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertTrue(_dp.isDirty() == expectedDirty); + assertFalse(_dp._schemaChanged); + + // verify the lookup object returned + Lookup l = _dp.getLookup(); + + if (l == null) + { + // lookup can be null if we specified a containerId that is invalid or + // we specified a valid containerId (including null) but schema or query is null + if (containerId != null && null == ContainerManager.getForId(containerId)) + assertTrue(true); + else if (query == null || schema == null) + assertTrue(true); + else + assertTrue(false); + } + else + { + if (containerId != null) + assertTrue(Strings.CS.equals(l.getContainer().getId(), _pd.getLookupContainer())); + + assertTrue(Strings.CS.equals(l.getQueryName(), _pd.getLookupQuery())); + assertTrue(Strings.CS.equals(l.getSchemaKey().toString(), _pd.getLookupSchema())); + } + } + + private void resetProperties(Domain d, String domainUri, Container c) + { + _pd = getPropertyDescriptor(c, domainUri); + _dp = (DomainPropertyImpl) d.addProperty(); + _pd.copyTo(_dp.getPropertyDescriptor()); + } + + + private PropertyDescriptor getPropertyDescriptor(Container c, String domainURI) + { + PropertyDescriptor pd = new PropertyDescriptor(); + pd.setPropertyURI(domainURI + ":column"); + pd.setName("column"); + pd.setLabel("label"); + pd.setConceptURI(null); + pd.setRangeURI("http://www.w3.org/2001/XMLSchema#int"); + pd.setContainer(c); + pd.setDescription("description"); + pd.setURL(StringExpressionFactory.createURL((String)null)); + pd.setURLTarget(null); + pd.setImportAliases(null); + pd.setRequired(false); + pd.setHidden(false); + pd.setShownInInsertView(true); + pd.setShownInUpdateView(true); + pd.setShownInDetailsView(true); + pd.setDimension(false); + pd.setMeasure(true); + pd.setRecommendedVariable(false); + pd.setDefaultScale(DefaultScaleType.LINEAR); + pd.setFormat(null); + pd.setMvEnabled(false); + pd.setLookupContainer(c.getId()); + pd.setLookupSchema("lkSchema"); + pd.setLookupQuery("lkQuery"); + pd.setFacetingBehaviorType(FacetingBehaviorType.AUTOMATIC); + pd.setPHI(PHI.NotPHI); + pd.setExcludeFromShifting(false); + return pd; + } + } + + +} diff --git a/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java b/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java index 727423a1a70..8d588a3e4c1 100644 --- a/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java +++ b/experiment/src/org/labkey/experiment/api/property/StorageProvisionerImpl.java @@ -66,6 +66,7 @@ import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.PropertyColumn; import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; import org.labkey.api.exp.api.ExperimentUrls; import org.labkey.api.exp.api.StorageProvisioner; import org.labkey.api.exp.property.AbstractDomainKind; @@ -112,6 +113,8 @@ import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; + /** * Creates and maintains "hard" tables in the underlying database based on dynamically configured data types. * Will do CREATE TABLE and ALTER TABLE statements to make sure the table has the right set of requested columns. @@ -573,9 +576,35 @@ public void changePropertyType(Domain domain, DomainProperty prop) throws Change Set base = Sets.newCaseInsensitiveHashSet(); kind.getBaseProperties(domain).forEach(s -> base.add(s.getName())); + Map oldPropTypes = new HashMap<>(); if (!base.contains(prop.getName())) + { + if (prop instanceof DomainPropertyImpl dpi) + { + var oldPd = dpi._pdOld; + if (oldPd != null) + { + var newPd = dpi._pd; + if (oldPd.getPropertyType() == PropertyType.MULTI_CHOICE && TEXT_CHOICE_CONCEPT_URI.equals(newPd.getConceptURI())) + { + String sql = "SELECT COUNT(*) FROM " + kind.getStorageSchemaName() + "." + domain.getStorageTableName() + + " WHERE " + prop.getPropertyDescriptor().getStorageColumnName() + " IS NOT NULL AND " + + " array_length(" + prop.getPropertyDescriptor().getStorageColumnName() + ", 1) > 1"; + long count = new SqlSelector(scope, sql).getObject(Long.class); + if (count > 0) + { + throw new ChangePropertyDescriptorException("Unable to change property type. There are rows with multiple values stored for '" + prop.getName() + "'."); + } + } + oldPropTypes.put(prop.getName(), oldPd.getPropertyType()); + } + + } + propChange.addColumn(prop.getPropertyDescriptor()); + } + propChange.setOldPropertyTypes(oldPropTypes); propChange.execute(); } diff --git a/query/src/org/labkey/query/CustomViewQueryChangeListener.java b/query/src/org/labkey/query/CustomViewQueryChangeListener.java index d1c16e0fee3..135dad9dab1 100644 --- a/query/src/org/labkey/query/CustomViewQueryChangeListener.java +++ b/query/src/org/labkey/query/CustomViewQueryChangeListener.java @@ -20,6 +20,8 @@ import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.property.DomainProperty; import org.labkey.api.query.CustomView; import org.labkey.api.query.CustomViewChangeListener; import org.labkey.api.query.CustomViewInfo; @@ -28,6 +30,10 @@ import org.labkey.api.query.QueryService; import org.labkey.api.query.SchemaKey; import org.labkey.api.security.User; +import org.labkey.api.exp.PropertyType; + +import java.util.regex.Pattern; +import java.util.regex.Matcher; import org.springframework.mock.web.MockHttpServletRequest; import jakarta.servlet.http.HttpServletRequest; @@ -55,7 +61,7 @@ public void queryCreated(User user, Container container, ContainerFilter scope, } @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { if (property.equals(QueryProperty.Name)) { @@ -65,6 +71,64 @@ public void queryChanged(User user, Container container, ContainerFilter scope, { _updateCustomViewSchemaNameChange(user, container, changes); } + if (property.equals(QueryProperty.ColumnType)) + { + _updateCustomViewColumnTypeChange(user, container, schema, queryName, changes); + } + } + + + private void _updateCustomViewColumnTypeChange(User user, Container container, SchemaKey schema, String queryName, @NotNull Collection> changes) + { + for (QueryPropertyChange qpc : changes) + { + + PropertyDescriptor oldDp = (PropertyDescriptor) qpc.getOldValue(); + PropertyDescriptor newDp = (PropertyDescriptor) qpc.getNewValue(); + + if (oldDp == null || newDp == null) + continue; + + String columnName = newDp.getName() == null ? oldDp.getName() : newDp.getName(); + + List databaseCustomViews = QueryService.get().getDatabaseCustomViews(user, container, null, schema.toString(), queryName, false, false); + + for (CustomView customView : databaseCustomViews) + { + try + { + // update custom view filter and sort based on column type change + String filterAndSort = customView.getFilterAndSort(); + if (filterAndSort == null || filterAndSort.isEmpty()) + continue; + + /* Example: + * "/?filter.MCF2~arrayisnotempty=&filter.Name~in=S-5%3BS-6%3BS-8%3BS-9&filter.MCF~arraycontainsall=2%2C1%2C3&filter.sort=zz" + */ + String prefix = filterAndSort.startsWith("/?") ? "/?" : (filterAndSort.startsWith("?") ? "?" : ""); + String[] filterComponents = filterAndSort.substring(prefix.length()).split("&"); + StringBuilder updatedFilterAndSort = new StringBuilder(prefix); + for (String filterPart : filterComponents) + { + String updatedPart = QueryChangeListener.getUpdatedFilterStrOnColumnTypeUpdate(filterPart, columnName, oldDp.getPropertyType(), newDp.getPropertyType()); + updatedFilterAndSort.append(updatedPart); + } + + String updatedFilterAndSortStr = updatedFilterAndSort.toString(); + if (!updatedFilterAndSortStr.equals(filterAndSort)) + { + customView.setFilterAndSort(updatedFilterAndSortStr); + HttpServletRequest request = new MockHttpServletRequest(); + customView.save(customView.getModifiedBy(), request); + } + } + catch (Exception e) + { + LogManager.getLogger(CustomViewQueryChangeListener.class).error("An error occurred upgrading custom view properties: ", e); + } + } + } + } @Override diff --git a/query/src/org/labkey/query/QueryDefQueryChangeListener.java b/query/src/org/labkey/query/QueryDefQueryChangeListener.java index ec74cb74dc3..8f9ae84eae9 100644 --- a/query/src/org/labkey/query/QueryDefQueryChangeListener.java +++ b/query/src/org/labkey/query/QueryDefQueryChangeListener.java @@ -20,7 +20,7 @@ public void queryCreated(User user, Container container, ContainerFilter scope, {} @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { if (property.equals(QueryProperty.Name)) { diff --git a/query/src/org/labkey/query/QueryServiceImpl.java b/query/src/org/labkey/query/QueryServiceImpl.java index d44943d4f2a..0d49c024b5c 100644 --- a/query/src/org/labkey/query/QueryServiceImpl.java +++ b/query/src/org/labkey/query/QueryServiceImpl.java @@ -3268,7 +3268,13 @@ public void fireQueryCreated(User user, Container container, ContainerFilter sco @Override public void fireQueryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, QueryChangeListener.QueryProperty property, Collection> changes) { - QueryManager.get().fireQueryChanged(user, container, scope, schema, property, changes); + QueryManager.get().fireQueryChanged(user, container, scope, schema, null, property, changes); + } + + @Override + public void fireQueryColumnChanged(User user, Container container, @NotNull SchemaKey schemaPath, @NotNull String queryName, QueryChangeListener.QueryProperty property, Collection> changes) + { + QueryManager.get().fireQueryChanged(user, container, null, schemaPath, queryName, property, changes); } @Override diff --git a/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java b/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java index ef497ad7be1..4da70cab723 100644 --- a/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java +++ b/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java @@ -1,141 +1,141 @@ -/* - * Copyright (c) 2013-2017 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.query; - -import org.apache.logging.log4j.LogManager; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.snapshot.QuerySnapshotDefinition; -import org.labkey.api.security.User; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * QueryChangeListener for query snapshots. - * - * User: cnathe - * Date: 4/19/13 - */ -public class QuerySnapshotQueryChangeListener implements QueryChangeListener -{ - @Override - public void queryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) - { - } - - @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) - { - if (property.equals(QueryProperty.Name)) - { - _updateQuerySnapshotQueryNameChange(user, container, schema, changes); - } - if (property.equals(QueryProperty.SchemaName)) - { - _updateQuerySnapshotSchemaNameChange(user, container, changes); - } - } - - @Override - public void queryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) - { - } - - @Override - public Collection queryDependents(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) - { - // UNDONE - return Collections.emptyList(); - } - - private void _updateQuerySnapshotQueryNameChange(User user, Container container, SchemaKey schemaKey, Collection> changes) - { - // most property updates only care about the query name old value string and new value string - Map queryNameChangeMap = new HashMap<>(); - for (QueryPropertyChange qpc : changes) - { - String oldVal = (String)qpc.getOldValue(); - String newVal = (String)qpc.getNewValue(); - if (oldVal != null && !oldVal.equals(newVal)) - queryNameChangeMap.put((String)qpc.getOldValue(), (String)qpc.getNewValue()); - } - - for (QuerySnapshotDefinition qsd : QueryService.get().getQuerySnapshotDefs(container, schemaKey.toString())) - { - try - { - // update QueryTableName (stored in query.QuerySnapshotDef) - boolean changed = false; - String queryTableName = qsd.getQueryTableName(); - if (null != queryTableName && queryNameChangeMap.containsKey(queryTableName)) - { - qsd.setQueryTableName(queryNameChangeMap.get(queryTableName)); - changed = true; - } - String snapshotName = qsd.getName(); - if (null != snapshotName && queryNameChangeMap.containsKey(snapshotName)) - { - qsd.setName(queryNameChangeMap.get(snapshotName)); - changed = true; - } - if (changed) - qsd.save(qsd.getModifiedBy()); - } - catch (Exception e) - { - LogManager.getLogger(QuerySnapshotQueryChangeListener.class).error("An error occurred upgrading query snapshot properties: ", e); - } - } - } - - private void _updateQuerySnapshotSchemaNameChange(User user, Container container, Collection> changes) - { - Map schemaNameChangeMap = new HashMap<>(); - for (QueryPropertyChange qpc : changes) - { - if (qpc.getOldValue().equals(qpc.getNewValue())) - continue; - schemaNameChangeMap.put((String)qpc.getOldValue(), (String)qpc.getNewValue()); - } - - if (schemaNameChangeMap.isEmpty()) - return; - - for (String oldSchema : schemaNameChangeMap.keySet()) - { - String newSchema = schemaNameChangeMap.get(oldSchema); - for (QuerySnapshotDefinition qsd : QueryService.get().getQuerySnapshotDefs(container, oldSchema)) - { - qsd.setSchema(newSchema); - try - { - qsd.save(qsd.getModifiedBy()); - } - catch (Exception e) - { - LogManager.getLogger(QuerySnapshotQueryChangeListener.class).error("An error occurred upgrading query snapshot properties: ", e); - } - } - } - } -} +/* + * Copyright (c) 2013-2017 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.query; + +import org.apache.logging.log4j.LogManager; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.snapshot.QuerySnapshotDefinition; +import org.labkey.api.security.User; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * QueryChangeListener for query snapshots. + * + * User: cnathe + * Date: 4/19/13 + */ +public class QuerySnapshotQueryChangeListener implements QueryChangeListener +{ + @Override + public void queryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) + { + } + + @Override + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) + { + if (property.equals(QueryProperty.Name)) + { + _updateQuerySnapshotQueryNameChange(user, container, schema, changes); + } + if (property.equals(QueryProperty.SchemaName)) + { + _updateQuerySnapshotSchemaNameChange(user, container, changes); + } + } + + @Override + public void queryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) + { + } + + @Override + public Collection queryDependents(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) + { + // UNDONE + return Collections.emptyList(); + } + + private void _updateQuerySnapshotQueryNameChange(User user, Container container, SchemaKey schemaKey, Collection> changes) + { + // most property updates only care about the query name old value string and new value string + Map queryNameChangeMap = new HashMap<>(); + for (QueryPropertyChange qpc : changes) + { + String oldVal = (String)qpc.getOldValue(); + String newVal = (String)qpc.getNewValue(); + if (oldVal != null && !oldVal.equals(newVal)) + queryNameChangeMap.put((String)qpc.getOldValue(), (String)qpc.getNewValue()); + } + + for (QuerySnapshotDefinition qsd : QueryService.get().getQuerySnapshotDefs(container, schemaKey.toString())) + { + try + { + // update QueryTableName (stored in query.QuerySnapshotDef) + boolean changed = false; + String queryTableName = qsd.getQueryTableName(); + if (null != queryTableName && queryNameChangeMap.containsKey(queryTableName)) + { + qsd.setQueryTableName(queryNameChangeMap.get(queryTableName)); + changed = true; + } + String snapshotName = qsd.getName(); + if (null != snapshotName && queryNameChangeMap.containsKey(snapshotName)) + { + qsd.setName(queryNameChangeMap.get(snapshotName)); + changed = true; + } + if (changed) + qsd.save(qsd.getModifiedBy()); + } + catch (Exception e) + { + LogManager.getLogger(QuerySnapshotQueryChangeListener.class).error("An error occurred upgrading query snapshot properties: ", e); + } + } + } + + private void _updateQuerySnapshotSchemaNameChange(User user, Container container, Collection> changes) + { + Map schemaNameChangeMap = new HashMap<>(); + for (QueryPropertyChange qpc : changes) + { + if (qpc.getOldValue().equals(qpc.getNewValue())) + continue; + schemaNameChangeMap.put((String)qpc.getOldValue(), (String)qpc.getNewValue()); + } + + if (schemaNameChangeMap.isEmpty()) + return; + + for (String oldSchema : schemaNameChangeMap.keySet()) + { + String newSchema = schemaNameChangeMap.get(oldSchema); + for (QuerySnapshotDefinition qsd : QueryService.get().getQuerySnapshotDefs(container, oldSchema)) + { + qsd.setSchema(newSchema); + try + { + qsd.save(qsd.getModifiedBy()); + } + catch (Exception e) + { + LogManager.getLogger(QuerySnapshotQueryChangeListener.class).error("An error occurred upgrading query snapshot properties: ", e); + } + } + } + } +} diff --git a/query/src/org/labkey/query/persist/QueryManager.java b/query/src/org/labkey/query/persist/QueryManager.java index 28e10d84736..d9d5bda4a51 100644 --- a/query/src/org/labkey/query/persist/QueryManager.java +++ b/query/src/org/labkey/query/persist/QueryManager.java @@ -1,1177 +1,1177 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.query.persist; - -import org.apache.commons.collections4.Bag; -import org.apache.commons.collections4.bag.HashBag; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerType; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.FilterInfo; -import org.labkey.api.data.JsonWriter; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.ontology.Concept; -import org.labkey.api.ontology.OntologyService; -import org.labkey.api.query.AliasedColumn; -import org.labkey.api.query.CustomView; -import org.labkey.api.query.CustomViewChangeListener; -import org.labkey.api.query.CustomViewInfo; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryChangeListener.QueryPropertyChange; -import org.labkey.api.query.QueryDefinition; -import org.labkey.api.query.QueryParseException; -import org.labkey.api.query.QueryParseWarning; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.view.NotFoundException; -import org.labkey.query.ExternalSchema; -import org.labkey.query.ExternalSchemaDocumentProvider; -import org.springframework.jdbc.BadSqlGrammarException; - -import java.net.URISyntaxException; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.Function; -import java.util.stream.Collectors; - - -public class QueryManager -{ - private static final Logger _log = LogManager.getLogger(QueryManager.class); - private static final QueryManager instance = new QueryManager(); - private static final String SCHEMA_NAME = "query"; - private static final List QUERY_LISTENERS = new CopyOnWriteArrayList<>(); - private static final List VIEW_LISTENERS = new CopyOnWriteArrayList<>(); - - public static final int FLAG_INHERITABLE = 0x01; - public static final int FLAG_HIDDEN = 0x02; - public static final int FLAG_SNAPSHOT = 0x04; - - public static QueryManager get() - { - return instance; - } - - /** - * @param customQuery whether to look for custom queries or modified metadata on built-in tables - */ - public QueryDef getQueryDef(Container container, String schema, String name, boolean customQuery) - { - return QueryDefCache.getQueryDef(container, schema, name, customQuery); - } - - /** - * @param customQuery whether to look for custom queries or modified metadata on built-in tables - */ - public List getQueryDefs(Container container, @Nullable String schema, boolean inheritableOnly, boolean includeSnapshots, boolean customQuery) - { - return QueryDefCache.getQueryDefs(container, schema, inheritableOnly, includeSnapshots, customQuery); - } - - public Collection getQuerySnapshots(@Nullable Container container, @Nullable String schemaName) - { - return QuerySnapshotCache.getQuerySnapshotDefs(container, schemaName); - } - - public QuerySnapshotDef getQuerySnapshotDef(@NotNull Container container, @NotNull String schemaName, @NotNull String snapshotName) - { - return QuerySnapshotCache.getQuerySnapshotDef(container, schemaName, snapshotName); - } - - public QueryDef insert(User user, QueryDef queryDef) - { - QueryDef def = Table.insert(user, getTableInfoQueryDef(), queryDef); - QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); - return def; - } - - public QueryDef update(User user, QueryDef queryDef) - { - QueryDef def = Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); - QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); - return def; - } - - public void renameQuery(User user, Container container, String schema, String oldName, String newName) - { - QueryDef queryDef = getQueryDef(container, schema, oldName, false); - if (queryDef != null) - { - queryDef.setName(newName); - QueryDef def = Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); - QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); - } - } - - public void renameSchema(User user, Container container, String oldSchema, String newSchema) - { - List queryDefs = getQueryDefs(container, oldSchema, false, false, false); - for (QueryDef queryDef : queryDefs) - { - queryDef.setSchema(newSchema); - Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); - } - QueryDefCache.uncache(ContainerManager.getForId(container.getId())); - } - - public void delete(QueryDef queryDef) - { - Table.delete(getTableInfoQueryDef(), queryDef.getQueryDefId()); - QueryDefCache.uncache(ContainerManager.getForId(queryDef.getContainerId())); - } - - public void delete(QuerySnapshotDef querySnapshotDef) - { - Table.delete(getTableInfoQuerySnapshotDef(), querySnapshotDef.getRowId()); - QuerySnapshotCache.uncache(querySnapshotDef); - if (querySnapshotDef.getQueryDefId() != null) - { - Table.delete(getTableInfoQueryDef(), querySnapshotDef.getQueryDefId()); - QueryDefCache.uncache(querySnapshotDef.lookupContainer()); - } - } - - public QuerySnapshotDef insert(User user, QueryDef queryDef, QuerySnapshotDef snapshotDef) - { - if (queryDef != null && snapshotDef.getQueryTableName() == null) - { - QueryDef def = insert(user, queryDef); - snapshotDef.setQueryDefId(def.getQueryDefId()); - } - snapshotDef = Table.insert(user, getTableInfoQuerySnapshotDef(), snapshotDef); - QuerySnapshotCache.uncache(snapshotDef); - return snapshotDef; - } - - public QuerySnapshotDef update(User user, QueryDef queryDef, QuerySnapshotDef snapshotDef) - { - if (queryDef != null && snapshotDef.getQueryTableName() == null) - update(user, queryDef); - snapshotDef = Table.update(user, getTableInfoQuerySnapshotDef(), snapshotDef, snapshotDef.getRowId()); - QuerySnapshotCache.uncache(snapshotDef); - return snapshotDef; - } - - // Does not use the cache... but only used at save time - public QuerySnapshotDef getQuerySnapshotDef(int id) - { - return new TableSelector(getTableInfoQuerySnapshotDef()).getObject(id, QuerySnapshotDef.class); - } - - public CstmView getCustomView(Container container, int id) - { - CstmView view = CustomViewCache.getCstmView(container, id); - _log.debug(view); - return view; - } - - public CstmView getCustomView(Container container, String entityId) - { - CstmView view = CustomViewCache.getCstmViewByEntityId(container, entityId); - _log.debug(view); - return view; - } - - /** - * Get all shared custom views that are applicable. - * If inheritable is true, custom views from parent and Shared container are included. - * - */ - public List getAllSharedCstmViews(Container container, String schemaName, String queryName, boolean inheritable) - { - return getAllCstmViews(container, schemaName, queryName, null, inheritable, true); - } - - /** - * Get all custom views that are applicable for this user including shared custom views. - * If inheritable is true, custom views from parent and Shared container are included. - * - * @param container The current container. - * @param schemaName The schema name or null for all schemas. - * @param queryName The query name or null for all queries in the schema. - * @param owner The owner or null for all views (shared or owned by someone.) - * @param inheritable If true, look up container hierarchy and in Shared project for custom views. - * @param sharedOnly If true, ignore the user parameter and only include shared custom views. - * @return List of custom views entities in priority order. - */ - public List getAllCstmViews(Container container, String schemaName, String queryName, @Nullable User owner, boolean inheritable, boolean sharedOnly) - { - List views = new ArrayList<>(); - - getCstmViewsInContainer(views, container, schemaName, queryName, owner, false, sharedOnly); - if (!container.isContainerFor(ContainerType.DataType.customQueryViews)) - { - getCstmViewsInContainer(views, container.getContainerFor(ContainerType.DataType.customQueryViews), schemaName, queryName, owner, false, sharedOnly); - } - - if (!inheritable) - return views; - - Container containerCur = container.getParent(); - while (containerCur != null && !containerCur.isRoot()) - { - getCstmViewsInContainer(views, containerCur, schemaName, queryName, owner, true, sharedOnly); - containerCur = containerCur.getParent(); - } - - // look in the shared project - getCstmViewsInContainer(views, ContainerManager.getSharedContainer(), schemaName, queryName, owner, true, sharedOnly); - - return views; - } - - private void getCstmViewsInContainer(List views, Container container, String schemaName, String queryName, @Nullable User user, boolean inheritable, boolean sharedOnly) - { - if (sharedOnly) - { - // Get only shared custom views - views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, true)); - } - else - { - if (user != null) - { - // Custom views owned by the user first, then add shared custom views - views.addAll(getCstmViews(container, schemaName, queryName, null, user, inheritable, false)); - views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, true)); - } - else - { - // Get all custom views regardless of owner - views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, false)); - } - } - } - - public List getCstmViews(Container container, @Nullable String schemaName, @Nullable String queryName, @Nullable String viewName, @Nullable User user, boolean inheritableOnly, boolean sharedOnly) - { - return CustomViewCache.getCstmViews(container, schemaName, queryName, viewName, user, inheritableOnly, sharedOnly); - } - - public CstmView update(User user, CstmView view) - { - CstmView cstmView = Table.update(user, getTableInfoCustomView(), view, view.getCustomViewId()); - CustomViewCache.uncache(ContainerManager.getForId(cstmView.getContainerId())); - - return cstmView; - } - - public CstmView insert(User user, CstmView view) - { - CstmView cstmView = Table.insert(user, getTableInfoCustomView(), view); - CustomViewCache.uncache(ContainerManager.getForId(cstmView.getContainerId())); - - return cstmView; - } - - public void delete(CstmView view) - { - Table.delete(getTableInfoCustomView(), view.getCustomViewId()); - CustomViewCache.uncache(ContainerManager.getForId(view.getContainerId())); - } - - @Nullable - public ExternalSchemaDef getExternalSchemaDef(Container c, int rowId) - { - return ExternalSchemaDefCache.getSchemaDef(c, rowId, ExternalSchemaDef.class); - } - - @NotNull - public List getExternalSchemaDefs(@Nullable Container container) - { - return ExternalSchemaDefCache.getSchemaDefs(container, ExternalSchemaDef.class); - } - - @Nullable - public ExternalSchemaDef getExternalSchemaDef(Container container, @Nullable String userSchemaName) - { - return ExternalSchemaDefCache.getSchemaDef(container, userSchemaName, ExternalSchemaDef.class); - } - - @Nullable - public LinkedSchemaDef getLinkedSchemaDef(Container c, int rowId) - { - return ExternalSchemaDefCache.getSchemaDef(c, rowId, LinkedSchemaDef.class); - } - - @NotNull - public List getLinkedSchemaDefs(@Nullable Container c) - { - return ExternalSchemaDefCache.getSchemaDefs(c, LinkedSchemaDef.class); - } - - @Nullable - public LinkedSchemaDef getLinkedSchemaDef(Container c, @Nullable String userSchemaName) - { - return ExternalSchemaDefCache.getSchemaDef(c, userSchemaName, LinkedSchemaDef.class); - } - - public void delete(@NotNull AbstractExternalSchemaDef def) - { - Container c = def.lookupContainer(); - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(getTableInfoExternalSchema().getColumn("ExternalSchemaId"), def.getExternalSchemaId()); - Table.delete(getTableInfoExternalSchema(), filter); - updateExternalSchemas(def.lookupContainer()); - } - - public LinkedSchemaDef insertLinkedSchema(User user, LinkedSchemaDef def) - { - LinkedSchemaDef newDef = Table.insert(user, getTableInfoExternalSchema(), def); - updateExternalSchemas(def.lookupContainer()); - return newDef; - } - - public void deleteLinkedSchema(Container container, String userSchemaName) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromString("UserSchemaName"), userSchemaName); - filter.addCondition(FieldKey.fromString("SchemaType"), AbstractExternalSchemaDef.SchemaType.linked); - Table.delete(getTableInfoExternalSchema(), filter); - updateExternalSchemas(container); - } - - // Uncaches and re-indexes all external schemas in a container. Called any time an external schema or linked schema - // changes in any way (insert/update/delete). - public void updateExternalSchemas(Container c) - { - QueryService.get().updateLastModified(); - if (null != c) - { - ExternalSchemaDefCache.uncache(c); - ExternalSchemaDocumentProvider.getInstance().enumerateDocuments(SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified), null); - } - } - - public void reloadAllExternalSchemas(Container c) - { - getExternalSchemaDefs(c).forEach(this::reloadExternalSchema); - } - - public void reloadExternalSchema(ExternalSchemaDef def) - { - ExternalSchema.uncache(def); - } - - public boolean canInherit(int flag) - { - return (flag & FLAG_INHERITABLE) != 0; - } - - public int setCanInherit(int flag, boolean f) - { - if (f) - { - return flag | FLAG_INHERITABLE; - } - else - { - return flag & ~FLAG_INHERITABLE; - } - } - - public boolean isHidden(int flag) - { - return (flag & FLAG_HIDDEN) != 0; - } - - public int setIsHidden(int flag, boolean f) - { - if (f) - { - return flag | FLAG_HIDDEN; - } - else - { - return flag & ~FLAG_HIDDEN; - } - } - - public boolean isSnapshot(int flag) - { - return (flag & FLAG_SNAPSHOT) != 0; - } - - public int setIsSnapshot(int flag, boolean f) - { - if (f) - { - return flag | FLAG_SNAPSHOT; - } - else - { - return flag & ~FLAG_SNAPSHOT; - } - } - - public String getDbSchemaName() - { - return SCHEMA_NAME; - } - - public DbSchema getDbSchema() - { - return DbSchema.get(SCHEMA_NAME, DbSchemaType.Module); - } - - public TableInfo getTableInfoQueryDef() - { - return getDbSchema().getTable("QueryDef"); - } - - public TableInfo getTableInfoQuerySnapshotDef() - { - return getDbSchema().getTable("QuerySnapshotDef"); - } - - public TableInfo getTableInfoCustomView() - { - return getDbSchema().getTable("CustomView"); - } - - public TableInfo getTableInfoExternalSchema() - { - return getDbSchema().getTable("ExternalSchema"); - } - - public TableInfo getTableInfoOlapDef() - { - return getDbSchema().getTable("OlapDef"); - } - - public void containerDeleted(Container c) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - Table.delete(getTableInfoQuerySnapshotDef(), filter); - QuerySnapshotCache.uncache(c); - Table.delete(getTableInfoCustomView(), filter); - CustomViewCache.uncache(c); - Table.delete(getTableInfoQueryDef(), filter); - QueryDefCache.uncache(c); - Table.delete(getTableInfoExternalSchema(), filter); - ExternalSchemaDefCache.uncache(c); - Table.delete(getTableInfoOlapDef(), filter); - } - - public void addQueryListener(QueryChangeListener listener) - { - QUERY_LISTENERS.add(listener); - } - - public void removeQueryListener(QueryChangeListener listener) - { - QUERY_LISTENERS.remove(listener); - } - - public void fireQueryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) - { - QueryService.get().updateLastModified(); - for (QueryChangeListener l : QUERY_LISTENERS) - l.queryCreated(user, container, scope, schema, queries); - } - - public void fireQueryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryChangeListener.QueryProperty property, @NotNull Collection> changes) - { - QueryService.get().updateLastModified(); - assert checkChanges(property, changes); - for (QueryChangeListener l : QUERY_LISTENERS) - l.queryChanged(user, container, scope, schema, property, changes); - } - - // Checks all changes have the correct property and type. - private boolean checkChanges(QueryChangeListener.QueryProperty property, Collection> changes) - { - if (property == null) - { - _log.error("Null property not allowed."); - return false; - } - - boolean valid = true; - for (QueryPropertyChange change : changes) - { - if (change.getProperty() != property) - { - _log.error(String.format("Property '%s' doesn't match change property '%s'", property, change.getProperty())); - valid = false; - } - if (change.getOldValue() != null && !property.getPropertyClass().isInstance(change.getOldValue())) - { - _log.error(String.format("Old value '%s' isn't an instance of property '%s' class '%s'", change.getOldValue(), property, property.getPropertyClass())); - valid = false; - } - if (change.getNewValue() != null && !property.getPropertyClass().isInstance(change.getNewValue())) - { - _log.error(String.format("New value '%s' isn't an instance of property '%s' class '%s'", change.getNewValue(), property, property.getPropertyClass())); - valid = false; - } - } - return valid; - } - - public void fireQueryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries) - { - QueryService.get().updateLastModified(); - for (QueryChangeListener l : QUERY_LISTENERS) - l.queryDeleted(user, container, scope, schema, queries); - } - - public Collection getQueryDependents(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries) - { - ArrayList dependents = new ArrayList<>(); - for (QueryChangeListener l : QUERY_LISTENERS) - dependents.addAll(l.queryDependents(user, container, scope, schema, queries)); - return dependents; - } - - public void addCustomViewListener(CustomViewChangeListener listener) - { - VIEW_LISTENERS.add(listener); - } - - public void removeCustomViewListener(CustomViewChangeListener listener) - { - VIEW_LISTENERS.remove(listener); - } - - public void fireViewCreated(CustomView view) - { - QueryService.get().updateLastModified(); - for (CustomViewChangeListener l : VIEW_LISTENERS) - l.viewCreated(view); - } - - public void fireViewChanged(CustomView view) - { - QueryService.get().updateLastModified(); - for (CustomViewChangeListener l : VIEW_LISTENERS) - l.viewChanged(view); - } - - public void fireViewDeleted(CustomView view) - { - QueryService.get().updateLastModified(); - for (CustomViewChangeListener l : VIEW_LISTENERS) - l.viewDeleted(view); - } - - public Collection getViewDepedents(CustomView view) - { - ArrayList dependents = new ArrayList<>(); - for (CustomViewChangeListener l : VIEW_LISTENERS) - dependents.addAll(l.viewDependents(view)); - return dependents; - } - - static public final ContainerManager.ContainerListener CONTAINER_LISTENER = new ContainerManager.ContainerListener() - { - @Override - public void containerDeleted(Container c, User user) - { - QueryManager.get().containerDeleted(c); - } - }; - - public boolean validateQuery(SchemaKey schemaPath, String queryName, User user, Container container, @NotNull List errors, - @NotNull List warnings) - { - UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); - if (null == schema) - throw new IllegalArgumentException("Could not find the schema '" + schemaPath.toDisplayString() + "'!"); - - TableInfo table = schema.getTable(queryName); - if (null == table) - throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schemaPath.toDisplayString() + "'!"); - - return validateQuery(table, true, errors, warnings); - } - - public boolean validateQuery(TableInfo table, boolean testAllColumns, @NotNull List errors, - @NotNull List warnings) - { - errors.addAll(table.getWarnings()); - - Collection params = table.getNamedParameters(); - Map parameters = new HashMap<>(); - for (QueryService.ParameterDecl p : params) - { - if (!p.isRequired()) - continue; - parameters.put(p.getName(), null); - } - - TableSelector selector; - - // Note this check had been inverted for years, but was fixed in 14.1. Previously, testAllColumns == true meant - // the default column list was computed but discarded, and testAllColumns == false was completely broken - if (testAllColumns) - { - selector = new TableSelector(table); - } - else - { - List defVisCols = table.getDefaultVisibleColumns(); - Map colMap = QueryService.get().getColumns(table, defVisCols); - List cols = new ArrayList<>(colMap.values()); - - selector = new TableSelector(table, cols, null, null); - } - - // set forDisplay to mimic the behavior one would get in the UI - // try to execute with a rowcount of 0 (will throw SQLException to client if it fails) - selector.setForDisplay(true).setNamedParameters(parameters).setMaxRows(Table.NO_ROWS); - - //noinspection EmptyTryBlock,UnusedDeclaration - try (ResultSet rs = selector.getResultSet()) - { - } - catch (SQLException e) - { - errors.add(new QueryParseException(e.getMessage(), e, 0, 0)); - } - catch (BadSqlGrammarException e) - { - errors.add(new QueryParseException(e.getSQLException().getMessage(), e, 0, 0)); - } - - UserSchema schema = table.getUserSchema(); - if (schema != null) - { - QueryDefinition queryDef = schema.getQueryDef(table.getName()); - if (queryDef != null) - { - queryDef.validateQuery(schema, errors, warnings); - } - } - - OntologyService os = OntologyService.get(); - if (null != os) - { - for (var col : table.getColumns()) - { - String code = col.getPrincipalConceptCode(); - if (null != code) - { - Concept concept = os.resolveCode(code); - if (null == concept) - warnings.add(new QueryParseException("Concept not found: " + code, null, 0, 0)); - } - } - } - - return errors.isEmpty(); - } - - /** - * Experimental. The goal is to provide a more thorough validation of query metadata, including warnings of potentially - * invalid conditions, like autoincrement columns set userEditable=true. - */ - public boolean validateQueryMetadata(SchemaKey schemaPath, String queryName, User user, Container container, - @NotNull List errors, @NotNull List warnings) - { - Set columns = new HashSet<>(); - UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); - if (null == schema) - throw new IllegalArgumentException("Could not find the schema '" + schemaPath.getName() + "'!"); - - TableInfo table = schema.getTable(queryName); - if (null == table) - throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schemaPath.getName() + "'!"); - - if (table.isPublic() && table.getPublicSchemaName() != null && !schemaPath.toString().equalsIgnoreCase(table.getPublicSchemaName())) - warnings.add(new QueryParseWarning("(metadata) TableInfo.getPublicSchemaName() does not match: set to '" + table.getPublicSchemaName() + "', expected '" + schemaPath + "'", null, 0,0)); - - try - { - //validate foreign keys and other metadata warnings - columns.addAll(table.getColumns()); - columns.addAll(QueryService.get().getColumns(table, table.getDefaultVisibleColumns()).values()); - } - catch(QueryParseException e) - { - errors.add(e); - } - - for (ColumnInfo col : columns) - { - validateColumn(col, user, container, table, errors, warnings); - } - - return errors.isEmpty(); - } - - /** - * Experimental. See validateQueryMetadata() - */ - private boolean validateColumn(ColumnInfo col, User user, Container container, @Nullable TableInfo parentTable, - @NotNull List errors, @NotNull List warnings) - { - if(parentTable == null) - parentTable = col.getParentTable(); - - String publicSchema = col.getParentTable().getPublicSchemaName() != null ? col.getParentTable().getPublicSchemaName() : col.getParentTable().getSchema().toString(); - String publicQuery = col.getParentTable().getPublicName() != null ? col.getParentTable().getPublicName() : col.getParentTable().getName(); - String errorBase = "(metadata) for column '" + col.getFieldKey() + "' in " + publicSchema + "." + publicQuery + ": "; - - validateFk(col, user, container, parentTable, errors, warnings, errorBase); - - Set specialCols = new CaseInsensitiveHashSet(); - specialCols.add("LSID"); - specialCols.add("entityId"); - specialCols.add("container"); - specialCols.add("created"); - specialCols.add("createdby"); - specialCols.add("modified"); - specialCols.add("modifiedby"); - - if(specialCols.contains(col.getName())) - { - if(col.isUserEditable()) - warnings.add(new QueryParseWarning(errorBase + " column is user editable, which is not expected based on its name", null, 0,0)); - if(col.isShownInInsertView()) - warnings.add(new QueryParseWarning(errorBase + " column has shownInInsertView set to true, which is not expected based on its name", null, 0, 0)); - if(col.isShownInUpdateView()) - warnings.add(new QueryParseWarning(errorBase + " column has shownInUpdateView set to true, which is not expected based on its name", null, 0, 0)); - } - - if(col.isAutoIncrement() && col.isUserEditable()) - warnings.add(new QueryParseException(errorBase + " column is autoIncrement, but has userEditable set to true", null, 0, 0)); - if(col.isAutoIncrement() && col.isShownInInsertView()) - warnings.add(new QueryParseWarning(errorBase + " column is autoIncrement, but has shownInInsertView set to true", null, 0, 0)); - if(col.isAutoIncrement() && col.isShownInUpdateView()) - warnings.add(new QueryParseWarning(errorBase + " column is autoIncrement, but has shownInUpdateView set to true", null, 0, 0)); - - try - { - if (StringUtils.isNotBlank(col.getDisplayWidth()) && Integer.parseInt(col.getDisplayWidth()) > 200 && !"textarea".equalsIgnoreCase(col.getInputType())) - { - if (col.isUserEditable() && col.getJdbcType() != null && col.getJdbcType().getJavaClass() == String.class) - warnings.add(new QueryParseWarning(errorBase + " column has a displayWidth > 200, but does not use a textarea as the inputType", null, 0, 0)); - } - } - catch (NumberFormatException e) - { - warnings.add(new QueryParseWarning(errorBase + " column has invalid value for displayWidth: '" + col.getDisplayWidth() + "'", null, 0, 0)); - } - return errors.isEmpty(); - } - - /** - * Experimental. See validateQueryMetadata() - */ - private boolean validateFk(ColumnInfo col, User user, Container container, TableInfo parentTable, - @NotNull List errors, @NotNull List warnings, - String errorBase) - - { - //NOTE: this is the same code that writes JSON to the client - JSONObject o = JsonWriter.getLookupInfo(col, false); - if (o == null) - return true; - - boolean isPublic = o.getBoolean("isPublic"); - SchemaKey schemaPath = SchemaKey.fromString(o.optString("schemaName")); - String queryName = o.optString("queryName"); - if (queryName == null) - { - // Likely a lookup that targets something not exposed via a UserSchema. Bail out without further validation - return true; - } - String displayColumn = o.optString("displayColumn"); - String keyColumn = o.optString("keyColumn"); - String containerPath = o.optString("containerPath"); - - Container lookupContainer = containerPath == null ? container : ContainerManager.getForPath(containerPath); - if (lookupContainer == null) - { - warnings.add(new QueryParseWarning(errorBase + " Unable to find container" + containerPath, null, 0, 0)); - } - - //String publicSchema = col.getParentTable().getPublicSchemaName() != null ? col.getParentTable().getPublicSchemaName() : col.getParentTable().getSchema().toString(); - //String publicQuery = col.getParentTable().getPublicName() != null ? col.getParentTable().getPublicName() : col.getParentTable().getName(); - if (col.getFk() == null) - return errors.isEmpty(); - - if (!isPublic) - { - warnings.add(new QueryParseWarning(errorBase + " has a lookup to a non-public table: " + (schemaPath == null ? "" : schemaPath.toDisplayString()) + "." + queryName, null, 0, 0)); - return errors.isEmpty(); - } - - UserSchema userSchema = QueryService.get().getUserSchema(user, lookupContainer, schemaPath); - if (userSchema == null) - { - warnings.add(new QueryParseWarning(errorBase + " unable to find the user schema: " + schemaPath.toDisplayString(), null, 0, 0)); - return errors.isEmpty(); - } - - TableInfo fkTable = userSchema.getTable(queryName); - if(fkTable == null) - { - warnings.add(new QueryParseWarning(errorBase + " has a lookup to a table that does not exist: " + schemaPath.toDisplayString() + "." + queryName, null, 0, 0)); - return errors.isEmpty(); - } - - //a FK can have a table non-visible to the client, so long as public is set to false - if (fkTable.isPublic()){ - String fkt = schemaPath.toDisplayString() + "." + queryName; - - QueryManager.get().validateQuery(schemaPath, queryName, user, lookupContainer, errors, warnings); - if (displayColumn != null) - { - FieldKey displayFieldKey = FieldKey.fromString(displayColumn); - Map cols = QueryService.get().getColumns(fkTable, Collections.singleton(displayFieldKey)); - if (!cols.containsKey(displayFieldKey)) - { - warnings.add(new QueryParseWarning(errorBase + " reports a foreign key with displayColumn of " + displayColumn + " in the table " + schemaPath.toDisplayString() + "." + queryName + ", but the column does not exist", null, 0, 0)); - } - else - { - ColumnInfo ci = cols.get(displayFieldKey); - if (!displayColumn.equals(ci.getFieldKey().toString())) - { - warnings.add(new QueryParseWarning(errorBase + ", the lookup to " + schemaPath.toDisplayString() + "." + queryName + "' did not match the expected case, which was '" + ci.getFieldKey().toString() + "'. Actual: '" + displayColumn + "'", null, 0, 0)); - } - } - } - - if (keyColumn != null) - { - FieldKey keyFieldKey = FieldKey.fromString(keyColumn); - Map cols = QueryService.get().getColumns(fkTable, Collections.singleton(keyFieldKey)); - if (!cols.containsKey(keyFieldKey)) - { - warnings.add(new QueryParseException(errorBase + " reports a foreign key with keyColumn of " + keyColumn + " in the table " + schemaPath.toDisplayString() + "." + queryName + ", but the column does not exist", null, 0, 0)); - } - else - { - ColumnInfo ci = cols.get(keyFieldKey); - if (!keyColumn.equals(ci.getFieldKey().toString())) - { - warnings.add(new QueryParseWarning(errorBase + ", the lookup to " + schemaPath.toDisplayString() + "." + queryName + "' did not match the expected case, which was '" + ci.getFieldKey().toString() + "'. Actual: '" + keyColumn + "'", null, 0, 0)); - } - } - } - else - { - warnings.add(new QueryParseWarning(errorBase + ", there is a lookup where the keyColumn is blank", null, 0, 0)); - } - } - - return errors.isEmpty(); - } - - /** - * Experimental. The goal is to provide a more thorough validation of saved views, including errors like invalid - * column names or case errors (which cause problems for case-sensitive js) - */ - public boolean validateQueryViews(SchemaKey schemaPath, String queryName, User user, Container container, - @NotNull List errors, @NotNull List warnings) throws QueryParseException - { - UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); - if (null == schema) - throw new IllegalArgumentException("Could not find the schema '" + schemaPath.getName() + "'!"); - - TableInfo table = schema.getTable(queryName); - if (null == table) - throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schema.getSchemaName() + "'!"); - - //validate views - try - { - List views = QueryService.get().getCustomViews(user, container, null, schema.getSchemaName(), queryName, true); - for (CustomView v : views) - { - validateViewColumns(user, container, v, "columns", v.getColumns(), table, errors, warnings); - - if (!StringUtils.isEmpty(v.getFilterAndSort())) - { - try - { - CustomViewInfo.FilterAndSort fs = CustomViewInfo.FilterAndSort.fromString(v.getFilterAndSort()); - List filterCols = new ArrayList<>(); - for (FilterInfo f : fs.getFilter()) - { - filterCols.add(f.getField()); - } - validateViewColumns(user, container, v, "filter", filterCols, table, errors, warnings); - - List sortCols = new ArrayList<>(); - for (Sort.SortField f : fs.getSort()) - { - sortCols.add(f.getFieldKey()); - } - validateViewColumns(user, container, v, "sort", sortCols, table, errors, warnings); - - } - catch (URISyntaxException e) - { - warnings.add(new QueryParseWarning("unable to process the filter/sort section of view: " + v.getName(), null, 0, 0)); - } - } - } - } - catch (NotFoundException e) - { - errors.add(new QueryParseException("Cannot get views: ", e, 0, 0)); - } - - - return errors.isEmpty(); - } - - private void validateViewColumns(User user, Container container, CustomView v, String identifier, List viewCols, TableInfo sourceTable, - @NotNull List errors, @NotNull List warnings) throws QueryParseException - { - //verify columns match, accounting for case - Map colMap = QueryService.get().getColumns(sourceTable, viewCols); - - for (FieldKey f : viewCols) - { - boolean found = false; - boolean matchCase = false; - FieldKey fk = null; - ColumnInfo c = colMap.get(f); - if(c != null) - { - found = true; - fk = c.getFieldKey(); - if(c instanceof AliasedColumn) - fk = ((AliasedColumn)c).getColumn().getFieldKey(); - - if(fk.toString().equals(f.toString())) - { - matchCase = true; - } - } - - if (!found){ - warnings.add(new QueryParseWarning("In the saved view '" + (v.getName() == null ? "default" : v.getName()) + "', in the " + identifier + " section, the column '" + f.toString() + "' in " + v.getSchemaName() + "." + v.getQueryName() + " could not be matched to a column", null, 0, 0)); - continue; - } - - if (!matchCase){ - warnings.add(new QueryParseWarning("In the saved view '" + (v.getName() == null ? "default" : v.getName()) + "', in the " + identifier + " section, the column '" + f + "' in " + v.getSchemaName() + "." + v.getQueryName() + "' did not match the expected case, which was '" + fk + "'", null, 0, 0)); - } - - //queryErrors.addAll(validateColumn(c, user, container)); - } - } - - public static void registerUsageMetrics(String moduleName) - { - UsageMetricsService svc = UsageMetricsService.get(); - if (null != svc) - { - svc.registerUsageMetrics(moduleName, () -> { - Bag bag = DbScope.getDbScopes().stream() - .filter(scope -> !scope.isLabKeyScope()).map(DbScope::getDatabaseProductName) - .collect(Collectors.toCollection(HashBag::new)); - - Map statsMap = bag.uniqueSet().stream() - .collect(Collectors.toMap(Function.identity(), bag::getCount)); - - return Map.of("externalDatasources", statsMap, - "customViewCounts", - Map.of( - "DataClasses", getSchemaCustomViewCounts("exp.data"), - "SampleTypes", getSchemaCustomViewCounts("samples"), - "Assays", getSchemaCustomViewCounts("assay"), - "Inventory", getSchemaCustomViewCounts("inventory") - ), - "customViewWithLineageColumn", getLineageCustomViewMetrics(), - "queryDefWithCalculatedFieldsCounts", getCalculatedFieldsCountsMetric() - ); - }); - } - } - - private static Map getCalculatedFieldsCountsMetric() - { - DbSchema dbSchema = CoreSchema.getInstance().getSchema(); - return new SqlSelector(dbSchema, - new SQLFragment("SELECT \"schema\", COUNT(*) AS count FROM (\n" + - " SELECT CASE WHEN \"schema\" LIKE 'assay.%' THEN 'assay' ELSE \"schema\" END AS \"schema\" FROM query.querydef WHERE metadata LIKE '%(), (x, m) -> { - x.put(m.get("schema").toString(), m.get("count")); - return x; - }); - } - - private static Map getSchemaCustomViewCounts(String schema) - { - DbSchema dbSchema = DbSchema.get("query"); - TableInfo customView = dbSchema.getTable("customview"); - var schemaField = customView.getColumn("schema").getSelectIdentifier(); - SQLFragment schemaClause; - if (schema.equalsIgnoreCase("assay")) - schemaClause = new SQLFragment("C.").appendIdentifier(schemaField).append(" LIKE 'assay.%'"); - else - schemaClause = new SQLFragment("C.").appendIdentifier(schemaField).append(" = ").appendValue(schema); - return Map.of( - "defaultOverrides", new SqlSelector(dbSchema, - new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags < 2 AND C.name IS NULL")).getObject(Long.class), // possibly inheritable, no hidden, not snapshot - "inheritable", new SqlSelector(dbSchema, - new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags = 1")).getObject(Long.class), // inheritable, not hidden, not snapshot - "namedViews", new SqlSelector(dbSchema, - new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags < 2 AND C.name IS NOT NULL")).getObject(Long.class), // possibly inheritable, no hidden, not snapshot - "shared", new SqlSelector(dbSchema, - new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.customviewowner IS NULL")).getObject(Long.class), - "identifyingFieldsViews", new SqlSelector(dbSchema, - new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.name = '~~identifyingfields~~'")).getObject(Long.class) - ); - } - - - private static Long percentile(double percentile, List sortedCounts) { - if (percentile <= 0.01) - return sortedCounts.get(0); - if (percentile >= 99.99) - return sortedCounts.get(sortedCounts.size() - 1); - return sortedCounts.get((int) Math.round(percentile / 100.0 * (sortedCounts.size() - 1))); - } - - /** - * customViewsCountWithLineageCol: total number of non-hidden saved custom views that has at least one input/output/ancestor column - * customViewsCountWithAncestorCol: total number of non-hidden saved custom views that has at least one ancestor column - * totalLineageColumnsInAllViews: total number of input/output/ancestor columns defined for all saved non-hidden custom views - * totalAncestorColumnsInAllViews: total number of ancestor columns defined for all saved non-hidden custom views - * lineageColumnsCountMin: the minimum count of input/output/ancestor columns in any view with such column - * lineageColumnsCount25: the 25 percentile count of input/output/ancestor columns in all views with such column - * lineageColumnsCount50: the 50 percentile / median count of input/output/ancestor columns in all views with such column - * lineageColumnsCount75: the 75 percentile count of input/output/ancestor columns in all views with such column - * lineageColumnsCountMax: the maximum count of input/output/ancestor columns in any view with such column - * lineageColumnsCountAvg: the average count of input/output/ancestor columns in any view with such column - * ancestorColumnsCountMin: the minimum count of ancestor columns in any view with ancestor columns - * ancestorColumnsCount25: the 25 percentile count of ancestor columns in all views with ancestor columns - * ancestorColumnsCount50: the 50 percentile / median count of ancestor columns in all views with ancestor columns - * ancestorColumnsCount75: the 75 percentile count of ancestor columns in all views with ancestor columns - * ancestorColumnsCountMax: the maximum count of ancestor columns in any view with ancestor columns - * ancestorColumnsCountAvg: the average count of ancestor columns in any view with ancestor columns - */ - private static Map getLineageCustomViewMetrics() - { - List ancestorColCounts = new ArrayList<>(); - List lineageColCounts = new ArrayList<>(); - final String ANCESTOR_PREFIX = "ancestors/"; - final String INPUT_PREFIX = "inputs/"; - final String OUTPUT_PREFIX = "outputs/"; - - Map metrics = new HashMap<>(); - - DbSchema schema = DbSchema.get("query", DbSchemaType.Module); - SqlDialect sqlDialect = schema.getSqlDialect(); - SQLFragment sql = new SQLFragment() - .append("SELECT columns FROM query.customview WHERE flags < 2 AND (columns LIKE ? OR columns LIKE ? OR columns LIKE ?)") - .add("%" + sqlDialect.encodeLikeOpSearchString("Ancestors%2F") + "%") - .add("%" + sqlDialect.encodeLikeOpSearchString("Inputs%2F") + "%") - .add("%" + sqlDialect.encodeLikeOpSearchString("Outputs%2F") + "%"); - List viewsColumnStrs = new SqlSelector(schema, sql).getArrayList(String.class); - - for (String columnStr : viewsColumnStrs) - { - long lineageColCount = 0L; - long ancestorColCount = 0L; - for (Map.Entry> entry : CustomViewInfo.decodeProperties(columnStr)) - { - String fieldName = entry.getKey().toString().toLowerCase(); - if (fieldName.startsWith(ANCESTOR_PREFIX)) - { - ancestorColCount++; - lineageColCount++; - } - else if (fieldName.startsWith(INPUT_PREFIX) || fieldName.startsWith(OUTPUT_PREFIX)) - { - lineageColCount++; - } - } - if (ancestorColCount > 0) - ancestorColCounts.add(ancestorColCount); - if (lineageColCount > 0) - lineageColCounts.add(lineageColCount); - } - - Collections.sort(lineageColCounts); - int lineageViewCount = lineageColCounts.size(); - metrics.put("customViewsCountWithLineageColumnsCount", lineageViewCount); - if (lineageViewCount != 0) - { - long totalLineageCols = lineageColCounts.stream().mapToLong(Long::longValue).sum(); - metrics.put("totalLineageColumnsInAllViews", totalLineageCols); - metrics.put("lineageColumnsCountMin", percentile(0, lineageColCounts)); - metrics.put("lineageColumnsCount25", percentile(25, lineageColCounts)); - metrics.put("lineageColumnsCount50", percentile(50, lineageColCounts)); - metrics.put("lineageColumnsCount75", percentile(75, lineageColCounts)); - metrics.put("lineageColumnsCountMax", percentile(100, lineageColCounts)); - metrics.put("lineageColumnsCountAvg", Math.round((float) totalLineageCols / lineageViewCount)); - } - - Collections.sort(ancestorColCounts); - int ancestorViewCount = ancestorColCounts.size(); - metrics.put("customViewsWithAncestorColumnsCounts", ancestorViewCount); - if (ancestorViewCount != 0) - { - long totalAncestorCols = ancestorColCounts.stream().mapToLong(Long::longValue).sum(); - metrics.put("totalAncestorColumnsInAllViews", totalAncestorCols); - metrics.put("ancestorColumnsCountMin", percentile(0, ancestorColCounts)); - metrics.put("ancestorColumnsCount25", percentile(25, ancestorColCounts)); - metrics.put("ancestorColumnsCount50", percentile(50, ancestorColCounts)); - metrics.put("ancestorColumnsCount75", percentile(75, ancestorColCounts)); - metrics.put("ancestorColumnsCountMax", percentile(100, ancestorColCounts)); - metrics.put("ancestorColumnsCountAvg", Math.round((float) totalAncestorCols / ancestorViewCount)); - } - - return metrics; - } - - -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.query.persist; + +import org.apache.commons.collections4.Bag; +import org.apache.commons.collections4.bag.HashBag; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerType; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.FilterInfo; +import org.labkey.api.data.JsonWriter; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.ontology.Concept; +import org.labkey.api.ontology.OntologyService; +import org.labkey.api.query.AliasedColumn; +import org.labkey.api.query.CustomView; +import org.labkey.api.query.CustomViewChangeListener; +import org.labkey.api.query.CustomViewInfo; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryChangeListener.QueryPropertyChange; +import org.labkey.api.query.QueryDefinition; +import org.labkey.api.query.QueryParseException; +import org.labkey.api.query.QueryParseWarning; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.view.NotFoundException; +import org.labkey.query.ExternalSchema; +import org.labkey.query.ExternalSchemaDocumentProvider; +import org.springframework.jdbc.BadSqlGrammarException; + +import java.net.URISyntaxException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Function; +import java.util.stream.Collectors; + + +public class QueryManager +{ + private static final Logger _log = LogManager.getLogger(QueryManager.class); + private static final QueryManager instance = new QueryManager(); + private static final String SCHEMA_NAME = "query"; + private static final List QUERY_LISTENERS = new CopyOnWriteArrayList<>(); + private static final List VIEW_LISTENERS = new CopyOnWriteArrayList<>(); + + public static final int FLAG_INHERITABLE = 0x01; + public static final int FLAG_HIDDEN = 0x02; + public static final int FLAG_SNAPSHOT = 0x04; + + public static QueryManager get() + { + return instance; + } + + /** + * @param customQuery whether to look for custom queries or modified metadata on built-in tables + */ + public QueryDef getQueryDef(Container container, String schema, String name, boolean customQuery) + { + return QueryDefCache.getQueryDef(container, schema, name, customQuery); + } + + /** + * @param customQuery whether to look for custom queries or modified metadata on built-in tables + */ + public List getQueryDefs(Container container, @Nullable String schema, boolean inheritableOnly, boolean includeSnapshots, boolean customQuery) + { + return QueryDefCache.getQueryDefs(container, schema, inheritableOnly, includeSnapshots, customQuery); + } + + public Collection getQuerySnapshots(@Nullable Container container, @Nullable String schemaName) + { + return QuerySnapshotCache.getQuerySnapshotDefs(container, schemaName); + } + + public QuerySnapshotDef getQuerySnapshotDef(@NotNull Container container, @NotNull String schemaName, @NotNull String snapshotName) + { + return QuerySnapshotCache.getQuerySnapshotDef(container, schemaName, snapshotName); + } + + public QueryDef insert(User user, QueryDef queryDef) + { + QueryDef def = Table.insert(user, getTableInfoQueryDef(), queryDef); + QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); + return def; + } + + public QueryDef update(User user, QueryDef queryDef) + { + QueryDef def = Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); + QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); + return def; + } + + public void renameQuery(User user, Container container, String schema, String oldName, String newName) + { + QueryDef queryDef = getQueryDef(container, schema, oldName, false); + if (queryDef != null) + { + queryDef.setName(newName); + QueryDef def = Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); + QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); + } + } + + public void renameSchema(User user, Container container, String oldSchema, String newSchema) + { + List queryDefs = getQueryDefs(container, oldSchema, false, false, false); + for (QueryDef queryDef : queryDefs) + { + queryDef.setSchema(newSchema); + Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); + } + QueryDefCache.uncache(ContainerManager.getForId(container.getId())); + } + + public void delete(QueryDef queryDef) + { + Table.delete(getTableInfoQueryDef(), queryDef.getQueryDefId()); + QueryDefCache.uncache(ContainerManager.getForId(queryDef.getContainerId())); + } + + public void delete(QuerySnapshotDef querySnapshotDef) + { + Table.delete(getTableInfoQuerySnapshotDef(), querySnapshotDef.getRowId()); + QuerySnapshotCache.uncache(querySnapshotDef); + if (querySnapshotDef.getQueryDefId() != null) + { + Table.delete(getTableInfoQueryDef(), querySnapshotDef.getQueryDefId()); + QueryDefCache.uncache(querySnapshotDef.lookupContainer()); + } + } + + public QuerySnapshotDef insert(User user, QueryDef queryDef, QuerySnapshotDef snapshotDef) + { + if (queryDef != null && snapshotDef.getQueryTableName() == null) + { + QueryDef def = insert(user, queryDef); + snapshotDef.setQueryDefId(def.getQueryDefId()); + } + snapshotDef = Table.insert(user, getTableInfoQuerySnapshotDef(), snapshotDef); + QuerySnapshotCache.uncache(snapshotDef); + return snapshotDef; + } + + public QuerySnapshotDef update(User user, QueryDef queryDef, QuerySnapshotDef snapshotDef) + { + if (queryDef != null && snapshotDef.getQueryTableName() == null) + update(user, queryDef); + snapshotDef = Table.update(user, getTableInfoQuerySnapshotDef(), snapshotDef, snapshotDef.getRowId()); + QuerySnapshotCache.uncache(snapshotDef); + return snapshotDef; + } + + // Does not use the cache... but only used at save time + public QuerySnapshotDef getQuerySnapshotDef(int id) + { + return new TableSelector(getTableInfoQuerySnapshotDef()).getObject(id, QuerySnapshotDef.class); + } + + public CstmView getCustomView(Container container, int id) + { + CstmView view = CustomViewCache.getCstmView(container, id); + _log.debug(view); + return view; + } + + public CstmView getCustomView(Container container, String entityId) + { + CstmView view = CustomViewCache.getCstmViewByEntityId(container, entityId); + _log.debug(view); + return view; + } + + /** + * Get all shared custom views that are applicable. + * If inheritable is true, custom views from parent and Shared container are included. + * + */ + public List getAllSharedCstmViews(Container container, String schemaName, String queryName, boolean inheritable) + { + return getAllCstmViews(container, schemaName, queryName, null, inheritable, true); + } + + /** + * Get all custom views that are applicable for this user including shared custom views. + * If inheritable is true, custom views from parent and Shared container are included. + * + * @param container The current container. + * @param schemaName The schema name or null for all schemas. + * @param queryName The query name or null for all queries in the schema. + * @param owner The owner or null for all views (shared or owned by someone.) + * @param inheritable If true, look up container hierarchy and in Shared project for custom views. + * @param sharedOnly If true, ignore the user parameter and only include shared custom views. + * @return List of custom views entities in priority order. + */ + public List getAllCstmViews(Container container, String schemaName, String queryName, @Nullable User owner, boolean inheritable, boolean sharedOnly) + { + List views = new ArrayList<>(); + + getCstmViewsInContainer(views, container, schemaName, queryName, owner, false, sharedOnly); + if (!container.isContainerFor(ContainerType.DataType.customQueryViews)) + { + getCstmViewsInContainer(views, container.getContainerFor(ContainerType.DataType.customQueryViews), schemaName, queryName, owner, false, sharedOnly); + } + + if (!inheritable) + return views; + + Container containerCur = container.getParent(); + while (containerCur != null && !containerCur.isRoot()) + { + getCstmViewsInContainer(views, containerCur, schemaName, queryName, owner, true, sharedOnly); + containerCur = containerCur.getParent(); + } + + // look in the shared project + getCstmViewsInContainer(views, ContainerManager.getSharedContainer(), schemaName, queryName, owner, true, sharedOnly); + + return views; + } + + private void getCstmViewsInContainer(List views, Container container, String schemaName, String queryName, @Nullable User user, boolean inheritable, boolean sharedOnly) + { + if (sharedOnly) + { + // Get only shared custom views + views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, true)); + } + else + { + if (user != null) + { + // Custom views owned by the user first, then add shared custom views + views.addAll(getCstmViews(container, schemaName, queryName, null, user, inheritable, false)); + views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, true)); + } + else + { + // Get all custom views regardless of owner + views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, false)); + } + } + } + + public List getCstmViews(Container container, @Nullable String schemaName, @Nullable String queryName, @Nullable String viewName, @Nullable User user, boolean inheritableOnly, boolean sharedOnly) + { + return CustomViewCache.getCstmViews(container, schemaName, queryName, viewName, user, inheritableOnly, sharedOnly); + } + + public CstmView update(User user, CstmView view) + { + CstmView cstmView = Table.update(user, getTableInfoCustomView(), view, view.getCustomViewId()); + CustomViewCache.uncache(ContainerManager.getForId(cstmView.getContainerId())); + + return cstmView; + } + + public CstmView insert(User user, CstmView view) + { + CstmView cstmView = Table.insert(user, getTableInfoCustomView(), view); + CustomViewCache.uncache(ContainerManager.getForId(cstmView.getContainerId())); + + return cstmView; + } + + public void delete(CstmView view) + { + Table.delete(getTableInfoCustomView(), view.getCustomViewId()); + CustomViewCache.uncache(ContainerManager.getForId(view.getContainerId())); + } + + @Nullable + public ExternalSchemaDef getExternalSchemaDef(Container c, int rowId) + { + return ExternalSchemaDefCache.getSchemaDef(c, rowId, ExternalSchemaDef.class); + } + + @NotNull + public List getExternalSchemaDefs(@Nullable Container container) + { + return ExternalSchemaDefCache.getSchemaDefs(container, ExternalSchemaDef.class); + } + + @Nullable + public ExternalSchemaDef getExternalSchemaDef(Container container, @Nullable String userSchemaName) + { + return ExternalSchemaDefCache.getSchemaDef(container, userSchemaName, ExternalSchemaDef.class); + } + + @Nullable + public LinkedSchemaDef getLinkedSchemaDef(Container c, int rowId) + { + return ExternalSchemaDefCache.getSchemaDef(c, rowId, LinkedSchemaDef.class); + } + + @NotNull + public List getLinkedSchemaDefs(@Nullable Container c) + { + return ExternalSchemaDefCache.getSchemaDefs(c, LinkedSchemaDef.class); + } + + @Nullable + public LinkedSchemaDef getLinkedSchemaDef(Container c, @Nullable String userSchemaName) + { + return ExternalSchemaDefCache.getSchemaDef(c, userSchemaName, LinkedSchemaDef.class); + } + + public void delete(@NotNull AbstractExternalSchemaDef def) + { + Container c = def.lookupContainer(); + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(getTableInfoExternalSchema().getColumn("ExternalSchemaId"), def.getExternalSchemaId()); + Table.delete(getTableInfoExternalSchema(), filter); + updateExternalSchemas(def.lookupContainer()); + } + + public LinkedSchemaDef insertLinkedSchema(User user, LinkedSchemaDef def) + { + LinkedSchemaDef newDef = Table.insert(user, getTableInfoExternalSchema(), def); + updateExternalSchemas(def.lookupContainer()); + return newDef; + } + + public void deleteLinkedSchema(Container container, String userSchemaName) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromString("UserSchemaName"), userSchemaName); + filter.addCondition(FieldKey.fromString("SchemaType"), AbstractExternalSchemaDef.SchemaType.linked); + Table.delete(getTableInfoExternalSchema(), filter); + updateExternalSchemas(container); + } + + // Uncaches and re-indexes all external schemas in a container. Called any time an external schema or linked schema + // changes in any way (insert/update/delete). + public void updateExternalSchemas(Container c) + { + QueryService.get().updateLastModified(); + if (null != c) + { + ExternalSchemaDefCache.uncache(c); + ExternalSchemaDocumentProvider.getInstance().enumerateDocuments(SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified), null); + } + } + + public void reloadAllExternalSchemas(Container c) + { + getExternalSchemaDefs(c).forEach(this::reloadExternalSchema); + } + + public void reloadExternalSchema(ExternalSchemaDef def) + { + ExternalSchema.uncache(def); + } + + public boolean canInherit(int flag) + { + return (flag & FLAG_INHERITABLE) != 0; + } + + public int setCanInherit(int flag, boolean f) + { + if (f) + { + return flag | FLAG_INHERITABLE; + } + else + { + return flag & ~FLAG_INHERITABLE; + } + } + + public boolean isHidden(int flag) + { + return (flag & FLAG_HIDDEN) != 0; + } + + public int setIsHidden(int flag, boolean f) + { + if (f) + { + return flag | FLAG_HIDDEN; + } + else + { + return flag & ~FLAG_HIDDEN; + } + } + + public boolean isSnapshot(int flag) + { + return (flag & FLAG_SNAPSHOT) != 0; + } + + public int setIsSnapshot(int flag, boolean f) + { + if (f) + { + return flag | FLAG_SNAPSHOT; + } + else + { + return flag & ~FLAG_SNAPSHOT; + } + } + + public String getDbSchemaName() + { + return SCHEMA_NAME; + } + + public DbSchema getDbSchema() + { + return DbSchema.get(SCHEMA_NAME, DbSchemaType.Module); + } + + public TableInfo getTableInfoQueryDef() + { + return getDbSchema().getTable("QueryDef"); + } + + public TableInfo getTableInfoQuerySnapshotDef() + { + return getDbSchema().getTable("QuerySnapshotDef"); + } + + public TableInfo getTableInfoCustomView() + { + return getDbSchema().getTable("CustomView"); + } + + public TableInfo getTableInfoExternalSchema() + { + return getDbSchema().getTable("ExternalSchema"); + } + + public TableInfo getTableInfoOlapDef() + { + return getDbSchema().getTable("OlapDef"); + } + + public void containerDeleted(Container c) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + Table.delete(getTableInfoQuerySnapshotDef(), filter); + QuerySnapshotCache.uncache(c); + Table.delete(getTableInfoCustomView(), filter); + CustomViewCache.uncache(c); + Table.delete(getTableInfoQueryDef(), filter); + QueryDefCache.uncache(c); + Table.delete(getTableInfoExternalSchema(), filter); + ExternalSchemaDefCache.uncache(c); + Table.delete(getTableInfoOlapDef(), filter); + } + + public void addQueryListener(QueryChangeListener listener) + { + QUERY_LISTENERS.add(listener); + } + + public void removeQueryListener(QueryChangeListener listener) + { + QUERY_LISTENERS.remove(listener); + } + + public void fireQueryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) + { + QueryService.get().updateLastModified(); + for (QueryChangeListener l : QUERY_LISTENERS) + l.queryCreated(user, container, scope, schema, queries); + } + + public void fireQueryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @Nullable String queryName, @NotNull QueryChangeListener.QueryProperty property, @NotNull Collection> changes) + { + QueryService.get().updateLastModified(); + assert checkChanges(property, changes); + for (QueryChangeListener l : QUERY_LISTENERS) + l.queryChanged(user, container, scope, schema, queryName, property, changes); + } + + // Checks all changes have the correct property and type. + private boolean checkChanges(QueryChangeListener.QueryProperty property, Collection> changes) + { + if (property == null) + { + _log.error("Null property not allowed."); + return false; + } + + boolean valid = true; + for (QueryPropertyChange change : changes) + { + if (change.getProperty() != property) + { + _log.error(String.format("Property '%s' doesn't match change property '%s'", property, change.getProperty())); + valid = false; + } + if (change.getOldValue() != null && !property.getPropertyClass().isInstance(change.getOldValue())) + { + _log.error(String.format("Old value '%s' isn't an instance of property '%s' class '%s'", change.getOldValue(), property, property.getPropertyClass())); + valid = false; + } + if (change.getNewValue() != null && !property.getPropertyClass().isInstance(change.getNewValue())) + { + _log.error(String.format("New value '%s' isn't an instance of property '%s' class '%s'", change.getNewValue(), property, property.getPropertyClass())); + valid = false; + } + } + return valid; + } + + public void fireQueryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries) + { + QueryService.get().updateLastModified(); + for (QueryChangeListener l : QUERY_LISTENERS) + l.queryDeleted(user, container, scope, schema, queries); + } + + public Collection getQueryDependents(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries) + { + ArrayList dependents = new ArrayList<>(); + for (QueryChangeListener l : QUERY_LISTENERS) + dependents.addAll(l.queryDependents(user, container, scope, schema, queries)); + return dependents; + } + + public void addCustomViewListener(CustomViewChangeListener listener) + { + VIEW_LISTENERS.add(listener); + } + + public void removeCustomViewListener(CustomViewChangeListener listener) + { + VIEW_LISTENERS.remove(listener); + } + + public void fireViewCreated(CustomView view) + { + QueryService.get().updateLastModified(); + for (CustomViewChangeListener l : VIEW_LISTENERS) + l.viewCreated(view); + } + + public void fireViewChanged(CustomView view) + { + QueryService.get().updateLastModified(); + for (CustomViewChangeListener l : VIEW_LISTENERS) + l.viewChanged(view); + } + + public void fireViewDeleted(CustomView view) + { + QueryService.get().updateLastModified(); + for (CustomViewChangeListener l : VIEW_LISTENERS) + l.viewDeleted(view); + } + + public Collection getViewDepedents(CustomView view) + { + ArrayList dependents = new ArrayList<>(); + for (CustomViewChangeListener l : VIEW_LISTENERS) + dependents.addAll(l.viewDependents(view)); + return dependents; + } + + static public final ContainerManager.ContainerListener CONTAINER_LISTENER = new ContainerManager.ContainerListener() + { + @Override + public void containerDeleted(Container c, User user) + { + QueryManager.get().containerDeleted(c); + } + }; + + public boolean validateQuery(SchemaKey schemaPath, String queryName, User user, Container container, @NotNull List errors, + @NotNull List warnings) + { + UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); + if (null == schema) + throw new IllegalArgumentException("Could not find the schema '" + schemaPath.toDisplayString() + "'!"); + + TableInfo table = schema.getTable(queryName); + if (null == table) + throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schemaPath.toDisplayString() + "'!"); + + return validateQuery(table, true, errors, warnings); + } + + public boolean validateQuery(TableInfo table, boolean testAllColumns, @NotNull List errors, + @NotNull List warnings) + { + errors.addAll(table.getWarnings()); + + Collection params = table.getNamedParameters(); + Map parameters = new HashMap<>(); + for (QueryService.ParameterDecl p : params) + { + if (!p.isRequired()) + continue; + parameters.put(p.getName(), null); + } + + TableSelector selector; + + // Note this check had been inverted for years, but was fixed in 14.1. Previously, testAllColumns == true meant + // the default column list was computed but discarded, and testAllColumns == false was completely broken + if (testAllColumns) + { + selector = new TableSelector(table); + } + else + { + List defVisCols = table.getDefaultVisibleColumns(); + Map colMap = QueryService.get().getColumns(table, defVisCols); + List cols = new ArrayList<>(colMap.values()); + + selector = new TableSelector(table, cols, null, null); + } + + // set forDisplay to mimic the behavior one would get in the UI + // try to execute with a rowcount of 0 (will throw SQLException to client if it fails) + selector.setForDisplay(true).setNamedParameters(parameters).setMaxRows(Table.NO_ROWS); + + //noinspection EmptyTryBlock,UnusedDeclaration + try (ResultSet rs = selector.getResultSet()) + { + } + catch (SQLException e) + { + errors.add(new QueryParseException(e.getMessage(), e, 0, 0)); + } + catch (BadSqlGrammarException e) + { + errors.add(new QueryParseException(e.getSQLException().getMessage(), e, 0, 0)); + } + + UserSchema schema = table.getUserSchema(); + if (schema != null) + { + QueryDefinition queryDef = schema.getQueryDef(table.getName()); + if (queryDef != null) + { + queryDef.validateQuery(schema, errors, warnings); + } + } + + OntologyService os = OntologyService.get(); + if (null != os) + { + for (var col : table.getColumns()) + { + String code = col.getPrincipalConceptCode(); + if (null != code) + { + Concept concept = os.resolveCode(code); + if (null == concept) + warnings.add(new QueryParseException("Concept not found: " + code, null, 0, 0)); + } + } + } + + return errors.isEmpty(); + } + + /** + * Experimental. The goal is to provide a more thorough validation of query metadata, including warnings of potentially + * invalid conditions, like autoincrement columns set userEditable=true. + */ + public boolean validateQueryMetadata(SchemaKey schemaPath, String queryName, User user, Container container, + @NotNull List errors, @NotNull List warnings) + { + Set columns = new HashSet<>(); + UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); + if (null == schema) + throw new IllegalArgumentException("Could not find the schema '" + schemaPath.getName() + "'!"); + + TableInfo table = schema.getTable(queryName); + if (null == table) + throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schemaPath.getName() + "'!"); + + if (table.isPublic() && table.getPublicSchemaName() != null && !schemaPath.toString().equalsIgnoreCase(table.getPublicSchemaName())) + warnings.add(new QueryParseWarning("(metadata) TableInfo.getPublicSchemaName() does not match: set to '" + table.getPublicSchemaName() + "', expected '" + schemaPath + "'", null, 0,0)); + + try + { + //validate foreign keys and other metadata warnings + columns.addAll(table.getColumns()); + columns.addAll(QueryService.get().getColumns(table, table.getDefaultVisibleColumns()).values()); + } + catch(QueryParseException e) + { + errors.add(e); + } + + for (ColumnInfo col : columns) + { + validateColumn(col, user, container, table, errors, warnings); + } + + return errors.isEmpty(); + } + + /** + * Experimental. See validateQueryMetadata() + */ + private boolean validateColumn(ColumnInfo col, User user, Container container, @Nullable TableInfo parentTable, + @NotNull List errors, @NotNull List warnings) + { + if(parentTable == null) + parentTable = col.getParentTable(); + + String publicSchema = col.getParentTable().getPublicSchemaName() != null ? col.getParentTable().getPublicSchemaName() : col.getParentTable().getSchema().toString(); + String publicQuery = col.getParentTable().getPublicName() != null ? col.getParentTable().getPublicName() : col.getParentTable().getName(); + String errorBase = "(metadata) for column '" + col.getFieldKey() + "' in " + publicSchema + "." + publicQuery + ": "; + + validateFk(col, user, container, parentTable, errors, warnings, errorBase); + + Set specialCols = new CaseInsensitiveHashSet(); + specialCols.add("LSID"); + specialCols.add("entityId"); + specialCols.add("container"); + specialCols.add("created"); + specialCols.add("createdby"); + specialCols.add("modified"); + specialCols.add("modifiedby"); + + if(specialCols.contains(col.getName())) + { + if(col.isUserEditable()) + warnings.add(new QueryParseWarning(errorBase + " column is user editable, which is not expected based on its name", null, 0,0)); + if(col.isShownInInsertView()) + warnings.add(new QueryParseWarning(errorBase + " column has shownInInsertView set to true, which is not expected based on its name", null, 0, 0)); + if(col.isShownInUpdateView()) + warnings.add(new QueryParseWarning(errorBase + " column has shownInUpdateView set to true, which is not expected based on its name", null, 0, 0)); + } + + if(col.isAutoIncrement() && col.isUserEditable()) + warnings.add(new QueryParseException(errorBase + " column is autoIncrement, but has userEditable set to true", null, 0, 0)); + if(col.isAutoIncrement() && col.isShownInInsertView()) + warnings.add(new QueryParseWarning(errorBase + " column is autoIncrement, but has shownInInsertView set to true", null, 0, 0)); + if(col.isAutoIncrement() && col.isShownInUpdateView()) + warnings.add(new QueryParseWarning(errorBase + " column is autoIncrement, but has shownInUpdateView set to true", null, 0, 0)); + + try + { + if (StringUtils.isNotBlank(col.getDisplayWidth()) && Integer.parseInt(col.getDisplayWidth()) > 200 && !"textarea".equalsIgnoreCase(col.getInputType())) + { + if (col.isUserEditable() && col.getJdbcType() != null && col.getJdbcType().getJavaClass() == String.class) + warnings.add(new QueryParseWarning(errorBase + " column has a displayWidth > 200, but does not use a textarea as the inputType", null, 0, 0)); + } + } + catch (NumberFormatException e) + { + warnings.add(new QueryParseWarning(errorBase + " column has invalid value for displayWidth: '" + col.getDisplayWidth() + "'", null, 0, 0)); + } + return errors.isEmpty(); + } + + /** + * Experimental. See validateQueryMetadata() + */ + private boolean validateFk(ColumnInfo col, User user, Container container, TableInfo parentTable, + @NotNull List errors, @NotNull List warnings, + String errorBase) + + { + //NOTE: this is the same code that writes JSON to the client + JSONObject o = JsonWriter.getLookupInfo(col, false); + if (o == null) + return true; + + boolean isPublic = o.getBoolean("isPublic"); + SchemaKey schemaPath = SchemaKey.fromString(o.optString("schemaName")); + String queryName = o.optString("queryName"); + if (queryName == null) + { + // Likely a lookup that targets something not exposed via a UserSchema. Bail out without further validation + return true; + } + String displayColumn = o.optString("displayColumn"); + String keyColumn = o.optString("keyColumn"); + String containerPath = o.optString("containerPath"); + + Container lookupContainer = containerPath == null ? container : ContainerManager.getForPath(containerPath); + if (lookupContainer == null) + { + warnings.add(new QueryParseWarning(errorBase + " Unable to find container" + containerPath, null, 0, 0)); + } + + //String publicSchema = col.getParentTable().getPublicSchemaName() != null ? col.getParentTable().getPublicSchemaName() : col.getParentTable().getSchema().toString(); + //String publicQuery = col.getParentTable().getPublicName() != null ? col.getParentTable().getPublicName() : col.getParentTable().getName(); + if (col.getFk() == null) + return errors.isEmpty(); + + if (!isPublic) + { + warnings.add(new QueryParseWarning(errorBase + " has a lookup to a non-public table: " + (schemaPath == null ? "" : schemaPath.toDisplayString()) + "." + queryName, null, 0, 0)); + return errors.isEmpty(); + } + + UserSchema userSchema = QueryService.get().getUserSchema(user, lookupContainer, schemaPath); + if (userSchema == null) + { + warnings.add(new QueryParseWarning(errorBase + " unable to find the user schema: " + schemaPath.toDisplayString(), null, 0, 0)); + return errors.isEmpty(); + } + + TableInfo fkTable = userSchema.getTable(queryName); + if(fkTable == null) + { + warnings.add(new QueryParseWarning(errorBase + " has a lookup to a table that does not exist: " + schemaPath.toDisplayString() + "." + queryName, null, 0, 0)); + return errors.isEmpty(); + } + + //a FK can have a table non-visible to the client, so long as public is set to false + if (fkTable.isPublic()){ + String fkt = schemaPath.toDisplayString() + "." + queryName; + + QueryManager.get().validateQuery(schemaPath, queryName, user, lookupContainer, errors, warnings); + if (displayColumn != null) + { + FieldKey displayFieldKey = FieldKey.fromString(displayColumn); + Map cols = QueryService.get().getColumns(fkTable, Collections.singleton(displayFieldKey)); + if (!cols.containsKey(displayFieldKey)) + { + warnings.add(new QueryParseWarning(errorBase + " reports a foreign key with displayColumn of " + displayColumn + " in the table " + schemaPath.toDisplayString() + "." + queryName + ", but the column does not exist", null, 0, 0)); + } + else + { + ColumnInfo ci = cols.get(displayFieldKey); + if (!displayColumn.equals(ci.getFieldKey().toString())) + { + warnings.add(new QueryParseWarning(errorBase + ", the lookup to " + schemaPath.toDisplayString() + "." + queryName + "' did not match the expected case, which was '" + ci.getFieldKey().toString() + "'. Actual: '" + displayColumn + "'", null, 0, 0)); + } + } + } + + if (keyColumn != null) + { + FieldKey keyFieldKey = FieldKey.fromString(keyColumn); + Map cols = QueryService.get().getColumns(fkTable, Collections.singleton(keyFieldKey)); + if (!cols.containsKey(keyFieldKey)) + { + warnings.add(new QueryParseException(errorBase + " reports a foreign key with keyColumn of " + keyColumn + " in the table " + schemaPath.toDisplayString() + "." + queryName + ", but the column does not exist", null, 0, 0)); + } + else + { + ColumnInfo ci = cols.get(keyFieldKey); + if (!keyColumn.equals(ci.getFieldKey().toString())) + { + warnings.add(new QueryParseWarning(errorBase + ", the lookup to " + schemaPath.toDisplayString() + "." + queryName + "' did not match the expected case, which was '" + ci.getFieldKey().toString() + "'. Actual: '" + keyColumn + "'", null, 0, 0)); + } + } + } + else + { + warnings.add(new QueryParseWarning(errorBase + ", there is a lookup where the keyColumn is blank", null, 0, 0)); + } + } + + return errors.isEmpty(); + } + + /** + * Experimental. The goal is to provide a more thorough validation of saved views, including errors like invalid + * column names or case errors (which cause problems for case-sensitive js) + */ + public boolean validateQueryViews(SchemaKey schemaPath, String queryName, User user, Container container, + @NotNull List errors, @NotNull List warnings) throws QueryParseException + { + UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); + if (null == schema) + throw new IllegalArgumentException("Could not find the schema '" + schemaPath.getName() + "'!"); + + TableInfo table = schema.getTable(queryName); + if (null == table) + throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schema.getSchemaName() + "'!"); + + //validate views + try + { + List views = QueryService.get().getCustomViews(user, container, null, schema.getSchemaName(), queryName, true); + for (CustomView v : views) + { + validateViewColumns(user, container, v, "columns", v.getColumns(), table, errors, warnings); + + if (!StringUtils.isEmpty(v.getFilterAndSort())) + { + try + { + CustomViewInfo.FilterAndSort fs = CustomViewInfo.FilterAndSort.fromString(v.getFilterAndSort()); + List filterCols = new ArrayList<>(); + for (FilterInfo f : fs.getFilter()) + { + filterCols.add(f.getField()); + } + validateViewColumns(user, container, v, "filter", filterCols, table, errors, warnings); + + List sortCols = new ArrayList<>(); + for (Sort.SortField f : fs.getSort()) + { + sortCols.add(f.getFieldKey()); + } + validateViewColumns(user, container, v, "sort", sortCols, table, errors, warnings); + + } + catch (URISyntaxException e) + { + warnings.add(new QueryParseWarning("unable to process the filter/sort section of view: " + v.getName(), null, 0, 0)); + } + } + } + } + catch (NotFoundException e) + { + errors.add(new QueryParseException("Cannot get views: ", e, 0, 0)); + } + + + return errors.isEmpty(); + } + + private void validateViewColumns(User user, Container container, CustomView v, String identifier, List viewCols, TableInfo sourceTable, + @NotNull List errors, @NotNull List warnings) throws QueryParseException + { + //verify columns match, accounting for case + Map colMap = QueryService.get().getColumns(sourceTable, viewCols); + + for (FieldKey f : viewCols) + { + boolean found = false; + boolean matchCase = false; + FieldKey fk = null; + ColumnInfo c = colMap.get(f); + if(c != null) + { + found = true; + fk = c.getFieldKey(); + if(c instanceof AliasedColumn) + fk = ((AliasedColumn)c).getColumn().getFieldKey(); + + if(fk.toString().equals(f.toString())) + { + matchCase = true; + } + } + + if (!found){ + warnings.add(new QueryParseWarning("In the saved view '" + (v.getName() == null ? "default" : v.getName()) + "', in the " + identifier + " section, the column '" + f.toString() + "' in " + v.getSchemaName() + "." + v.getQueryName() + " could not be matched to a column", null, 0, 0)); + continue; + } + + if (!matchCase){ + warnings.add(new QueryParseWarning("In the saved view '" + (v.getName() == null ? "default" : v.getName()) + "', in the " + identifier + " section, the column '" + f + "' in " + v.getSchemaName() + "." + v.getQueryName() + "' did not match the expected case, which was '" + fk + "'", null, 0, 0)); + } + + //queryErrors.addAll(validateColumn(c, user, container)); + } + } + + public static void registerUsageMetrics(String moduleName) + { + UsageMetricsService svc = UsageMetricsService.get(); + if (null != svc) + { + svc.registerUsageMetrics(moduleName, () -> { + Bag bag = DbScope.getDbScopes().stream() + .filter(scope -> !scope.isLabKeyScope()).map(DbScope::getDatabaseProductName) + .collect(Collectors.toCollection(HashBag::new)); + + Map statsMap = bag.uniqueSet().stream() + .collect(Collectors.toMap(Function.identity(), bag::getCount)); + + return Map.of("externalDatasources", statsMap, + "customViewCounts", + Map.of( + "DataClasses", getSchemaCustomViewCounts("exp.data"), + "SampleTypes", getSchemaCustomViewCounts("samples"), + "Assays", getSchemaCustomViewCounts("assay"), + "Inventory", getSchemaCustomViewCounts("inventory") + ), + "customViewWithLineageColumn", getLineageCustomViewMetrics(), + "queryDefWithCalculatedFieldsCounts", getCalculatedFieldsCountsMetric() + ); + }); + } + } + + private static Map getCalculatedFieldsCountsMetric() + { + DbSchema dbSchema = CoreSchema.getInstance().getSchema(); + return new SqlSelector(dbSchema, + new SQLFragment("SELECT \"schema\", COUNT(*) AS count FROM (\n" + + " SELECT CASE WHEN \"schema\" LIKE 'assay.%' THEN 'assay' ELSE \"schema\" END AS \"schema\" FROM query.querydef WHERE metadata LIKE '%(), (x, m) -> { + x.put(m.get("schema").toString(), m.get("count")); + return x; + }); + } + + private static Map getSchemaCustomViewCounts(String schema) + { + DbSchema dbSchema = DbSchema.get("query"); + TableInfo customView = dbSchema.getTable("customview"); + var schemaField = customView.getColumn("schema").getSelectIdentifier(); + SQLFragment schemaClause; + if (schema.equalsIgnoreCase("assay")) + schemaClause = new SQLFragment("C.").appendIdentifier(schemaField).append(" LIKE 'assay.%'"); + else + schemaClause = new SQLFragment("C.").appendIdentifier(schemaField).append(" = ").appendValue(schema); + return Map.of( + "defaultOverrides", new SqlSelector(dbSchema, + new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags < 2 AND C.name IS NULL")).getObject(Long.class), // possibly inheritable, no hidden, not snapshot + "inheritable", new SqlSelector(dbSchema, + new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags = 1")).getObject(Long.class), // inheritable, not hidden, not snapshot + "namedViews", new SqlSelector(dbSchema, + new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags < 2 AND C.name IS NOT NULL")).getObject(Long.class), // possibly inheritable, no hidden, not snapshot + "shared", new SqlSelector(dbSchema, + new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.customviewowner IS NULL")).getObject(Long.class), + "identifyingFieldsViews", new SqlSelector(dbSchema, + new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.name = '~~identifyingfields~~'")).getObject(Long.class) + ); + } + + + private static Long percentile(double percentile, List sortedCounts) { + if (percentile <= 0.01) + return sortedCounts.get(0); + if (percentile >= 99.99) + return sortedCounts.get(sortedCounts.size() - 1); + return sortedCounts.get((int) Math.round(percentile / 100.0 * (sortedCounts.size() - 1))); + } + + /** + * customViewsCountWithLineageCol: total number of non-hidden saved custom views that has at least one input/output/ancestor column + * customViewsCountWithAncestorCol: total number of non-hidden saved custom views that has at least one ancestor column + * totalLineageColumnsInAllViews: total number of input/output/ancestor columns defined for all saved non-hidden custom views + * totalAncestorColumnsInAllViews: total number of ancestor columns defined for all saved non-hidden custom views + * lineageColumnsCountMin: the minimum count of input/output/ancestor columns in any view with such column + * lineageColumnsCount25: the 25 percentile count of input/output/ancestor columns in all views with such column + * lineageColumnsCount50: the 50 percentile / median count of input/output/ancestor columns in all views with such column + * lineageColumnsCount75: the 75 percentile count of input/output/ancestor columns in all views with such column + * lineageColumnsCountMax: the maximum count of input/output/ancestor columns in any view with such column + * lineageColumnsCountAvg: the average count of input/output/ancestor columns in any view with such column + * ancestorColumnsCountMin: the minimum count of ancestor columns in any view with ancestor columns + * ancestorColumnsCount25: the 25 percentile count of ancestor columns in all views with ancestor columns + * ancestorColumnsCount50: the 50 percentile / median count of ancestor columns in all views with ancestor columns + * ancestorColumnsCount75: the 75 percentile count of ancestor columns in all views with ancestor columns + * ancestorColumnsCountMax: the maximum count of ancestor columns in any view with ancestor columns + * ancestorColumnsCountAvg: the average count of ancestor columns in any view with ancestor columns + */ + private static Map getLineageCustomViewMetrics() + { + List ancestorColCounts = new ArrayList<>(); + List lineageColCounts = new ArrayList<>(); + final String ANCESTOR_PREFIX = "ancestors/"; + final String INPUT_PREFIX = "inputs/"; + final String OUTPUT_PREFIX = "outputs/"; + + Map metrics = new HashMap<>(); + + DbSchema schema = DbSchema.get("query", DbSchemaType.Module); + SqlDialect sqlDialect = schema.getSqlDialect(); + SQLFragment sql = new SQLFragment() + .append("SELECT columns FROM query.customview WHERE flags < 2 AND (columns LIKE ? OR columns LIKE ? OR columns LIKE ?)") + .add("%" + sqlDialect.encodeLikeOpSearchString("Ancestors%2F") + "%") + .add("%" + sqlDialect.encodeLikeOpSearchString("Inputs%2F") + "%") + .add("%" + sqlDialect.encodeLikeOpSearchString("Outputs%2F") + "%"); + List viewsColumnStrs = new SqlSelector(schema, sql).getArrayList(String.class); + + for (String columnStr : viewsColumnStrs) + { + long lineageColCount = 0L; + long ancestorColCount = 0L; + for (Map.Entry> entry : CustomViewInfo.decodeProperties(columnStr)) + { + String fieldName = entry.getKey().toString().toLowerCase(); + if (fieldName.startsWith(ANCESTOR_PREFIX)) + { + ancestorColCount++; + lineageColCount++; + } + else if (fieldName.startsWith(INPUT_PREFIX) || fieldName.startsWith(OUTPUT_PREFIX)) + { + lineageColCount++; + } + } + if (ancestorColCount > 0) + ancestorColCounts.add(ancestorColCount); + if (lineageColCount > 0) + lineageColCounts.add(lineageColCount); + } + + Collections.sort(lineageColCounts); + int lineageViewCount = lineageColCounts.size(); + metrics.put("customViewsCountWithLineageColumnsCount", lineageViewCount); + if (lineageViewCount != 0) + { + long totalLineageCols = lineageColCounts.stream().mapToLong(Long::longValue).sum(); + metrics.put("totalLineageColumnsInAllViews", totalLineageCols); + metrics.put("lineageColumnsCountMin", percentile(0, lineageColCounts)); + metrics.put("lineageColumnsCount25", percentile(25, lineageColCounts)); + metrics.put("lineageColumnsCount50", percentile(50, lineageColCounts)); + metrics.put("lineageColumnsCount75", percentile(75, lineageColCounts)); + metrics.put("lineageColumnsCountMax", percentile(100, lineageColCounts)); + metrics.put("lineageColumnsCountAvg", Math.round((float) totalLineageCols / lineageViewCount)); + } + + Collections.sort(ancestorColCounts); + int ancestorViewCount = ancestorColCounts.size(); + metrics.put("customViewsWithAncestorColumnsCounts", ancestorViewCount); + if (ancestorViewCount != 0) + { + long totalAncestorCols = ancestorColCounts.stream().mapToLong(Long::longValue).sum(); + metrics.put("totalAncestorColumnsInAllViews", totalAncestorCols); + metrics.put("ancestorColumnsCountMin", percentile(0, ancestorColCounts)); + metrics.put("ancestorColumnsCount25", percentile(25, ancestorColCounts)); + metrics.put("ancestorColumnsCount50", percentile(50, ancestorColCounts)); + metrics.put("ancestorColumnsCount75", percentile(75, ancestorColCounts)); + metrics.put("ancestorColumnsCountMax", percentile(100, ancestorColCounts)); + metrics.put("ancestorColumnsCountAvg", Math.round((float) totalAncestorCols / ancestorViewCount)); + } + + return metrics; + } + + +} diff --git a/query/src/org/labkey/query/reports/ReportQueryChangeListener.java b/query/src/org/labkey/query/reports/ReportQueryChangeListener.java index 57ff5483034..f4c05a89536 100644 --- a/query/src/org/labkey/query/reports/ReportQueryChangeListener.java +++ b/query/src/org/labkey/query/reports/ReportQueryChangeListener.java @@ -75,7 +75,7 @@ public void queryCreated(User user, Container container, ContainerFilter scope, } @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { if (property.equals(QueryProperty.Name)) { diff --git a/study/src/org/labkey/study/query/QueryDatasetQueryChangeListener.java b/study/src/org/labkey/study/query/QueryDatasetQueryChangeListener.java index 043bf89c69d..fa4823fd99b 100644 --- a/study/src/org/labkey/study/query/QueryDatasetQueryChangeListener.java +++ b/study/src/org/labkey/study/query/QueryDatasetQueryChangeListener.java @@ -25,7 +25,7 @@ public void queryCreated(User user, Container container, ContainerFilter scope, } @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull QueryProperty property, @NotNull Collection> changes) + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) { if (property.equals(QueryProperty.Name)) { From fc22b72d41ab207db4bc2f51a6a454cdedf71fb0 Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 22 Jan 2026 15:45:49 -0800 Subject: [PATCH 2/4] crlf --- .../core/dialect/PostgreSql92Dialect.java | 2446 +++++++------- .../api/property/DomainPropertyImpl.java | 2796 ++++++++--------- .../QuerySnapshotQueryChangeListener.java | 282 +- .../labkey/query/persist/QueryManager.java | 2354 +++++++------- 4 files changed, 3939 insertions(+), 3939 deletions(-) diff --git a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java index 01f833d58f5..a25da4256ad 100644 --- a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java +++ b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java @@ -1,1223 +1,1223 @@ -/* - * Copyright (c) 2012-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.core.dialect; - -import jakarta.servlet.ServletException; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.Constraint; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DatabaseIdentifier; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.InClauseGenerator; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.ParameterMarkerInClauseGenerator; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.Selector; -import org.labkey.api.data.Selector.ForEachBlock; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableChange; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TempTableInClauseGenerator; -import org.labkey.api.data.TempTableTracker; -import org.labkey.api.data.dialect.BasePostgreSqlDialect; -import org.labkey.api.data.dialect.DialectStringHandler; -import org.labkey.api.data.dialect.JdbcHelper; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.data.dialect.StandardJdbcHelper; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.query.AliasManager; -import org.labkey.api.util.ConfigurationException; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.view.template.Warnings; -import org.labkey.core.admin.sql.ScriptReorderer; -import org.springframework.jdbc.BadSqlGrammarException; - -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.Driver; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Level; -import java.util.logging.LogManager; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/* - * This is the base class defining PostgreSQL-specific (i.e., not Redshift) behavior. PostgreSQL 9.2 is no longer - * supported; however, we keep this class to track changes we implemented specifically for this version. - */ -abstract class PostgreSql92Dialect extends BasePostgreSqlDialect -{ - public static final String PRODUCT_NAME = "PostgreSQL"; - - // This has been the standard PostgreSQL identifier max byte length for many years. However, this could change in - // the future plus servers can be compiled with a different limit, so we query this setting on first connection to - // each database. - private int _maxIdentifierByteLength = 63; - private InClauseGenerator _inClauseGenerator; - - private final TempTableInClauseGenerator _tempTableInClauseGenerator = new TempTableInClauseGenerator(); - private final AtomicBoolean _arraySortFunctionExists = new AtomicBoolean(false); - - @Override - public void handleCreateDatabaseException(SQLException e) throws ServletException - { - if ("55006".equals(e.getSQLState())) - { - LOG.error("You must close down pgAdmin III and all other applications accessing PostgreSQL."); - throw (new ServletException("Close down or disconnect pgAdmin III and all other applications accessing PostgreSQL", e)); - } - else - { - super.handleCreateDatabaseException(e); - } - } - - @Override - public void prepareDriver(Class driverClass) - { - // PostgreSQL driver 42.0.0 added logging via the Java Logging API (java.util.logging). This caused the driver to - // start logging SQLExceptions (such as the initial connection failure on bootstrap) to the console... harmless - // but annoying. This code suppresses the driver logging. - Logger pgjdbcLogger = LogManager.getLogManager().getLogger("org.postgresql"); - - if (null != pgjdbcLogger) - pgjdbcLogger.setLevel(Level.OFF); - } - - // Make sure that the PL/pgSQL language is enabled in the associated database. If not, throw. Since 9.0, PostgreSQL has - // shipped with PL/pgSQL enabled by default, so the check is no longer critical, but continue to verify just to be safe. - @Override - public void prepareNewLabKeyDatabase(DbScope scope) - { - if (new SqlSelector(scope, "SELECT * FROM pg_language WHERE lanname = 'plpgsql'").exists()) - return; - - String dbName = scope.getDatabaseName(); - String message = "PL/pgSQL is not enabled in the \"" + dbName + "\" database because it is not enabled in your Template1 master database."; - String advice = "Use PostgreSQL's 'createlang' command line utility to enable PL/pgSQL in the \"" + dbName + "\" database then restart Tomcat."; - - throw new ConfigurationException(message, advice); - } - - @Override - public String prepare(DbScope scope) - { - initializeInClauseGenerator(scope); - determineIfArraySortFunctionExists(scope); - return super.prepare(scope); - } - - @NotNull - @Override - protected Set getReservedWords() - { - Set words = super.getReservedWords(); - words.add("collation"); - - return words; - } - - /* - These override method implementations were migrated from PostgreSql91Dialect when that class was promoted to api: - getProductName() - createStringHandler() - getJdbcHelper() - getScriptWarnings() - initializeInClauseGenerator() - */ - - @Override - public String getProductName() - { - return PRODUCT_NAME; - } - - // Query PostgreSQL-specific settings - @Override - protected void determineSettings(DbScope scope) - { - if (getServerType().supportsSpecialMetadataQueries()) - { - super.determineSettings(scope); - - String value = new SqlSelector(scope, "SELECT setting FROM pg_settings WHERE name = 'max_identifier_length'").getObject(String.class); - try - { - _maxIdentifierByteLength = Integer.valueOf(value); - } - catch (NumberFormatException e) - { - LOG.error("Couldn't parse max_identifier_length; continuing with default value of {}", _maxIdentifierByteLength, e); - } - } - } - - @Override - protected DialectStringHandler createStringHandler() - { - // TODO: Isn't this the wrong setting? Should we be looking at the "backslash_quote" setting instead? - if (getStandardConformingStrings()) - return super.createStringHandler(); - else - return new PostgreSqlNonConformingStringHandler(); - } - - /* - PostgreSQL example connection URLs we need to parse: - - jdbc:postgresql:database - jdbc:postgresql://host/database - jdbc:postgresql://host:port/database - jdbc:postgresql:database?user=fred&password=secret&ssl=true - jdbc:postgresql://host/database?user=fred&password=secret&ssl=true - jdbc:postgresql://host:port/database?user=fred&password=secret&ssl=true - */ - @Override - public JdbcHelper getJdbcHelper() - { - return new StandardJdbcHelper(PostgreSqlDialectFactory.JDBC_PREFIX); - } - - @Override - public String getDefaultDatabaseName() - { - return "template1"; - } - - @Override - public boolean canExecuteUpgradeScripts() - { - return true; - } - - @Override - public Collection getScriptWarnings(String name, String sql) - { - // Strip out all block- and single-line comments - Pattern commentPattern = Pattern.compile(ScriptReorderer.COMMENT_REGEX, Pattern.DOTALL + Pattern.MULTILINE); - Matcher matcher = commentPattern.matcher(sql); - String noComments = matcher.replaceAll(""); - - List warnings = new LinkedList<>(); - - // Split statements by semicolon and CRLF - for (String statement : noComments.split(";[\\n\\r]+")) - { - if (Strings.CI.startsWith(statement.trim(), "SET ")) - warnings.add(statement); - } - - return warnings; - } - - @Override - public String getSQLScriptPath() - { - return "postgresql"; - } - - @Override - public String getUniqueIdentType() - { - return "SERIAL"; - } - - @Override - public boolean supportsGroupConcat() - { - return getServerType().supportsGroupConcat(); - } - - @Override - public boolean supportsSelectConcat() - { - return true; - } - - @Override - public SQLFragment getSelectConcat(SQLFragment selectSql, String delimiter) - { - SQLFragment result = new SQLFragment("array_to_string(array("); - result.append(selectSql); - result.append("), "); - result.append(getStringHandler().quoteStringLiteral(delimiter)); - result.append(")"); - return result; - } - - // Does this datasource include our sort array function? The LabKey datasource should always have it, but external datasources might not - private void determineIfArraySortFunctionExists(DbScope scope) - { - if (getServerType().supportsSpecialMetadataQueries()) - { - Selector selector = new SqlSelector(scope, "SELECT * FROM pg_catalog.pg_namespace n INNER JOIN pg_catalog.pg_proc p ON pronamespace = n.oid WHERE nspname = 'core' AND proname = 'sort'"); - _arraySortFunctionExists.set(selector.exists()); - } - - // Array sort function should always exist in LabKey scope (for now) - assert !scope.isLabKeyScope() || _arraySortFunctionExists.get(); - } - - @Override - public SQLFragment getGroupConcat(SQLFragment sql, boolean distinct, boolean sorted, @NotNull SQLFragment delimiterSQL, boolean includeNulls) - { - // Sort function might not exist in external datasource; skip that syntax if not - boolean useSortFunction = sorted && _arraySortFunctionExists.get(); - SQLFragment result = new SQLFragment(); - - if (useSortFunction) - { - result.append("array_to_string("); - result.append("core.sort("); // TODO: Switch to use ORDER BY option inside array aggregate instead of our custom function - result.append("array_agg("); - if (distinct) - { - result.append("DISTINCT "); - } - - if (includeNulls) - { - result.append("COALESCE(CAST("); - result.append(sql); - result.append(" AS VARCHAR), '')"); - } - else - { - result.append(sql); - } - - result.append(")"); // array_agg - result.append(")"); // core.sort - } - else - { - result.append("string_agg("); - if (distinct) - { - result.append("DISTINCT "); - } - - if (includeNulls) - { - result.append("COALESCE("); - result.append(sql); - result.append("::text, '')"); - } - else - { - result.append(sql); - result.append("::text"); - } - } - - result.append(", "); - result.append(delimiterSQL); - result.append(")"); // array_to_string | string_agg - - return result; - } - - @Override - public SQLFragment getAnalyzeCommandForTable(String tableName) - { - return new SQLFragment("ANALYZE ").appendIdentifier(tableName); - } - - private void initializeInClauseGenerator(DbScope scope) - { - _inClauseGenerator = getJdbcVersion(scope) >= 4 ? new ArrayParameterInClauseGenerator(scope) : new ParameterMarkerInClauseGenerator(); - } - - @Override - public InClauseGenerator getDefaultInClauseGenerator() - { - return _inClauseGenerator; - } - - @Override - public TempTableInClauseGenerator getTempTableInClauseGenerator() - { - return _tempTableInClauseGenerator; - } - - @Override - public void addAdminWarningMessages(Warnings warnings, boolean showAllWarnings) - { - super.addAdminWarningMessages(warnings, showAllWarnings); - if (showAllWarnings) - warnings.add(HtmlString.of(PostgreSqlDialectFactory.getStandardWarningMessage("has not been tested against", getMajorVersion() + ".x"))); - } - - private int getIdentifierMaxByteLength() - { - return _maxIdentifierByteLength; - } - - @Override - public boolean isIdentifierTooLong(String identifier) - { - return identifier.getBytes(StandardCharsets.UTF_8).length > getIdentifierMaxByteLength(); - } - - @Override - public String truncateAndJoin(String... parts) - { - String ret = String.join("$", parts); - - if (isIdentifierTooLong(ret)) - { - int maxBytes = getIdentifierMaxByteLength(); - StringBuilder sb = new StringBuilder(maxBytes); - int partsLength = parts.length; - int remainingBytes = maxBytes - partsLength + 1; // Make room for dollar signs - for (int i = 0; i < partsLength; i++) - { - String truncated = truncateBytes(parts[i], remainingBytes / (partsLength - i)); - if (i > 0) - sb.append("$"); - sb.append(truncated); - remainingBytes -= truncated.getBytes(StandardCharsets.UTF_8).length; - } - ret = sb.toString(); - assert ret.getBytes(StandardCharsets.UTF_8).length <= maxBytes; - } - - return ret; - } - - @Override - public String truncate(String str, int reserved) - { - return truncateBytes(str, getIdentifierMaxByteLength() - reserved); - } - - // Truncates based on UTF-8 bytes - private static String truncateBytes(String str, int maxBytes) - { - if (maxBytes < 13) - throw new IllegalStateException("maxBytes for legal name too small: " + maxBytes); - int len = str.getBytes(StandardCharsets.UTF_8).length; - if (len > maxBytes) - { - String prefix = generateIdentifierPrefix(str); - str = prefix + StringUtilsLabKey.rightUtf8Bytes(str, maxBytes - prefix.getBytes(StandardCharsets.UTF_8).length); - } - assert str.getBytes(StandardCharsets.UTF_8).length <= maxBytes; - assert !StringUtilsLabKey.hasBrokenSurrogate(str); - return str; - } - - @Override - public boolean canShowExecutionPlan(ExecutionPlanType type) - { - return true; - } - - @Override - protected Collection getQueryExecutionPlan(Connection conn, DbScope scope, SQLFragment sql, ExecutionPlanType type) - { - SQLFragment copy = new SQLFragment(sql); - copy.insert(0, type == ExecutionPlanType.Estimated ? "EXPLAIN " : "EXPLAIN ANALYZE "); - - return new SqlSelector(scope, conn, copy).getCollection(String.class); - } - - @Override - // No need to split up PostgreSQL scripts; execute all statements in a single block (unless we have a special stored proc call). - protected Pattern getSQLScriptSplitPattern() - { - return null; - } - - private static final Pattern PROC_PATTERN = Pattern.compile("^\\s*SELECT\\s+core\\.(executeJava(?:Upgrade|Initialization)Code\\s*\\(\\s*'(.+)'\\s*\\))\\s*;\\s*$", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE); - - @NotNull - @Override - protected Pattern getSQLScriptProcPattern() - { - return PROC_PATTERN; - } - - @Override - protected void checkSqlScript(String lowerNoComments, String lowerNoCommentsNoWhiteSpace, Collection errors) - { - if (lowerNoCommentsNoWhiteSpace.contains("setsearch_pathto")) - errors.add("Do not use \"SET search_path TO \". Instead, schema-qualify references to all objects."); - - if (!lowerNoCommentsNoWhiteSpace.endsWith(";")) - errors.add("Script must end with a semicolon"); - } - - @Override - public @NotNull Collection getAutoIncrementSequences(TableInfo table) - { - SQLFragment sql = new SQLFragment(""" - SELECT SchemaName, TableName, ColumnName, LastValue FROM ( - SELECT - s.relname AS SequenceName, -- Not used - tns.nspname AS SchemaName, - t.relname AS TableName, - a.attname AS ColumnName, - seq.last_value AS LastValue, - sns.nspname AS SequenceSchema -- Not used. In theory, sequence could live in a different schema, but not our practice - FROM - pg_depend d - JOIN - pg_class s ON d.objid = s.oid -- The sequence - JOIN - pg_namespace sns ON s.relnamespace = sns.oid - JOIN - pg_class t ON d.refobjid = t.oid -- The table - JOIN - pg_namespace tns ON t.relnamespace = tns.oid - JOIN - pg_attribute a ON d.refobjid = a.attrelid AND d.refobjsubid = a.attnum - JOIN - pg_sequences seq ON s.relname = seq.SequenceName AND tns.nspname = seq.SchemaName -- maybe sns.nspname instead? but that is slower... - WHERE - s.relkind = 'S' -- Sequence - AND t.relkind IN ('r', 'P') -- Table (regular table or partitioned table) - AND d.deptype IN ('a', 'i') -- Automatic dependency for DEFAULT or index-related for PK - ) AS x - WHERE SchemaName ILIKE ? AND TableName ILIKE ? - """, - table.getSchema().getName(), - table.getName() - ); - return new SqlSelector(table.getSchema(), sql).getCollection(Sequence.class); - } - - @Override - public String getBinaryDataType() - { - return "BYTEA"; - } - - @Override - public String getGlobalTempTablePrefix() - { - return DbSchema.TEMP_SCHEMA_NAME + "."; - } - - @Override - public String getDropIndexCommand(String tableName, String indexName) - { - return "DROP INDEX " + indexName; - } - - @Override - public String getCreateDatabaseSql(String dbName) - { - // This will handle both mixed case and special characters on PostgreSQL - var legal = makeIdentifierFromMetaDataName(dbName); - return new SQLFragment("CREATE DATABASE ").appendIdentifier(legal).append(" WITH ENCODING 'UTF8'").getRawSQL(); - } - - @Override - public String getCreateSchemaSql(String schemaName) - { - if (!isLegalName(schemaName) || isReserved(schemaName)) - throw new IllegalArgumentException("Not a legal schema name: " + schemaName); - - //Quoted schema names are bad news - return "CREATE SCHEMA " + schemaName; - } - - @Override - public String getTruncateSql(String tableName) - { - // To be consistent with MS SQL server, always restart the sequence. Note that the default for postgres - // is to continue the sequence but we don't have this option with MS SQL Server - return "TRUNCATE TABLE " + tableName + " RESTART IDENTITY"; - } - - @Override - public List getChangeStatements(TableChange change) - { - List result = new ArrayList<>(); - switch (change.getType()) - { - case CreateTable -> result.addAll(getCreateTableStatements(change)); - case DropTable -> { - SQLFragment f = new SQLFragment("DROP TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - result.add(f); - } - case AddColumns -> result.addAll(getAddColumnsStatements(change)); - case DropColumns -> result.add(getDropColumnsStatement(change)); - case RenameColumns -> result.addAll(getRenameColumnsStatement(change)); - case DropIndicesByName -> result.addAll(getDropIndexByNameStatements(change)); - case AddIndices -> result.addAll(getCreateIndexStatements(change)); - case ResizeColumns, ChangeColumnTypes -> result.addAll(getChangeColumnTypeStatement(change)); - case DropConstraints -> result.addAll(getDropConstraintsStatement(change)); - case AddConstraints -> result.addAll(getAddConstraintsStatement(change)); - default -> throw new IllegalArgumentException("Unsupported change type: " + change.getType()); - } - - return result; - } - - private Collection getDropIndexByNameStatements(TableChange change) - { - List statements = new ArrayList<>(); - for (String indexName : change.getIndicesToBeDroppedByName()) - { - statements.add(getDropIndexCommand(change, indexName)); - } - return statements; - } - - private SQLFragment getDropIndexCommand(TableChange change, String indexName) - { - SQLFragment f = new SQLFragment("DROP INDEX "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(indexName); - return f; - } - - /** - * We've historically created lower-cased column names in provisioned tables in Postgres. Keep doing that - * for consistency, though ideally we'd stop doing this and update all existing provisioned tables. - */ - private DatabaseIdentifier makePropertyIdentifier(String name) - { - if (isIdentifierTooLong(name)) - throw new UnsupportedOperationException("Name is too long: " + name); - return new _DatabaseIdentifier(name, quoteIdentifier(name.toLowerCase()), this); - } - - /** - * Generate the Alter Table statement to change the size or type of the column - *

- * NOTE: expects data size check to be done prior, - * will throw a SQL exception if not able to change size due to existing data - */ - private List getChangeColumnTypeStatement(TableChange change) - { - List statements = new ArrayList<>(); - - // Postgres allows executing multiple ALTER COLUMN statements under one ALTER TABLE - List nonDateTimeClauses = new ArrayList<>(); - - for (PropertyStorageSpec column : change.getColumns()) - { - PropertyType oldPropertyType = change.getOldPropTypes().get(column.getName()); - DatabaseIdentifier columnIdent = makePropertyIdentifier(column.getName()); - if (column.getJdbcType().isDateOrTime()) - { - String tempColumnName = column.getName() + "~~temp~~"; - DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); - - // 1) ADD temp column - SQLFragment addTemp = new SQLFragment("ALTER TABLE "); - addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); - statements.add(addTemp); - - // 2) UPDATE: copy casted value to temp column - SQLFragment update = new SQLFragment("UPDATE "); - update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - update.append(" SET ").appendIdentifier(tempColumnIdent); - update.append(" = CAST(").appendIdentifier(columnIdent).append(" AS ").append(getSqlTypeName(column)).append(")"); - statements.add(update); - - // 3) DROP original column - SQLFragment drop = new SQLFragment("ALTER TABLE "); - drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); - statements.add(drop); - - // 4) RENAME temp column to original column name - SQLFragment rename = new SQLFragment("ALTER TABLE "); - rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); - statements.add(rename); - } - else if (oldPropertyType == PropertyType.MULTI_CHOICE && column.getJdbcType().isText()) - { - // Converting from text[] (array) to text requires an intermediate column and transformation - String tempColumnName = column.getName() + "~~temp~~"; - DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); - - // 1) ADD temp column of text type - SQLFragment addTemp = new SQLFragment("ALTER TABLE "); - addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); - statements.add(addTemp); - - // 2) UPDATE: convert and copy value to temp column - // - NULL array -> NULL - // - empty array -> NULL - // - non-empty array -> concatenate array elements with comma (', ') - SQLFragment update = new SQLFragment("UPDATE "); - update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - update.append(" SET ").appendIdentifier(tempColumnIdent).append(" = CASE "); - update.append(" WHEN ").appendIdentifier(columnIdent).append(" IS NULL THEN NULL "); - update.append(" WHEN COALESCE(array_length(").appendIdentifier(columnIdent).append(", 1), 0) = 0 THEN NULL "); - update.append(" ELSE array_to_string(").appendIdentifier(columnIdent).append(", ', ') END"); - statements.add(update); - - // 3) DROP original column - SQLFragment drop = new SQLFragment("ALTER TABLE "); - drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); - statements.add(drop); - - // 4) RENAME temp column to original column name - SQLFragment rename = new SQLFragment("ALTER TABLE "); - rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); - statements.add(rename); - } - else if (column.getJdbcType() == JdbcType.ARRAY) - { - // Converting from text to text[] requires an intermediate column and transformation - String tempColumnName = column.getName() + "~~temp~~"; - DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); - - // 1) ADD temp column of array type (e.g., text[]) - SQLFragment addTemp = new SQLFragment("ALTER TABLE "); - addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); - statements.add(addTemp); - - // 2) UPDATE: copy converted value to temp column as single-element array - // - NULL or blank ('') -> empty array [] - // - otherwise -> single-element array [text] - SQLFragment update = new SQLFragment("UPDATE "); - update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - update.append(" SET ").appendIdentifier(tempColumnIdent); - update.append(" = CASE WHEN ").appendIdentifier(columnIdent).append(" IS NULL OR ").appendIdentifier(columnIdent).append(" = '' THEN ARRAY[]::text[] ELSE ARRAY["); - update.appendIdentifier(columnIdent).append("]::text[] END"); - statements.add(update); - - // 3) DROP original column - SQLFragment drop = new SQLFragment("ALTER TABLE "); - drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); - statements.add(drop); - - // 4) RENAME temp column to original column name - SQLFragment rename = new SQLFragment("ALTER TABLE "); - rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); - statements.add(rename); - } - else - { - String dbType; - if (column.getJdbcType().isText()) - { - // Using the common default max size to make type change to text - dbType = column.getSize() == -1 || column.getSize() > SqlDialect.MAX_VARCHAR_SIZE ? - getSqlTypeName(JdbcType.LONGVARCHAR) : - getSqlTypeName(column.getJdbcType()) + "(" + column.getSize().toString() + ")"; - } - else if (column.getJdbcType().isDecimal()) - { - dbType = getSqlTypeName(column.getJdbcType()) + DEFAULT_DECIMAL_SCALE_PRECISION; - } - else - { - dbType = getSqlTypeName(column.getJdbcType()); - } - - SQLFragment clause = new SQLFragment(); - clause.append("ALTER COLUMN ").appendIdentifier(columnIdent).append(" TYPE ").append(dbType); - nonDateTimeClauses.add(clause); - } - } - - if (!nonDateTimeClauses.isEmpty()) - { - SQLFragment alter = new SQLFragment("ALTER TABLE "); - alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - alter.append(" "); - String sep = ""; - for (SQLFragment c : nonDateTimeClauses) - { - alter.append(sep).append(c); - sep = ", "; - } - statements.add(alter); - } - - return statements; - } - - private List getRenameColumnsStatement(TableChange change) - { - List statements = new ArrayList<>(); - for (Map.Entry oldToNew : change.getColumnRenames().entrySet()) - { - DatabaseIdentifier oldIdentifier = makePropertyIdentifier(oldToNew.getKey()); - DatabaseIdentifier newIdentifier = makePropertyIdentifier(oldToNew.getValue()); - if (!oldIdentifier.equals(newIdentifier)) - { - SQLFragment f = new SQLFragment("ALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" RENAME COLUMN ").appendIdentifier(oldIdentifier).append(" TO ").appendIdentifier(newIdentifier); - statements.add(f); - } - } - - // TODO: This loop should not guess the name of the old indices; instead, it should look them up. - // TableChange.setIndexedColumns() could set _indexRenames providing the name, and then this code uses that info. - // Or maybe schemaTableInfo.getAllIndices() and then use Index.isSameIndex() to find names. Issue 53838. - for (Map.Entry oldToNew : change.getIndexRenames().entrySet()) - { - PropertyStorageSpec.Index oldIndex = oldToNew.getKey(); - PropertyStorageSpec.Index newIndex = oldToNew.getValue(); - String oldName = nameIndex(change.getTableName(), oldIndex.columnNames); // TODO: Look up name - String newName = nameIndex(change.getTableName(), newIndex.columnNames); - if (!oldName.equals(newName)) - { - SQLFragment f = new SQLFragment("ALTER INDEX "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(oldName); - f.append(" RENAME TO ").appendIdentifier(newName); - statements.add(f); - } - } - - return statements; - } - - private SQLFragment getDropColumnsStatement(TableChange change) - { - List sqlParts = new ArrayList<>(); - for (PropertyStorageSpec prop : change.getColumns()) - { - SQLFragment sql = new SQLFragment("DROP COLUMN "); - if (prop.getExactName()) - { - sql.append(quoteIdentifier(prop.getName())); - } - else - { - sql.appendIdentifier(makePropertyIdentifier(prop.getName())); - } - sqlParts.add(sql); - } - - SQLFragment f = new SQLFragment("ALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" ").append(sqlParts, ", "); - return f; - } - - // TODO if there are cases where user-defined columns need indices, this method will need to support - // creating indices like getCreateTableStatement does. - private List getAddColumnsStatements(TableChange change) - { - List statements = new ArrayList<>(); - String pkColumn = null; - Constraint constraint = null; - - List columnSpecs = new ArrayList<>(); - for (PropertyStorageSpec prop : change.getColumns()) - { - columnSpecs.add(getSqlColumnSpec(prop)); - if (prop.isPrimaryKey()) - { - assert null == pkColumn : "no more than one primary key defined"; - pkColumn = prop.getName(); - constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); - } - } - - SQLFragment alter = new SQLFragment("ALTER TABLE "); - alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - alter.append(" "); - String sep = ""; - for (SQLFragment col : columnSpecs) - { - alter.append(sep); - alter.append("ADD COLUMN "); - alter.append(col); - sep = ", "; - } - statements.add(alter); - if (null != pkColumn) - { - SQLFragment addPk = new SQLFragment("ALTER TABLE "); - addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) - .append(" ").append(constraint.getType().toString()).append(" (") - .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); - statements.add(addPk); - } - - return statements; - } - - private List getDropConstraintsStatement(TableChange change) - { - return change.getConstraints().stream().map(constraint -> { - SQLFragment f = new SQLFragment("ALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" DROP CONSTRAINT ").appendIdentifier(constraint.getName()); - return f; - }).collect(Collectors.toList()); - } - - private List getAddConstraintsStatement(TableChange change) - { - List statements = new ArrayList<>(); - Collection constraints = change.getConstraints(); - - if (null!=constraints && !constraints.isEmpty()) - { - statements = constraints.stream().map(constraint -> { - List columns = new ArrayList<>(); - for (String col : constraint.getColumns()) - { - columns.add(new SQLFragment().appendIdentifier(col)); - } - - SQLFragment f = new SQLFragment(); - f.append("DO $$\nBEGIN\nIF NOT EXISTS\n(SELECT 1 FROM information_schema.constraint_column_usage\nWHERE table_name = ") - .appendStringLiteral(change.getSchemaName() + "." + change.getTableName(), this) - .append(" and constraint_name = ") - .appendStringLiteral(constraint.getName(), this) - .append(") THEN\nALTER TABLE "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()).append(" ") - .append(constraint.getType().toString()).append(" (") - .append(columns, ",") - .append(")").appendEOS().append("\nEND IF)").appendEOS().append("\nEND$$").appendEOS(); - return f; - }).collect(Collectors.toList()); - } - - return statements; - } - - private List getCreateTableStatements(TableChange change) - { - List statements = new ArrayList<>(); - List createTableSqlParts = new ArrayList<>(); - String pkColumn = null; - for (PropertyStorageSpec prop : change.getColumns()) - { - createTableSqlParts.add(getSqlColumnSpec(prop)); - if (prop.isPrimaryKey()) - { - assert null == pkColumn : "no more than one primary key defined"; - pkColumn = prop.getName(); - } - } - - for (PropertyStorageSpec.ForeignKey foreignKey : change.getForeignKeys()) - { - DbSchema schema = DbSchema.get(foreignKey.getSchemaName(), DbSchemaType.Module); - TableInfo tableInfo = foreignKey.isProvisioned() ? - foreignKey.getTableInfoProvisioned() : - schema.getTable(foreignKey.getTableName()); - String constraintName = "fk_" + foreignKey.getColumnName() + "_" + change.getTableName() + "_" + tableInfo.getName(); - SQLFragment fkFrag = new SQLFragment("CONSTRAINT "); - fkFrag.appendIdentifier(constraintName) - .append(" FOREIGN KEY (") - .appendIdentifier(makePropertyIdentifier(foreignKey.getColumnName())) - .append(") REFERENCES ") - .appendIdentifier(tableInfo.getSchema().getName()).append(".").appendIdentifier(tableInfo.getName()) - .append(" (") - .appendIdentifier(makePropertyIdentifier(foreignKey.getForeignColumnName())) - .append(")"); - createTableSqlParts.add(fkFrag); - } - - SQLFragment create = new SQLFragment("CREATE TABLE "); - create.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - create.append(" (").append(createTableSqlParts, ", ").append(")"); - statements.add(create); - if (null != pkColumn) - { - // Making this just for consistent naming - Constraint constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); - - SQLFragment addPk = new SQLFragment("ALTER TABLE "); - addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) - .append(" ").append(constraint.getType().toString()).append(" (") - .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); - statements.add(addPk); - } - - statements.addAll(getCreateIndexStatements(change)); - statements.addAll(getAddConstraintsStatement(change)); - return statements; - } - - private List getCreateIndexStatements(TableChange change) - { - List statements = new ArrayList<>(); - for (PropertyStorageSpec.Index index : change.getIndexedColumns()) - { - String newIndexName = nameIndex(change.getTableName(), index.columnNames); - SQLFragment f = new SQLFragment("CREATE "); - if (index.isUnique) - f.append("UNIQUE "); - f.append("INDEX ").appendIdentifier(newIndexName).append(" ON "); - f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - f.append(" ("); - String separator = ""; - for (String columnName : index.columnNames) - { - f.append(separator).appendIdentifier(makePropertyIdentifier(columnName)); - separator = ", "; - } - f.append(")"); - f.appendEOS(); - statements.add(f); - - if (index.isClustered) - { - SQLFragment c = new SQLFragment(); - c.append(PropertyStorageSpec.CLUSTER_TYPE.CLUSTER.toString()).append(" "); - c.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); - c.append(" USING ").appendIdentifier(newIndexName); - statements.add(c); - } - } - return statements; - } - - @Override - public String nameIndex(String tableName, String[] indexedColumns) - { - return AliasManager.makeLegalName(tableName + '_' + StringUtils.join(indexedColumns, "_"), this); - } - - private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop) - { - return getSqlColumnSpec(prop, prop.getName()); - } - - private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop, String columnName) - { - SQLFragment colSpec = new SQLFragment(); - colSpec.appendIdentifier(makePropertyIdentifier(columnName)).append(" "); - colSpec.append(getSqlTypeName(prop)); - - // Apply size and precision to varchar and Decimal types - if (prop.getJdbcType() == JdbcType.VARCHAR && prop.getSize() != -1 && prop.getSize() <= SqlDialect.MAX_VARCHAR_SIZE) - { - colSpec.append("(").append(prop.getSize().toString()).append(")"); - } - else if (prop.getJdbcType() == JdbcType.DECIMAL) - { - colSpec.append(DEFAULT_DECIMAL_SCALE_PRECISION); - } - - if (prop.isPrimaryKey() || !prop.isNullable()) - colSpec.append(" NOT NULL"); - - if (null != prop.getDefaultValue()) - { - if (prop.getJdbcType() == JdbcType.BOOLEAN) - { - colSpec.append(" DEFAULT "); - colSpec.append((Boolean)prop.getDefaultValue() ? getBooleanTRUE() : getBooleanFALSE()); - } - else if (prop.getJdbcType() == JdbcType.VARCHAR) - { - colSpec.append(" DEFAULT "); - colSpec.append(getStringHandler().quoteStringLiteral(prop.getDefaultValue().toString())); - } - else - { - throw new IllegalArgumentException("Default value on type " + prop.getJdbcType().name() + " is not supported."); - } - } - return colSpec; - } - - @Override - public void purgeTempSchema(Map createdTableNames) - { - try - { - trackTempTables(createdTableNames); - } - catch (SQLException e) - { - LOG.warn("error cleaning up temp schema", e); - } - - DbSchema coreSchema = CoreSchema.getInstance().getSchema(); - SqlExecutor executor = new SqlExecutor(coreSchema); - - //rs = conn.getMetaData().getFunctions(dbName, tempSchemaName, "%"); - - new SqlSelector(coreSchema, "SELECT proname AS SPECIFIC_NAME, CAST(proargtypes AS VARCHAR) FROM pg_proc WHERE pronamespace=(select oid from pg_namespace where nspname = ?)", DbSchema.getTemp().getName()).forEach( - new ForEachBlock<>() - { - private Map _types = null; - - @Override - public void exec(ResultSet rs) throws SQLException - { - if (null == _types) - { - _types = new HashMap<>(); - new SqlSelector(coreSchema, "SELECT CAST(oid AS VARCHAR) as oid, typname, (select nspname from pg_namespace where oid = typnamespace) as nspname FROM pg_type").forEach(type -> - _types.put(type.getString(1), quoteIdentifier(type.getString(3)) + "." + quoteIdentifier(type.getString(2)))); - } - - String name = rs.getString(1); - String[] oids = StringUtils.split(rs.getString(2), ' '); - SQLFragment drop = new SQLFragment("DROP FUNCTION temp.").append(name); - drop.append("("); - String comma = ""; - for (String oid : oids) - { - drop.append(comma).append(_types.get(oid)); - comma = ","; - } - drop.append(")"); - - try - { - executor.execute(drop); - } - catch (BadSqlGrammarException x) - { - LOG.warn("could not clean up postgres function : temp." + name, x); - } - } - }); - - // TODO delete types in temp schema as well! search for "CREATE TYPE" in StatementUtils.java - } - - // - // ARRAY and SET syntax - // - - // NOTE LabKey currently does not support ARRAY[VARCHAR], use ARRAY[text] instead - // - // Postgres string literals can be auto-cast to both VARCHAR and TEXT. These all work - // 'color' = 'color'::varchar - // 'color' = 'color'::text - // ARRAY['color'] = ARRAY['color'::text]; - // However, ARRAY[text] cannot be auto cast to ARRAY[varchar] - // ARRAY['color'] = ARRAY['color'::varchar]; -- ERROR! - // - - - @Override - public boolean supportsArrays() - { - return true; - } - - @Override - public SQLFragment array_construct(SQLFragment[] elements) - { - SQLFragment ret = new SQLFragment(); - ret.append("ARRAY["); - String separator = ""; - for (SQLFragment element : elements) - { - ret.append(separator); - ret.append(element); - separator = ", "; - } - ret.append("]"); - return ret; - } - - @Override - public SQLFragment array_all_in_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append("(").append(a).append(") <@ (").append(b).append(")"); - return ret; - } - - @Override - public SQLFragment array_some_in_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append("(").append(a).append(") && (").append(b).append(")"); - return ret; - } - - @Override - public SQLFragment array_none_in_array(SQLFragment a, SQLFragment b) - { - return new SQLFragment(" NOT (").append(array_some_in_array(a, b)).append(")"); - } - - @Override - public SQLFragment array_same_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append(array_all_in_array(a, b)).append(" AND ").append(array_all_in_array(b, a)); - return ret; - } - - @Override - public SQLFragment array_not_same_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append("NOT (").append(array_all_in_array(a, b)).append(") OR NOT (").append(array_all_in_array(b, a)).append(")"); - return ret; - } - - @Override - public SQLFragment element_in_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append("(").append(a).append(")"); - // DOCs imply that IS NOT DISTINCT FROM ANY should work, but it doesn't??? - // ret.append(" IS NOT DISTINCT FROM ANY("); - ret.append(" = ANY("); - ret.append(b); - ret.append(")"); - return ret; - } - - @Override - public SQLFragment element_not_in_array(SQLFragment a, SQLFragment b) - { - SQLFragment ret = new SQLFragment(); - ret.append("(").append(a).append(")"); - // DOCs imply that IS NOT DISTINCT FROM ANY should work, but it doesn't??? - // ret.append(" IS DISTINCT FROM ALL("); - ret.append(" <> ALL("); - ret.append(b); - ret.append(")"); - return ret; - } -} +/* + * Copyright (c) 2012-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.core.dialect; + +import jakarta.servlet.ServletException; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Constraint; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DatabaseIdentifier; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.InClauseGenerator; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.ParameterMarkerInClauseGenerator; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.Selector; +import org.labkey.api.data.Selector.ForEachBlock; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableChange; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TempTableInClauseGenerator; +import org.labkey.api.data.TempTableTracker; +import org.labkey.api.data.dialect.BasePostgreSqlDialect; +import org.labkey.api.data.dialect.DialectStringHandler; +import org.labkey.api.data.dialect.JdbcHelper; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.data.dialect.StandardJdbcHelper; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.query.AliasManager; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.view.template.Warnings; +import org.labkey.core.admin.sql.ScriptReorderer; +import org.springframework.jdbc.BadSqlGrammarException; + +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/* + * This is the base class defining PostgreSQL-specific (i.e., not Redshift) behavior. PostgreSQL 9.2 is no longer + * supported; however, we keep this class to track changes we implemented specifically for this version. + */ +abstract class PostgreSql92Dialect extends BasePostgreSqlDialect +{ + public static final String PRODUCT_NAME = "PostgreSQL"; + + // This has been the standard PostgreSQL identifier max byte length for many years. However, this could change in + // the future plus servers can be compiled with a different limit, so we query this setting on first connection to + // each database. + private int _maxIdentifierByteLength = 63; + private InClauseGenerator _inClauseGenerator; + + private final TempTableInClauseGenerator _tempTableInClauseGenerator = new TempTableInClauseGenerator(); + private final AtomicBoolean _arraySortFunctionExists = new AtomicBoolean(false); + + @Override + public void handleCreateDatabaseException(SQLException e) throws ServletException + { + if ("55006".equals(e.getSQLState())) + { + LOG.error("You must close down pgAdmin III and all other applications accessing PostgreSQL."); + throw (new ServletException("Close down or disconnect pgAdmin III and all other applications accessing PostgreSQL", e)); + } + else + { + super.handleCreateDatabaseException(e); + } + } + + @Override + public void prepareDriver(Class driverClass) + { + // PostgreSQL driver 42.0.0 added logging via the Java Logging API (java.util.logging). This caused the driver to + // start logging SQLExceptions (such as the initial connection failure on bootstrap) to the console... harmless + // but annoying. This code suppresses the driver logging. + Logger pgjdbcLogger = LogManager.getLogManager().getLogger("org.postgresql"); + + if (null != pgjdbcLogger) + pgjdbcLogger.setLevel(Level.OFF); + } + + // Make sure that the PL/pgSQL language is enabled in the associated database. If not, throw. Since 9.0, PostgreSQL has + // shipped with PL/pgSQL enabled by default, so the check is no longer critical, but continue to verify just to be safe. + @Override + public void prepareNewLabKeyDatabase(DbScope scope) + { + if (new SqlSelector(scope, "SELECT * FROM pg_language WHERE lanname = 'plpgsql'").exists()) + return; + + String dbName = scope.getDatabaseName(); + String message = "PL/pgSQL is not enabled in the \"" + dbName + "\" database because it is not enabled in your Template1 master database."; + String advice = "Use PostgreSQL's 'createlang' command line utility to enable PL/pgSQL in the \"" + dbName + "\" database then restart Tomcat."; + + throw new ConfigurationException(message, advice); + } + + @Override + public String prepare(DbScope scope) + { + initializeInClauseGenerator(scope); + determineIfArraySortFunctionExists(scope); + return super.prepare(scope); + } + + @NotNull + @Override + protected Set getReservedWords() + { + Set words = super.getReservedWords(); + words.add("collation"); + + return words; + } + + /* + These override method implementations were migrated from PostgreSql91Dialect when that class was promoted to api: + getProductName() + createStringHandler() + getJdbcHelper() + getScriptWarnings() + initializeInClauseGenerator() + */ + + @Override + public String getProductName() + { + return PRODUCT_NAME; + } + + // Query PostgreSQL-specific settings + @Override + protected void determineSettings(DbScope scope) + { + if (getServerType().supportsSpecialMetadataQueries()) + { + super.determineSettings(scope); + + String value = new SqlSelector(scope, "SELECT setting FROM pg_settings WHERE name = 'max_identifier_length'").getObject(String.class); + try + { + _maxIdentifierByteLength = Integer.valueOf(value); + } + catch (NumberFormatException e) + { + LOG.error("Couldn't parse max_identifier_length; continuing with default value of {}", _maxIdentifierByteLength, e); + } + } + } + + @Override + protected DialectStringHandler createStringHandler() + { + // TODO: Isn't this the wrong setting? Should we be looking at the "backslash_quote" setting instead? + if (getStandardConformingStrings()) + return super.createStringHandler(); + else + return new PostgreSqlNonConformingStringHandler(); + } + + /* + PostgreSQL example connection URLs we need to parse: + + jdbc:postgresql:database + jdbc:postgresql://host/database + jdbc:postgresql://host:port/database + jdbc:postgresql:database?user=fred&password=secret&ssl=true + jdbc:postgresql://host/database?user=fred&password=secret&ssl=true + jdbc:postgresql://host:port/database?user=fred&password=secret&ssl=true + */ + @Override + public JdbcHelper getJdbcHelper() + { + return new StandardJdbcHelper(PostgreSqlDialectFactory.JDBC_PREFIX); + } + + @Override + public String getDefaultDatabaseName() + { + return "template1"; + } + + @Override + public boolean canExecuteUpgradeScripts() + { + return true; + } + + @Override + public Collection getScriptWarnings(String name, String sql) + { + // Strip out all block- and single-line comments + Pattern commentPattern = Pattern.compile(ScriptReorderer.COMMENT_REGEX, Pattern.DOTALL + Pattern.MULTILINE); + Matcher matcher = commentPattern.matcher(sql); + String noComments = matcher.replaceAll(""); + + List warnings = new LinkedList<>(); + + // Split statements by semicolon and CRLF + for (String statement : noComments.split(";[\\n\\r]+")) + { + if (Strings.CI.startsWith(statement.trim(), "SET ")) + warnings.add(statement); + } + + return warnings; + } + + @Override + public String getSQLScriptPath() + { + return "postgresql"; + } + + @Override + public String getUniqueIdentType() + { + return "SERIAL"; + } + + @Override + public boolean supportsGroupConcat() + { + return getServerType().supportsGroupConcat(); + } + + @Override + public boolean supportsSelectConcat() + { + return true; + } + + @Override + public SQLFragment getSelectConcat(SQLFragment selectSql, String delimiter) + { + SQLFragment result = new SQLFragment("array_to_string(array("); + result.append(selectSql); + result.append("), "); + result.append(getStringHandler().quoteStringLiteral(delimiter)); + result.append(")"); + return result; + } + + // Does this datasource include our sort array function? The LabKey datasource should always have it, but external datasources might not + private void determineIfArraySortFunctionExists(DbScope scope) + { + if (getServerType().supportsSpecialMetadataQueries()) + { + Selector selector = new SqlSelector(scope, "SELECT * FROM pg_catalog.pg_namespace n INNER JOIN pg_catalog.pg_proc p ON pronamespace = n.oid WHERE nspname = 'core' AND proname = 'sort'"); + _arraySortFunctionExists.set(selector.exists()); + } + + // Array sort function should always exist in LabKey scope (for now) + assert !scope.isLabKeyScope() || _arraySortFunctionExists.get(); + } + + @Override + public SQLFragment getGroupConcat(SQLFragment sql, boolean distinct, boolean sorted, @NotNull SQLFragment delimiterSQL, boolean includeNulls) + { + // Sort function might not exist in external datasource; skip that syntax if not + boolean useSortFunction = sorted && _arraySortFunctionExists.get(); + SQLFragment result = new SQLFragment(); + + if (useSortFunction) + { + result.append("array_to_string("); + result.append("core.sort("); // TODO: Switch to use ORDER BY option inside array aggregate instead of our custom function + result.append("array_agg("); + if (distinct) + { + result.append("DISTINCT "); + } + + if (includeNulls) + { + result.append("COALESCE(CAST("); + result.append(sql); + result.append(" AS VARCHAR), '')"); + } + else + { + result.append(sql); + } + + result.append(")"); // array_agg + result.append(")"); // core.sort + } + else + { + result.append("string_agg("); + if (distinct) + { + result.append("DISTINCT "); + } + + if (includeNulls) + { + result.append("COALESCE("); + result.append(sql); + result.append("::text, '')"); + } + else + { + result.append(sql); + result.append("::text"); + } + } + + result.append(", "); + result.append(delimiterSQL); + result.append(")"); // array_to_string | string_agg + + return result; + } + + @Override + public SQLFragment getAnalyzeCommandForTable(String tableName) + { + return new SQLFragment("ANALYZE ").appendIdentifier(tableName); + } + + private void initializeInClauseGenerator(DbScope scope) + { + _inClauseGenerator = getJdbcVersion(scope) >= 4 ? new ArrayParameterInClauseGenerator(scope) : new ParameterMarkerInClauseGenerator(); + } + + @Override + public InClauseGenerator getDefaultInClauseGenerator() + { + return _inClauseGenerator; + } + + @Override + public TempTableInClauseGenerator getTempTableInClauseGenerator() + { + return _tempTableInClauseGenerator; + } + + @Override + public void addAdminWarningMessages(Warnings warnings, boolean showAllWarnings) + { + super.addAdminWarningMessages(warnings, showAllWarnings); + if (showAllWarnings) + warnings.add(HtmlString.of(PostgreSqlDialectFactory.getStandardWarningMessage("has not been tested against", getMajorVersion() + ".x"))); + } + + private int getIdentifierMaxByteLength() + { + return _maxIdentifierByteLength; + } + + @Override + public boolean isIdentifierTooLong(String identifier) + { + return identifier.getBytes(StandardCharsets.UTF_8).length > getIdentifierMaxByteLength(); + } + + @Override + public String truncateAndJoin(String... parts) + { + String ret = String.join("$", parts); + + if (isIdentifierTooLong(ret)) + { + int maxBytes = getIdentifierMaxByteLength(); + StringBuilder sb = new StringBuilder(maxBytes); + int partsLength = parts.length; + int remainingBytes = maxBytes - partsLength + 1; // Make room for dollar signs + for (int i = 0; i < partsLength; i++) + { + String truncated = truncateBytes(parts[i], remainingBytes / (partsLength - i)); + if (i > 0) + sb.append("$"); + sb.append(truncated); + remainingBytes -= truncated.getBytes(StandardCharsets.UTF_8).length; + } + ret = sb.toString(); + assert ret.getBytes(StandardCharsets.UTF_8).length <= maxBytes; + } + + return ret; + } + + @Override + public String truncate(String str, int reserved) + { + return truncateBytes(str, getIdentifierMaxByteLength() - reserved); + } + + // Truncates based on UTF-8 bytes + private static String truncateBytes(String str, int maxBytes) + { + if (maxBytes < 13) + throw new IllegalStateException("maxBytes for legal name too small: " + maxBytes); + int len = str.getBytes(StandardCharsets.UTF_8).length; + if (len > maxBytes) + { + String prefix = generateIdentifierPrefix(str); + str = prefix + StringUtilsLabKey.rightUtf8Bytes(str, maxBytes - prefix.getBytes(StandardCharsets.UTF_8).length); + } + assert str.getBytes(StandardCharsets.UTF_8).length <= maxBytes; + assert !StringUtilsLabKey.hasBrokenSurrogate(str); + return str; + } + + @Override + public boolean canShowExecutionPlan(ExecutionPlanType type) + { + return true; + } + + @Override + protected Collection getQueryExecutionPlan(Connection conn, DbScope scope, SQLFragment sql, ExecutionPlanType type) + { + SQLFragment copy = new SQLFragment(sql); + copy.insert(0, type == ExecutionPlanType.Estimated ? "EXPLAIN " : "EXPLAIN ANALYZE "); + + return new SqlSelector(scope, conn, copy).getCollection(String.class); + } + + @Override + // No need to split up PostgreSQL scripts; execute all statements in a single block (unless we have a special stored proc call). + protected Pattern getSQLScriptSplitPattern() + { + return null; + } + + private static final Pattern PROC_PATTERN = Pattern.compile("^\\s*SELECT\\s+core\\.(executeJava(?:Upgrade|Initialization)Code\\s*\\(\\s*'(.+)'\\s*\\))\\s*;\\s*$", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE); + + @NotNull + @Override + protected Pattern getSQLScriptProcPattern() + { + return PROC_PATTERN; + } + + @Override + protected void checkSqlScript(String lowerNoComments, String lowerNoCommentsNoWhiteSpace, Collection errors) + { + if (lowerNoCommentsNoWhiteSpace.contains("setsearch_pathto")) + errors.add("Do not use \"SET search_path TO \". Instead, schema-qualify references to all objects."); + + if (!lowerNoCommentsNoWhiteSpace.endsWith(";")) + errors.add("Script must end with a semicolon"); + } + + @Override + public @NotNull Collection getAutoIncrementSequences(TableInfo table) + { + SQLFragment sql = new SQLFragment(""" + SELECT SchemaName, TableName, ColumnName, LastValue FROM ( + SELECT + s.relname AS SequenceName, -- Not used + tns.nspname AS SchemaName, + t.relname AS TableName, + a.attname AS ColumnName, + seq.last_value AS LastValue, + sns.nspname AS SequenceSchema -- Not used. In theory, sequence could live in a different schema, but not our practice + FROM + pg_depend d + JOIN + pg_class s ON d.objid = s.oid -- The sequence + JOIN + pg_namespace sns ON s.relnamespace = sns.oid + JOIN + pg_class t ON d.refobjid = t.oid -- The table + JOIN + pg_namespace tns ON t.relnamespace = tns.oid + JOIN + pg_attribute a ON d.refobjid = a.attrelid AND d.refobjsubid = a.attnum + JOIN + pg_sequences seq ON s.relname = seq.SequenceName AND tns.nspname = seq.SchemaName -- maybe sns.nspname instead? but that is slower... + WHERE + s.relkind = 'S' -- Sequence + AND t.relkind IN ('r', 'P') -- Table (regular table or partitioned table) + AND d.deptype IN ('a', 'i') -- Automatic dependency for DEFAULT or index-related for PK + ) AS x + WHERE SchemaName ILIKE ? AND TableName ILIKE ? + """, + table.getSchema().getName(), + table.getName() + ); + return new SqlSelector(table.getSchema(), sql).getCollection(Sequence.class); + } + + @Override + public String getBinaryDataType() + { + return "BYTEA"; + } + + @Override + public String getGlobalTempTablePrefix() + { + return DbSchema.TEMP_SCHEMA_NAME + "."; + } + + @Override + public String getDropIndexCommand(String tableName, String indexName) + { + return "DROP INDEX " + indexName; + } + + @Override + public String getCreateDatabaseSql(String dbName) + { + // This will handle both mixed case and special characters on PostgreSQL + var legal = makeIdentifierFromMetaDataName(dbName); + return new SQLFragment("CREATE DATABASE ").appendIdentifier(legal).append(" WITH ENCODING 'UTF8'").getRawSQL(); + } + + @Override + public String getCreateSchemaSql(String schemaName) + { + if (!isLegalName(schemaName) || isReserved(schemaName)) + throw new IllegalArgumentException("Not a legal schema name: " + schemaName); + + //Quoted schema names are bad news + return "CREATE SCHEMA " + schemaName; + } + + @Override + public String getTruncateSql(String tableName) + { + // To be consistent with MS SQL server, always restart the sequence. Note that the default for postgres + // is to continue the sequence but we don't have this option with MS SQL Server + return "TRUNCATE TABLE " + tableName + " RESTART IDENTITY"; + } + + @Override + public List getChangeStatements(TableChange change) + { + List result = new ArrayList<>(); + switch (change.getType()) + { + case CreateTable -> result.addAll(getCreateTableStatements(change)); + case DropTable -> { + SQLFragment f = new SQLFragment("DROP TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + result.add(f); + } + case AddColumns -> result.addAll(getAddColumnsStatements(change)); + case DropColumns -> result.add(getDropColumnsStatement(change)); + case RenameColumns -> result.addAll(getRenameColumnsStatement(change)); + case DropIndicesByName -> result.addAll(getDropIndexByNameStatements(change)); + case AddIndices -> result.addAll(getCreateIndexStatements(change)); + case ResizeColumns, ChangeColumnTypes -> result.addAll(getChangeColumnTypeStatement(change)); + case DropConstraints -> result.addAll(getDropConstraintsStatement(change)); + case AddConstraints -> result.addAll(getAddConstraintsStatement(change)); + default -> throw new IllegalArgumentException("Unsupported change type: " + change.getType()); + } + + return result; + } + + private Collection getDropIndexByNameStatements(TableChange change) + { + List statements = new ArrayList<>(); + for (String indexName : change.getIndicesToBeDroppedByName()) + { + statements.add(getDropIndexCommand(change, indexName)); + } + return statements; + } + + private SQLFragment getDropIndexCommand(TableChange change, String indexName) + { + SQLFragment f = new SQLFragment("DROP INDEX "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(indexName); + return f; + } + + /** + * We've historically created lower-cased column names in provisioned tables in Postgres. Keep doing that + * for consistency, though ideally we'd stop doing this and update all existing provisioned tables. + */ + private DatabaseIdentifier makePropertyIdentifier(String name) + { + if (isIdentifierTooLong(name)) + throw new UnsupportedOperationException("Name is too long: " + name); + return new _DatabaseIdentifier(name, quoteIdentifier(name.toLowerCase()), this); + } + + /** + * Generate the Alter Table statement to change the size or type of the column + *

+ * NOTE: expects data size check to be done prior, + * will throw a SQL exception if not able to change size due to existing data + */ + private List getChangeColumnTypeStatement(TableChange change) + { + List statements = new ArrayList<>(); + + // Postgres allows executing multiple ALTER COLUMN statements under one ALTER TABLE + List nonDateTimeClauses = new ArrayList<>(); + + for (PropertyStorageSpec column : change.getColumns()) + { + PropertyType oldPropertyType = change.getOldPropTypes().get(column.getName()); + DatabaseIdentifier columnIdent = makePropertyIdentifier(column.getName()); + if (column.getJdbcType().isDateOrTime()) + { + String tempColumnName = column.getName() + "~~temp~~"; + DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); + + // 1) ADD temp column + SQLFragment addTemp = new SQLFragment("ALTER TABLE "); + addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); + statements.add(addTemp); + + // 2) UPDATE: copy casted value to temp column + SQLFragment update = new SQLFragment("UPDATE "); + update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + update.append(" SET ").appendIdentifier(tempColumnIdent); + update.append(" = CAST(").appendIdentifier(columnIdent).append(" AS ").append(getSqlTypeName(column)).append(")"); + statements.add(update); + + // 3) DROP original column + SQLFragment drop = new SQLFragment("ALTER TABLE "); + drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); + statements.add(drop); + + // 4) RENAME temp column to original column name + SQLFragment rename = new SQLFragment("ALTER TABLE "); + rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); + statements.add(rename); + } + else if (oldPropertyType == PropertyType.MULTI_CHOICE && column.getJdbcType().isText()) + { + // Converting from text[] (array) to text requires an intermediate column and transformation + String tempColumnName = column.getName() + "~~temp~~"; + DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); + + // 1) ADD temp column of text type + SQLFragment addTemp = new SQLFragment("ALTER TABLE "); + addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); + statements.add(addTemp); + + // 2) UPDATE: convert and copy value to temp column + // - NULL array -> NULL + // - empty array -> NULL + // - non-empty array -> concatenate array elements with comma (', ') + SQLFragment update = new SQLFragment("UPDATE "); + update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + update.append(" SET ").appendIdentifier(tempColumnIdent).append(" = CASE "); + update.append(" WHEN ").appendIdentifier(columnIdent).append(" IS NULL THEN NULL "); + update.append(" WHEN COALESCE(array_length(").appendIdentifier(columnIdent).append(", 1), 0) = 0 THEN NULL "); + update.append(" ELSE array_to_string(").appendIdentifier(columnIdent).append(", ', ') END"); + statements.add(update); + + // 3) DROP original column + SQLFragment drop = new SQLFragment("ALTER TABLE "); + drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); + statements.add(drop); + + // 4) RENAME temp column to original column name + SQLFragment rename = new SQLFragment("ALTER TABLE "); + rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); + statements.add(rename); + } + else if (column.getJdbcType() == JdbcType.ARRAY) + { + // Converting from text to text[] requires an intermediate column and transformation + String tempColumnName = column.getName() + "~~temp~~"; + DatabaseIdentifier tempColumnIdent = makePropertyIdentifier(tempColumnName); + + // 1) ADD temp column of array type (e.g., text[]) + SQLFragment addTemp = new SQLFragment("ALTER TABLE "); + addTemp.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addTemp.append(" ADD COLUMN ").append(getSqlColumnSpec(column, tempColumnName)); + statements.add(addTemp); + + // 2) UPDATE: copy converted value to temp column as single-element array + // - NULL or blank ('') -> empty array [] + // - otherwise -> single-element array [text] + SQLFragment update = new SQLFragment("UPDATE "); + update.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + update.append(" SET ").appendIdentifier(tempColumnIdent); + update.append(" = CASE WHEN ").appendIdentifier(columnIdent).append(" IS NULL OR ").appendIdentifier(columnIdent).append(" = '' THEN ARRAY[]::text[] ELSE ARRAY["); + update.appendIdentifier(columnIdent).append("]::text[] END"); + statements.add(update); + + // 3) DROP original column + SQLFragment drop = new SQLFragment("ALTER TABLE "); + drop.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + drop.append(" DROP COLUMN ").appendIdentifier(columnIdent); + statements.add(drop); + + // 4) RENAME temp column to original column name + SQLFragment rename = new SQLFragment("ALTER TABLE "); + rename.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + rename.append(" RENAME COLUMN ").appendIdentifier(tempColumnIdent).append(" TO ").appendIdentifier(columnIdent); + statements.add(rename); + } + else + { + String dbType; + if (column.getJdbcType().isText()) + { + // Using the common default max size to make type change to text + dbType = column.getSize() == -1 || column.getSize() > SqlDialect.MAX_VARCHAR_SIZE ? + getSqlTypeName(JdbcType.LONGVARCHAR) : + getSqlTypeName(column.getJdbcType()) + "(" + column.getSize().toString() + ")"; + } + else if (column.getJdbcType().isDecimal()) + { + dbType = getSqlTypeName(column.getJdbcType()) + DEFAULT_DECIMAL_SCALE_PRECISION; + } + else + { + dbType = getSqlTypeName(column.getJdbcType()); + } + + SQLFragment clause = new SQLFragment(); + clause.append("ALTER COLUMN ").appendIdentifier(columnIdent).append(" TYPE ").append(dbType); + nonDateTimeClauses.add(clause); + } + } + + if (!nonDateTimeClauses.isEmpty()) + { + SQLFragment alter = new SQLFragment("ALTER TABLE "); + alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + alter.append(" "); + String sep = ""; + for (SQLFragment c : nonDateTimeClauses) + { + alter.append(sep).append(c); + sep = ", "; + } + statements.add(alter); + } + + return statements; + } + + private List getRenameColumnsStatement(TableChange change) + { + List statements = new ArrayList<>(); + for (Map.Entry oldToNew : change.getColumnRenames().entrySet()) + { + DatabaseIdentifier oldIdentifier = makePropertyIdentifier(oldToNew.getKey()); + DatabaseIdentifier newIdentifier = makePropertyIdentifier(oldToNew.getValue()); + if (!oldIdentifier.equals(newIdentifier)) + { + SQLFragment f = new SQLFragment("ALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" RENAME COLUMN ").appendIdentifier(oldIdentifier).append(" TO ").appendIdentifier(newIdentifier); + statements.add(f); + } + } + + // TODO: This loop should not guess the name of the old indices; instead, it should look them up. + // TableChange.setIndexedColumns() could set _indexRenames providing the name, and then this code uses that info. + // Or maybe schemaTableInfo.getAllIndices() and then use Index.isSameIndex() to find names. Issue 53838. + for (Map.Entry oldToNew : change.getIndexRenames().entrySet()) + { + PropertyStorageSpec.Index oldIndex = oldToNew.getKey(); + PropertyStorageSpec.Index newIndex = oldToNew.getValue(); + String oldName = nameIndex(change.getTableName(), oldIndex.columnNames); // TODO: Look up name + String newName = nameIndex(change.getTableName(), newIndex.columnNames); + if (!oldName.equals(newName)) + { + SQLFragment f = new SQLFragment("ALTER INDEX "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(oldName); + f.append(" RENAME TO ").appendIdentifier(newName); + statements.add(f); + } + } + + return statements; + } + + private SQLFragment getDropColumnsStatement(TableChange change) + { + List sqlParts = new ArrayList<>(); + for (PropertyStorageSpec prop : change.getColumns()) + { + SQLFragment sql = new SQLFragment("DROP COLUMN "); + if (prop.getExactName()) + { + sql.append(quoteIdentifier(prop.getName())); + } + else + { + sql.appendIdentifier(makePropertyIdentifier(prop.getName())); + } + sqlParts.add(sql); + } + + SQLFragment f = new SQLFragment("ALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" ").append(sqlParts, ", "); + return f; + } + + // TODO if there are cases where user-defined columns need indices, this method will need to support + // creating indices like getCreateTableStatement does. + private List getAddColumnsStatements(TableChange change) + { + List statements = new ArrayList<>(); + String pkColumn = null; + Constraint constraint = null; + + List columnSpecs = new ArrayList<>(); + for (PropertyStorageSpec prop : change.getColumns()) + { + columnSpecs.add(getSqlColumnSpec(prop)); + if (prop.isPrimaryKey()) + { + assert null == pkColumn : "no more than one primary key defined"; + pkColumn = prop.getName(); + constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); + } + } + + SQLFragment alter = new SQLFragment("ALTER TABLE "); + alter.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + alter.append(" "); + String sep = ""; + for (SQLFragment col : columnSpecs) + { + alter.append(sep); + alter.append("ADD COLUMN "); + alter.append(col); + sep = ", "; + } + statements.add(alter); + if (null != pkColumn) + { + SQLFragment addPk = new SQLFragment("ALTER TABLE "); + addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) + .append(" ").append(constraint.getType().toString()).append(" (") + .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); + statements.add(addPk); + } + + return statements; + } + + private List getDropConstraintsStatement(TableChange change) + { + return change.getConstraints().stream().map(constraint -> { + SQLFragment f = new SQLFragment("ALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" DROP CONSTRAINT ").appendIdentifier(constraint.getName()); + return f; + }).collect(Collectors.toList()); + } + + private List getAddConstraintsStatement(TableChange change) + { + List statements = new ArrayList<>(); + Collection constraints = change.getConstraints(); + + if (null!=constraints && !constraints.isEmpty()) + { + statements = constraints.stream().map(constraint -> { + List columns = new ArrayList<>(); + for (String col : constraint.getColumns()) + { + columns.add(new SQLFragment().appendIdentifier(col)); + } + + SQLFragment f = new SQLFragment(); + f.append("DO $$\nBEGIN\nIF NOT EXISTS\n(SELECT 1 FROM information_schema.constraint_column_usage\nWHERE table_name = ") + .appendStringLiteral(change.getSchemaName() + "." + change.getTableName(), this) + .append(" and constraint_name = ") + .appendStringLiteral(constraint.getName(), this) + .append(") THEN\nALTER TABLE "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()).append(" ") + .append(constraint.getType().toString()).append(" (") + .append(columns, ",") + .append(")").appendEOS().append("\nEND IF)").appendEOS().append("\nEND$$").appendEOS(); + return f; + }).collect(Collectors.toList()); + } + + return statements; + } + + private List getCreateTableStatements(TableChange change) + { + List statements = new ArrayList<>(); + List createTableSqlParts = new ArrayList<>(); + String pkColumn = null; + for (PropertyStorageSpec prop : change.getColumns()) + { + createTableSqlParts.add(getSqlColumnSpec(prop)); + if (prop.isPrimaryKey()) + { + assert null == pkColumn : "no more than one primary key defined"; + pkColumn = prop.getName(); + } + } + + for (PropertyStorageSpec.ForeignKey foreignKey : change.getForeignKeys()) + { + DbSchema schema = DbSchema.get(foreignKey.getSchemaName(), DbSchemaType.Module); + TableInfo tableInfo = foreignKey.isProvisioned() ? + foreignKey.getTableInfoProvisioned() : + schema.getTable(foreignKey.getTableName()); + String constraintName = "fk_" + foreignKey.getColumnName() + "_" + change.getTableName() + "_" + tableInfo.getName(); + SQLFragment fkFrag = new SQLFragment("CONSTRAINT "); + fkFrag.appendIdentifier(constraintName) + .append(" FOREIGN KEY (") + .appendIdentifier(makePropertyIdentifier(foreignKey.getColumnName())) + .append(") REFERENCES ") + .appendIdentifier(tableInfo.getSchema().getName()).append(".").appendIdentifier(tableInfo.getName()) + .append(" (") + .appendIdentifier(makePropertyIdentifier(foreignKey.getForeignColumnName())) + .append(")"); + createTableSqlParts.add(fkFrag); + } + + SQLFragment create = new SQLFragment("CREATE TABLE "); + create.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + create.append(" (").append(createTableSqlParts, ", ").append(")"); + statements.add(create); + if (null != pkColumn) + { + // Making this just for consistent naming + Constraint constraint = new Constraint(change.getTableName(), Constraint.CONSTRAINT_TYPES.PRIMARYKEY, false, null); + + SQLFragment addPk = new SQLFragment("ALTER TABLE "); + addPk.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + addPk.append(" ADD CONSTRAINT ").appendIdentifier(constraint.getName()) + .append(" ").append(constraint.getType().toString()).append(" (") + .appendIdentifier(makePropertyIdentifier(pkColumn)).append(")"); + statements.add(addPk); + } + + statements.addAll(getCreateIndexStatements(change)); + statements.addAll(getAddConstraintsStatement(change)); + return statements; + } + + private List getCreateIndexStatements(TableChange change) + { + List statements = new ArrayList<>(); + for (PropertyStorageSpec.Index index : change.getIndexedColumns()) + { + String newIndexName = nameIndex(change.getTableName(), index.columnNames); + SQLFragment f = new SQLFragment("CREATE "); + if (index.isUnique) + f.append("UNIQUE "); + f.append("INDEX ").appendIdentifier(newIndexName).append(" ON "); + f.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + f.append(" ("); + String separator = ""; + for (String columnName : index.columnNames) + { + f.append(separator).appendIdentifier(makePropertyIdentifier(columnName)); + separator = ", "; + } + f.append(")"); + f.appendEOS(); + statements.add(f); + + if (index.isClustered) + { + SQLFragment c = new SQLFragment(); + c.append(PropertyStorageSpec.CLUSTER_TYPE.CLUSTER.toString()).append(" "); + c.appendIdentifier(change.getSchemaName()).append(".").appendIdentifier(change.getTableName()); + c.append(" USING ").appendIdentifier(newIndexName); + statements.add(c); + } + } + return statements; + } + + @Override + public String nameIndex(String tableName, String[] indexedColumns) + { + return AliasManager.makeLegalName(tableName + '_' + StringUtils.join(indexedColumns, "_"), this); + } + + private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop) + { + return getSqlColumnSpec(prop, prop.getName()); + } + + private SQLFragment getSqlColumnSpec(PropertyStorageSpec prop, String columnName) + { + SQLFragment colSpec = new SQLFragment(); + colSpec.appendIdentifier(makePropertyIdentifier(columnName)).append(" "); + colSpec.append(getSqlTypeName(prop)); + + // Apply size and precision to varchar and Decimal types + if (prop.getJdbcType() == JdbcType.VARCHAR && prop.getSize() != -1 && prop.getSize() <= SqlDialect.MAX_VARCHAR_SIZE) + { + colSpec.append("(").append(prop.getSize().toString()).append(")"); + } + else if (prop.getJdbcType() == JdbcType.DECIMAL) + { + colSpec.append(DEFAULT_DECIMAL_SCALE_PRECISION); + } + + if (prop.isPrimaryKey() || !prop.isNullable()) + colSpec.append(" NOT NULL"); + + if (null != prop.getDefaultValue()) + { + if (prop.getJdbcType() == JdbcType.BOOLEAN) + { + colSpec.append(" DEFAULT "); + colSpec.append((Boolean)prop.getDefaultValue() ? getBooleanTRUE() : getBooleanFALSE()); + } + else if (prop.getJdbcType() == JdbcType.VARCHAR) + { + colSpec.append(" DEFAULT "); + colSpec.append(getStringHandler().quoteStringLiteral(prop.getDefaultValue().toString())); + } + else + { + throw new IllegalArgumentException("Default value on type " + prop.getJdbcType().name() + " is not supported."); + } + } + return colSpec; + } + + @Override + public void purgeTempSchema(Map createdTableNames) + { + try + { + trackTempTables(createdTableNames); + } + catch (SQLException e) + { + LOG.warn("error cleaning up temp schema", e); + } + + DbSchema coreSchema = CoreSchema.getInstance().getSchema(); + SqlExecutor executor = new SqlExecutor(coreSchema); + + //rs = conn.getMetaData().getFunctions(dbName, tempSchemaName, "%"); + + new SqlSelector(coreSchema, "SELECT proname AS SPECIFIC_NAME, CAST(proargtypes AS VARCHAR) FROM pg_proc WHERE pronamespace=(select oid from pg_namespace where nspname = ?)", DbSchema.getTemp().getName()).forEach( + new ForEachBlock<>() + { + private Map _types = null; + + @Override + public void exec(ResultSet rs) throws SQLException + { + if (null == _types) + { + _types = new HashMap<>(); + new SqlSelector(coreSchema, "SELECT CAST(oid AS VARCHAR) as oid, typname, (select nspname from pg_namespace where oid = typnamespace) as nspname FROM pg_type").forEach(type -> + _types.put(type.getString(1), quoteIdentifier(type.getString(3)) + "." + quoteIdentifier(type.getString(2)))); + } + + String name = rs.getString(1); + String[] oids = StringUtils.split(rs.getString(2), ' '); + SQLFragment drop = new SQLFragment("DROP FUNCTION temp.").append(name); + drop.append("("); + String comma = ""; + for (String oid : oids) + { + drop.append(comma).append(_types.get(oid)); + comma = ","; + } + drop.append(")"); + + try + { + executor.execute(drop); + } + catch (BadSqlGrammarException x) + { + LOG.warn("could not clean up postgres function : temp." + name, x); + } + } + }); + + // TODO delete types in temp schema as well! search for "CREATE TYPE" in StatementUtils.java + } + + // + // ARRAY and SET syntax + // + + // NOTE LabKey currently does not support ARRAY[VARCHAR], use ARRAY[text] instead + // + // Postgres string literals can be auto-cast to both VARCHAR and TEXT. These all work + // 'color' = 'color'::varchar + // 'color' = 'color'::text + // ARRAY['color'] = ARRAY['color'::text]; + // However, ARRAY[text] cannot be auto cast to ARRAY[varchar] + // ARRAY['color'] = ARRAY['color'::varchar]; -- ERROR! + // + + + @Override + public boolean supportsArrays() + { + return true; + } + + @Override + public SQLFragment array_construct(SQLFragment[] elements) + { + SQLFragment ret = new SQLFragment(); + ret.append("ARRAY["); + String separator = ""; + for (SQLFragment element : elements) + { + ret.append(separator); + ret.append(element); + separator = ", "; + } + ret.append("]"); + return ret; + } + + @Override + public SQLFragment array_all_in_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append("(").append(a).append(") <@ (").append(b).append(")"); + return ret; + } + + @Override + public SQLFragment array_some_in_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append("(").append(a).append(") && (").append(b).append(")"); + return ret; + } + + @Override + public SQLFragment array_none_in_array(SQLFragment a, SQLFragment b) + { + return new SQLFragment(" NOT (").append(array_some_in_array(a, b)).append(")"); + } + + @Override + public SQLFragment array_same_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append(array_all_in_array(a, b)).append(" AND ").append(array_all_in_array(b, a)); + return ret; + } + + @Override + public SQLFragment array_not_same_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append("NOT (").append(array_all_in_array(a, b)).append(") OR NOT (").append(array_all_in_array(b, a)).append(")"); + return ret; + } + + @Override + public SQLFragment element_in_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append("(").append(a).append(")"); + // DOCs imply that IS NOT DISTINCT FROM ANY should work, but it doesn't??? + // ret.append(" IS NOT DISTINCT FROM ANY("); + ret.append(" = ANY("); + ret.append(b); + ret.append(")"); + return ret; + } + + @Override + public SQLFragment element_not_in_array(SQLFragment a, SQLFragment b) + { + SQLFragment ret = new SQLFragment(); + ret.append("(").append(a).append(")"); + // DOCs imply that IS NOT DISTINCT FROM ANY should work, but it doesn't??? + // ret.append(" IS DISTINCT FROM ALL("); + ret.append(" <> ALL("); + ret.append(b); + ret.append(")"); + return ret; + } +} diff --git a/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java b/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java index 40e036aff70..1715b088f72 100644 --- a/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java +++ b/experiment/src/org/labkey/experiment/api/property/DomainPropertyImpl.java @@ -1,1398 +1,1398 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api.property; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.data.BooleanFormat; -import org.labkey.api.data.ColumnRenderPropertiesImpl; -import org.labkey.api.data.ConditionalFormat; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DatabaseIdentifier; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.PHI; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.exp.ChangePropertyDescriptorException; -import org.labkey.api.exp.DomainDescriptor; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.IPropertyType; -import org.labkey.api.exp.property.IPropertyValidator; -import org.labkey.api.exp.property.Lookup; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.gwt.client.DefaultScaleType; -import org.labkey.api.gwt.client.DefaultValueType; -import org.labkey.api.gwt.client.FacetingBehaviorType; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.security.User; -import org.labkey.api.util.StringExpressionFactory; -import org.labkey.api.util.TestContext; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -public class DomainPropertyImpl implements DomainProperty -{ - private final DomainImpl _domain; - - PropertyDescriptor _pd; - PropertyDescriptor _pdOld; - boolean _deleted; - - private boolean _schemaChanged; - private boolean _schemaImport; - private List _validators; - private List _formats; - private String _defaultValue; - - public DomainPropertyImpl(DomainImpl type, PropertyDescriptor pd) - { - this(type, pd, null); - } - - public DomainPropertyImpl(DomainImpl type, PropertyDescriptor pd, List formats) - { - _domain = type; - _pd = pd.clone(); - _formats = formats; - } - - @Override - public int getPropertyId() - { - return _pd.getPropertyId(); - } - - @Override - public Container getContainer() - { - return _pd.getContainer(); - } - - @Override - public String getPropertyURI() - { - return _pd.getPropertyURI(); - } - - @Override - public String getName() - { - return _pd.getName(); - } - - @Override - public String getDescription() - { - return _pd.getDescription(); - } - - @Override - public String getFormat() - { - return _pd.getFormat(); - } - - @Override - public String getLabel() - { - return _pd.getLabel(); - } - - @Override - public String getConceptURI() - { - return _pd.getConceptURI(); - } - - @Override - public Domain getDomain() - { - return _domain; - } - - @Override - public IPropertyType getType() - { - return PropertyService.get().getType(getContainer(), _pd.getRangeURI()); - } - - @Override - public boolean isRequired() - { - return _pd.isRequired(); - } - - @Override - public boolean isHidden() - { - return _pd.isHidden(); - } - - @Override - public boolean isDeleted() - { - return _deleted; - } - - @Override - public boolean isShownInInsertView() - { - return _pd.isShownInInsertView(); - } - - @Override - public boolean isShownInDetailsView() - { - return _pd.isShownInDetailsView(); - } - - @Override - public boolean isShownInUpdateView() - { - return _pd.isShownInUpdateView(); - } - - @Override - public boolean isShownInLookupView() - { - return _pd.isShownInLookupView(); - } - - @Override - public boolean isMeasure() - { - return _pd.isMeasure(); - } - - @Override - public boolean isDimension() - { - return _pd.isDimension(); - } - - @Override - public boolean isRecommendedVariable() - { - return _pd.isRecommendedVariable(); - } - - @Override - public DefaultScaleType getDefaultScale() - { - return _pd.getDefaultScale(); - } - - @Override - public PHI getPHI() - { - return _pd.getPHI(); - } - - @Override - public String getRedactedText() { return _pd.getRedactedText(); } - - @Override - public boolean isExcludeFromShifting() - { - return _pd.isExcludeFromShifting(); - } - - @Override - public boolean isMvEnabled() - { - return _pd.isMvEnabled(); - } - - @Override - public boolean isMvEnabledForDrop() - { - if (null != _pdOld) - return _pdOld.isMvEnabled(); // if we need to drop/recreate we care about the old one - return _pd.isMvEnabled(); - } - - @Override - public void delete() - { - _deleted = true; - } - - @Override - public void setSchemaImport(boolean isSchemaImport) - { - // if this flag is set True then the column is dropped and recreated by its Domain if there is a type change - _schemaImport = isSchemaImport; - } - - @Override - public void setName(String name) - { - if (Strings.CS.equals(name, getName())) - return; - edit().setName(name); - } - - @Override - public void setDescription(String description) - { - if (Strings.CS.equals(description, getDescription())) - return; - edit().setDescription(description); - } - - @Override - public void setType(IPropertyType domain) - { - edit().setRangeURI(domain.getTypeURI()); - } - - @Override - public void setPropertyURI(String uri) - { - if (Strings.CS.equals(uri, getPropertyURI())) - return; - edit().setPropertyURI(uri); - } - - @Override - public void setRangeURI(String rangeURI) - { - if (Strings.CS.equals(rangeURI, getRangeURI())) - return; - editSchema().setRangeURI(rangeURI); - } - - @Override - public String getRangeURI() - { - return _pd.getRangeURI(); - } - - @Override - public void setFormat(String s) - { - if (Strings.CS.equals(s, getFormat())) - return; - edit().setFormat(s); - } - - @Override - public void setLabel(String caption) - { - if (Strings.CS.equals(caption, getLabel())) - return; - edit().setLabel(caption); - } - - @Override - public void setConceptURI(String conceptURI) - { - if (Strings.CS.equals(conceptURI, getConceptURI())) - return; - edit().setConceptURI(conceptURI); - } - - @Override - public void setRequired(boolean required) - { - if (required == isRequired()) - return; - edit().setRequired(required); - } - - @Override - public void setHidden(boolean hidden) - { - if (hidden == isHidden()) - return; - edit().setHidden(hidden); - } - - @Override - public void setShownInDetailsView(boolean shown) - { - if (shown == isShownInDetailsView()) - return; - edit().setShownInDetailsView(shown); - } - - @Override - public void setShownInInsertView(boolean shown) - { - if (shown == isShownInInsertView()) - return; - edit().setShownInInsertView(shown); - } - - @Override - public void setShownInUpdateView(boolean shown) - { - if (shown == isShownInUpdateView()) - return; - edit().setShownInUpdateView(shown); - } - - @Override - public void setShownInLookupView(boolean shown) - { - if (shown == isShownInLookupView()) - return; - edit().setShownInLookupView(shown); - } - - @Override - public void setMeasure(boolean isMeasure) - { - // UNDONE: isMeasure() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. - if (!isEdited() && isMeasure == isMeasure()) - return; - edit().setMeasure(isMeasure); - } - - @Override - public void setDimension(boolean isDimension) - { - // UNDONE: isDimension() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. - if (!isEdited() && isDimension == isDimension()) - return; - edit().setDimension(isDimension); - } - - @Override - public void setRecommendedVariable(boolean isRecommendedVariable) - { - if (!isEdited() && isRecommendedVariable == isRecommendedVariable()) - return; - edit().setRecommendedVariable(isRecommendedVariable); - } - - @Override - public void setDefaultScale(DefaultScaleType defaultScale) - { - if (!isEdited() && getDefaultScale() == defaultScale) - return; - - edit().setDefaultScale(defaultScale); - } - - @Override - public void setPhi(PHI phi) - { - if (!isEdited() && getPHI() == phi) - return; - edit().setPHI(phi); - } - - @Override - public void setRedactedText(String redactedText) - { - if (!isEdited() && ((getRedactedText() != null && getRedactedText().equals(redactedText)) - || (getRedactedText() == null && redactedText == null))) - return; - edit().setRedactedText(redactedText); - } - - @Override - public void setExcludeFromShifting(boolean isExcludeFromShifting) - { - // UNDONE: isExcludeFromShifting() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. - if (!isEdited() && isExcludeFromShifting == isExcludeFromShifting()) - return; - edit().setExcludeFromShifting(isExcludeFromShifting); - } - - @Override - public void setMvEnabled(boolean mv) - { - if (mv == isMvEnabled()) - return; - edit().setMvEnabled(mv); - } - - @Override - public void setScale(int scale) - { - if (scale == getScale()) - return; - edit().setScale(scale); - } - - /** Need the string version of this method because it's called by reflection and must match by name */ - public void setImportAliases(String aliases) - { - if (Strings.CS.equals(aliases, getImportAliases())) - return; - edit().setImportAliases(aliases); - } - - /** Need the string version of this method because it's called by reflection and must match by name */ - public String getImportAliases() - { - return _pd.getImportAliases(); - } - - @Override - public void setImportAliasSet(Set aliases) - { - String current = getImportAliases(); - String newAliases = ColumnRenderPropertiesImpl.convertToString(aliases); - if (Strings.CS.equals(current, newAliases)) - return; - edit().setImportAliasesSet(aliases); - } - - @Override - public Set getImportAliasSet() - { - return _pd.getImportAliasSet(); - } - - @Override - public void setURL(String url) - { - if (Strings.CS.equals(getURL(), url)) - return; - - if (null == url) - edit().setURL(null); - else - edit().setURL(StringExpressionFactory.createURL(url)); - } - - @Override - public String getURL() - { - return _pd.getURL() == null ? null : _pd.getURL().toString(); - } - - @Override - public void setURLTarget(String urlTarget) - { - if (Strings.CS.equals(getURLTarget(), urlTarget)) - return; - edit().setURLTarget(urlTarget); - } - - @Override - public String getURLTarget() - { - return _pd.getURLTarget(); - } - - private boolean isEdited() - { - return null != _pdOld; - } - - private PropertyDescriptor editSchema() - { - PropertyDescriptor pd = edit(); - _schemaChanged = true; - _pd.clearPropertyType(); - return pd; - } - - public boolean isRecreateRequired() - { - return _schemaChanged && _schemaImport; - } - - public void markAsNew() - { - assert isRecreateRequired() && !isNew(); - _pd.setPropertyId(0); - } - - private PropertyDescriptor edit() - { - if (_pdOld == null) - { - _pdOld = _pd; - _pd = _pdOld.clone(); - } - return _pd; - } - - @Override - public PropertyType getPropertyType() - { - return _pd.getPropertyType(); - } - - @Override - public JdbcType getJdbcType() - { - return _pd.getPropertyType().getJdbcType(); - } - - @Override - public int getScale() - { - return _pd.getScale(); - } - - @Override - public String getInputType() - { - return _pd.getPropertyType().getInputType(); - } - - @Override - public DefaultValueType getDefaultValueTypeEnum() - { - return _pd.getDefaultValueTypeEnum(); - } - - @Override - public void setDefaultValueTypeEnum(DefaultValueType defaultValueType) - { - _pd.setDefaultValueTypeEnum(defaultValueType); - } - - public String getDefaultValueType() - { - return _pd.getDefaultValueType(); - } - - @Override - public void setDefaultValueType(String defaultValueTypeName) - { - if (getDefaultValueType() != null && getDefaultValueType().equals(defaultValueTypeName)) - return; - - if (getDefaultValueType() == null && defaultValueTypeName == null) - return; // if both are null, don't call edit(), with marks property as dirty - - edit().setDefaultValueType(defaultValueTypeName); - } - - @Override - public void setDefaultValue(String value) - { - _defaultValue = value; - } - - public String getDefaultValue() - { - return _defaultValue; - } - - @Override - public Lookup getLookup() - { - return _pd.getLookup(); - } - - @Override - public void setLookup(Lookup lookup) - { - Lookup current = getLookup(); - - if (current == lookup) - return; - - // current will return null if the schema or query is null so check - // for this case in the passed in lookup - if (current == null) - if (lookup.getQueryName() == null || lookup.getSchemaKey() == null) - return; - - if (current != null && current.equals(lookup)) - return; - - if (lookup == null) - { - edit().setLookupContainer(null); - edit().setLookupSchema(null); - edit().setLookupQuery(null); - return; - } - if (lookup.getContainer() == null) - { - edit().setLookupContainer(null); - } - else - { - edit().setLookupContainer(lookup.getContainer().getId()); - } - edit().setLookupQuery(lookup.getQueryName()); - edit().setLookupSchema(Objects.toString(lookup.getSchemaKey(),null)); - } - - @Override - public void setScannable(boolean scannable) - { - if (scannable != isScannable()) - edit().setScannable(scannable); - } - - @Override - public void setOldPropertyDescriptor(PropertyDescriptor oldPropertyDescriptor) - { - if (isEdited()) - return; - - _pdOld = oldPropertyDescriptor.clone(); - } - - @Override - public boolean isScannable() - { - return _pd.isScannable(); - } - - @Override - public void setPrincipalConceptCode(String code) - { - if (!Strings.CS.equals(code, getPrincipalConceptCode())) - edit().setPrincipalConceptCode(code); - } - - @Override - public String getPrincipalConceptCode() - { - return _pd.getPrincipalConceptCode(); - } - - @Override - public String getSourceOntology() - { - return _pd.getSourceOntology(); - } - - @Override - public void setSourceOntology(String sourceOntology) - { - if (!Strings.CS.equals(sourceOntology, getSourceOntology())) - edit().setSourceOntology(sourceOntology); - } - - @Override - public String getConceptSubtree() - { - return _pd.getConceptSubtree(); - } - - @Override - public void setConceptSubtree(String path) - { - if (!Strings.CS.equals(path, getConceptSubtree())) - edit().setConceptSubtree(path); - } - - @Override - public String getConceptImportColumn() - { - return _pd.getConceptImportColumn(); - } - - @Override - public void setConceptImportColumn(String conceptImportColumn) - { - if (!Strings.CS.equals(conceptImportColumn, getConceptImportColumn())) - edit().setConceptImportColumn(conceptImportColumn); - } - - @Override - public String getConceptLabelColumn() - { - return _pd.getConceptLabelColumn(); - } - - @Override - public void setConceptLabelColumn(String conceptLabelColumn) - { - if (!Strings.CS.equals(conceptLabelColumn, getConceptLabelColumn())) - edit().setConceptLabelColumn(conceptLabelColumn); - } - - @Override - public void setDerivationDataScope(String scope) - { - if (!Strings.CS.equals(scope, getDerivationDataScope())) - edit().setDerivationDataScope(scope); - } - - @Override - public String getDerivationDataScope() - { - return _pd.getDerivationDataScope(); - } - - @Override - public PropertyDescriptor getPropertyDescriptor() - { - return _pd; - } - - @Override - public List getConditionalFormats() - { - return ensureConditionalFormats(); - } - - public boolean isNew() - { - return _pd.getPropertyId() == 0; - } - - // Scenario to swap property descriptors on study upload to or from a system property, instead of updating the - // current property descriptor. Avoids overwriting a system property. - public boolean isSystemPropertySwap() - { - if (_pd.getPropertyId() == 0 && _pd.getPropertyURI() != null && _pdOld != null && _pdOld.getPropertyURI() != null - && !_pd.getPropertyURI().equals(_pdOld.getPropertyURI())) - { - return SystemProperty.getProperties().stream().anyMatch(sp -> - sp.getPropertyURI().equals(_pd.getPropertyURI()) || sp.getPropertyURI().equals(_pdOld.getPropertyURI())); - } - - return false; - } - - public boolean isDirty() - { - if (_pdOld != null) return true; - - for (PropertyValidatorImpl v : ensureValidators()) - { - if (v.isDirty() || v.isNew()) - return true; - } - return false; - } - - public void delete(User user) - { - DomainPropertyManager.get().removeValidatorsForPropertyDescriptor(getContainer(), getPropertyId()); - DomainPropertyManager.get().deleteConditionalFormats(getPropertyId()); - - DomainKind kind = getDomain().getDomainKind(); - if (null != kind) - kind.deletePropertyDescriptor(getDomain(), user, _pd); - OntologyManager.removePropertyDescriptorFromDomain(this); - } - - public void save(User user, DomainDescriptor dd, int sortOrder) throws ChangePropertyDescriptorException - { - if (isSystemPropertySwap()) - { - _pd = OntologyManager.insertOrUpdatePropertyDescriptor(_pd, dd, sortOrder); - OntologyManager.removePropertyDescriptorFromDomain(new DomainPropertyImpl((DomainImpl) getDomain(), _pdOld)); - } - else if (isNew()) - { - _pd = OntologyManager.insertOrUpdatePropertyDescriptor(_pd, dd, sortOrder); - } - else if (_pdOld != null) - { - PropertyType oldType = _pdOld.getPropertyType(); - PropertyType newType = _pd.getPropertyType(); - boolean changedType = false; - if (oldType.getJdbcType() != newType.getJdbcType()) - { - if (newType.getJdbcType().isText() || - (oldType.getJdbcType().isInteger() && newType.getJdbcType().isNumeric())) - { - changedType = true; - if (newType.getJdbcType().isText()) - { - // Remove any previously set formatting string as it won't apply to a text field - _pd.setFormat(null); - } - } - else if (newType.getJdbcType().isDateOrTime() && oldType.getJdbcType().isDateOrTime()) - { - changedType = true; - _pd.setFormat(null); - } - else if (newType == PropertyType.MULTI_CHOICE || oldType == PropertyType.MULTI_CHOICE) - { - changedType = true; - _pd.setFormat(null); - } - else - { - throw new ChangePropertyDescriptorException("Cannot convert an instance of " + oldType.getJdbcType() + " to " + newType.getJdbcType() + "."); - } - } - - // Issue 44711: Prevent attachment and file field types from being converted to a different type - if (PropertyType.FILE_LINK.getInputType().equalsIgnoreCase(oldType.getInputType()) && oldType != newType) - throw new ChangePropertyDescriptorException("Cannot convert an instance of " + oldType.name() + " to " + newType.name() + "."); - - OntologyManager.validatePropertyDescriptor(_pd); - Table.update(user, OntologyManager.getTinfoPropertyDescriptor(), _pd, _pdOld.getPropertyId()); - OntologyManager.ensurePropertyDomain(_pd, dd, sortOrder); - - boolean hasProvisioner = null != getDomain().getDomainKind() && null != getDomain().getDomainKind().getStorageSchemaName() && dd.getStorageTableName() != null; - SqlDialect dialect = OntologyManager.getExpSchema().getSqlDialect(); - - if (hasProvisioner) - { - boolean mvAdded = !_pdOld.isMvEnabled() && _pd.isMvEnabled(); - boolean mvDropped = _pdOld.isMvEnabled() && !_pd.isMvEnabled(); - boolean propRenamed = !_pdOld.getName().equals(_pd.getName()); - boolean propResized = _pd.isStringType() && _pdOld.getScale() != _pd.getScale(); - - // Drop first, so rename doesn't have to worry about it - if (mvDropped) - ((StorageProvisionerImpl)StorageProvisioner.get()).dropMvIndicator(this, _pdOld); - - if (propRenamed) - StorageProvisionerImpl.get().renameProperty(this.getDomain(), this, _pdOld, mvDropped); - - if (changedType) - { - var domainKind = _domain.getDomainKind(); - if (domainKind == null) - throw new ChangePropertyDescriptorException("Cannot change property type for domain, unknown domain kind."); - - StorageProvisionerImpl.get().changePropertyType(this.getDomain(), this); - if (_pdOld.getJdbcType() == JdbcType.BOOLEAN && _pd.getJdbcType().isText()) - { - updateBooleanValue( - new SQLFragment().appendIdentifier(domainKind.getStorageSchemaName()).append(".").appendIdentifier(_domain.getStorageTableName()), - _pd.getLegalSelectName(dialect), _pdOld.getFormat(), null); // GitHub Issue #647 - } - - TableInfo table = domainKind.getTableInfo(user, getContainer(), _domain, ContainerFilter.getUnsafeEverythingFilter()); - if (table != null && _pdOld.getPropertyType() != null) - QueryChangeListener.QueryPropertyChange.handleColumnTypeChange(_pdOld, _pd, SchemaKey.fromString(table.getUserSchema().getName()), table.getName(), user, getContainer()); - } - else if (propResized) - StorageProvisionerImpl.get().resizeProperty(this.getDomain(), this, _pdOld.getScale()); - - if (mvAdded) - StorageProvisionerImpl.get().addMvIndicator(this); - } - else if (changedType) - { - if (oldType.getJdbcType().isDateOrTime() && newType.getJdbcType().isText()) - { - new SqlExecutor(OntologyManager.getExpSchema()).execute( - new SQLFragment("UPDATE "). - append(OntologyManager.getTinfoObjectProperty()). - append(" SET StringValue = DateTimeValue, DateTimeValue = NULL WHERE PropertyId = ?"). - add(_pdOld.getPropertyId())); - } - else if (!oldType.getJdbcType().isText() && newType.getJdbcType().isText()) - { - new SqlExecutor(OntologyManager.getExpSchema()).execute( - new SQLFragment("UPDATE "). - append(OntologyManager.getTinfoObjectProperty()). - append(" SET StringValue = FloatValue, FloatValue = NULL WHERE PropertyId = ?"). - add(_pdOld.getPropertyId())); - } - else if (oldType.getJdbcType().isDateOrTime() && newType.getJdbcType().isDateOrTime()) - { - String sqlTypeName = dialect.getSqlTypeName(newType.getJdbcType()); - String update = String.format("CAST(DateTimeValue AS %s)", sqlTypeName); - if (newType.getJdbcType() == JdbcType.TIME) - update = dialect.getDateTimeToTimeCast("DateTimeValue"); - SQLFragment sqlFragment = new SQLFragment("UPDATE ") - .append(OntologyManager.getTinfoObjectProperty()) - .append(" SET DateTimeValue = ") - .append(update) - .append(" WHERE PropertyId = ?") - .add(_pdOld.getPropertyId()); - new SqlExecutor(OntologyManager.getExpSchema()).execute(sqlFragment); - } - else //noinspection StatementWithEmptyBody - if (oldType.getJdbcType().isInteger() && newType.getJdbcType().isReal()) - { - // Since exp.ObjectProperty stores these types in the same column, there's nothing for us to do - } - else - { - throw new ChangePropertyDescriptorException("Cannot convert from " + oldType.getJdbcType() + " to " + newType.getJdbcType() + " for non-provisioned table"); - } - } - - if (changedType && _pdOld.getJdbcType() == JdbcType.BOOLEAN && _pd.getJdbcType().isText()) - { - updateBooleanValue(OntologyManager.getTinfoObjectProperty().getSQLName(), dialect.makeDatabaseIdentifier("StringValue"), _pdOld.getFormat(), new SQLFragment("PropertyId = ?", _pdOld.getPropertyId())); - } - } - else - { - OntologyManager.ensurePropertyDomain(_pd, _domain._dd, sortOrder); - } - - _pdOld = null; - _schemaChanged = false; - _schemaImport = false; - - for (PropertyValidatorImpl validator : ensureValidators()) - { - if (validator.isDeleted()) - DomainPropertyManager.get().removePropertyValidator(this, validator); - else - DomainPropertyManager.get().savePropertyValidator(user, this, validator); - } - - DomainPropertyManager.get().saveConditionalFormats(user, getPropertyDescriptor(), ensureConditionalFormats()); - } - - /** - * Format values in columns that were just converted from booleans to strings with the DB's default type conversion. - * Postgres will now have 'true' and 'false', and SQLServer will have '0' and '1'. Use the format string to use the - * preferred format, and standardize on 'true' and 'false' in the absence of an explicitly configured format. - */ - private void updateBooleanValue(SQLFragment schemaTable, DatabaseIdentifier column, String formatString, @Nullable SQLFragment whereClause) - { - BooleanFormat f = BooleanFormat.getInstance(formatString); - String trueValue = StringUtils.trimToNull(f.format(true)); - String falseValue = StringUtils.trimToNull(f.format(false)); - String nullValue = StringUtils.trimToNull(f.format(null)); - SQLFragment sql = new SQLFragment("UPDATE ").append(schemaTable).append(" SET "). - appendIdentifier(column).append(" = CASE WHEN "). - appendIdentifier(column).append(" IN ('1', 'true') THEN ? WHEN "). - appendIdentifier(column).append(" IN ('0', 'false') THEN ? ELSE ? END"); - sql.add(trueValue); - sql.add(falseValue); - sql.add(nullValue); - if (whereClause != null) - { - sql.append(" WHERE "); - sql.append(whereClause); - } - new SqlExecutor(OntologyManager.getExpSchema()).execute(sql); - } - - @Override - @NotNull - public List getValidators() - { - return Collections.unmodifiableList(ensureValidators()); - } - - @Override - public void addValidator(IPropertyValidator validator) - { - if (validator != null) - { - if (0 != validator.getPropertyId() && getPropertyId() != validator.getPropertyId()) - throw new IllegalStateException(); - - // Ensure validator is a valid kind (ex. urn:lsid:labkey.com:PropertyValidator:length is no longer valid) - if ( null != PropertyService.get().getValidatorKind(validator.getTypeURI()) ) - { - PropertyValidator impl = new PropertyValidator(); - impl.copy(validator); - impl.setPropertyId(getPropertyId()); - ensureValidators().add(new PropertyValidatorImpl(impl)); - } - } - } - - @Override - public void removeValidator(IPropertyValidator validator) - { - int idx = ensureValidators().indexOf(validator); - if (idx != -1) - { - PropertyValidatorImpl impl = ensureValidators().get(idx); - impl.delete(); - } - } - - @Override - public void removeValidator(long validatorId) - { - if (validatorId == 0) return; - - for (PropertyValidatorImpl imp : ensureValidators()) - { - if (imp.getRowId() == validatorId) - { - imp.delete(); - break; - } - } - } - - @Override - public void copyFrom(DomainProperty propSrc, Container targetContainer) - { - setDescription(propSrc.getDescription()); - setFormat(propSrc.getFormat()); - setLabel(propSrc.getLabel()); - setName(propSrc.getName()); - setDescription(propSrc.getDescription()); - setConceptURI(propSrc.getConceptURI()); - setType(propSrc.getType()); - setDimension(propSrc.isDimension()); - setMeasure(propSrc.isMeasure()); - setRecommendedVariable(propSrc.isRecommendedVariable()); - setDefaultScale(propSrc.getDefaultScale()); - setRequired(propSrc.isRequired()); - setExcludeFromShifting(propSrc.isExcludeFromShifting()); - setFacetingBehavior(propSrc.getFacetingBehavior()); - setImportAliasSet(propSrc.getImportAliasSet()); - setPhi(propSrc.getPHI()); - setURL(propSrc.getURL()); - setURLTarget(propSrc.getURLTarget()); - setHidden(propSrc.isHidden()); - setShownInDetailsView(propSrc.isShownInDetailsView()); - setShownInInsertView(propSrc.isShownInInsertView()); - setShownInUpdateView(propSrc.isShownInUpdateView()); - setShownInLookupView(propSrc.isShownInLookupView()); - setMvEnabled(propSrc.isMvEnabled()); - setDefaultValueTypeEnum(propSrc.getDefaultValueTypeEnum()); - setScale(propSrc.getScale()); - setScannable(propSrc.isScannable()); - - setPrincipalConceptCode(propSrc.getPrincipalConceptCode()); - setSourceOntology(propSrc.getSourceOntology()); - setConceptSubtree(propSrc.getConceptSubtree()); - setConceptImportColumn(propSrc.getConceptImportColumn()); - setConceptLabelColumn(propSrc.getConceptLabelColumn()); - setDerivationDataScope(propSrc.getDerivationDataScope()); - - // check to see if we're moving a lookup column to another container: - Lookup lookup = propSrc.getLookup(); - if (lookup != null && !getContainer().equals(targetContainer)) - { - // we need to update the lookup properties if the lookup container is either the source or the destination container - if (lookup.getContainer() == null) - lookup.setContainer(propSrc.getContainer()); - else if (lookup.getContainer().equals(targetContainer)) - lookup.setContainer(null); - } - setLookup(lookup); - } - - @Override - public void setConditionalFormats(List formats) - { - String newVal = ConditionalFormat.toStringVal(formats); - String oldVal = ConditionalFormat.toStringVal(getConditionalFormats()); - - if (!Objects.equals(newVal, oldVal)) - edit(); - - _formats = formats; - } - - private List ensureValidators() - { - if (_validators == null) - { - _validators = new ArrayList<>(); - for (PropertyValidator validator : DomainPropertyManager.get().getValidators(this)) - { - _validators.add(new PropertyValidatorImpl(validator)); - } - } - return _validators; - } - - private List ensureConditionalFormats() - { - if (_formats == null) - { - _formats = new ArrayList<>(); - _formats.addAll(DomainPropertyManager.get().getConditionalFormats(this)); - } - return _formats; - } - - public PropertyDescriptor getOldProperty() - { - return _pdOld; - } - - @Override - public FacetingBehaviorType getFacetingBehavior() - { - return _pd.getFacetingBehaviorType(); - } - - @Override - public void setFacetingBehavior(FacetingBehaviorType type) - { - if (getFacetingBehavior() == type) - return; - - edit().setFacetingBehaviorType(type); - } - - @Override - public int hashCode() - { - return _pd.hashCode(); - } - - @Override - public boolean equals(Object obj) - { - if (obj == this) - return true; - if (!(obj instanceof DomainPropertyImpl)) - return false; - // once a domain property has been edited, it no longer equals any other domain property: - if (_pdOld != null || ((DomainPropertyImpl) obj)._pdOld != null) - return false; - return (_pd.equals(((DomainPropertyImpl) obj)._pd)); - } - - @Override - public String toString() - { - return super.toString() + _pd.getPropertyURI(); - } - - public Map getAuditRecordMap(@Nullable String validatorStr, @Nullable String conditionalFormatStr) - { - Map map = new LinkedHashMap<>(); - if (!StringUtils.isEmpty(getName())) - map.put("Name", getName()); - if (!StringUtils.isEmpty(getLabel())) - map.put("Label", getLabel()); - if (null != getPropertyType()) - { - if (org.labkey.api.gwt.client.ui.PropertyType.expFlag.getURI().equals(getConceptURI())) - map.put("Type", "Flag"); - else - map.put("Type", getPropertyType().getXarName()); - } - if (getPropertyType().getJdbcType().isText()) - map.put("Scale", getScale()); - if (!StringUtils.isEmpty(getDescription())) - map.put("Description", getDescription()); - if (!StringUtils.isEmpty(getFormat())) - map.put("Format", getFormat()); - if (!StringUtils.isEmpty(getURL())) - map.put("URL", getURL()); - if (!StringUtils.isEmpty(getURLTarget())) - map.put("URLTarget", getURLTarget()); - if (getPHI() != null) - map.put("PHI", getPHI().getLabel()); - if (getDefaultScale() != null) - map.put("DefaultScale", getDefaultScale().getLabel()); - map.put("Required", isRequired()); - map.put("Hidden", isHidden()); - map.put("MvEnabled", isMvEnabled()); - map.put("Measure", isMeasure()); - map.put("Dimension", isDimension()); - map.put("ShownInInsert", isShownInInsertView()); - map.put("ShownInDetails", isShownInDetailsView()); - map.put("ShownInUpdate", isShownInUpdateView()); - map.put("ShownInLookupView", isShownInLookupView()); - map.put("RecommendedVariable", isRecommendedVariable()); - map.put("ExcludedFromShifting", isExcludeFromShifting()); - map.put("Scannable", isScannable()); - if (!StringUtils.isEmpty(getDerivationDataScope())) - map.put("DerivationDataScope", getDerivationDataScope()); - String importAliasStr = StringUtils.join(getImportAliasSet(), ","); - if (!StringUtils.isEmpty(importAliasStr)) - map.put("ImportAliases", importAliasStr); - if (getDefaultValueTypeEnum() != null) - map.put("DefaultValueType", getDefaultValueTypeEnum().getLabel()); - if (getLookup() != null) - map.put("Lookup", getLookup().toJSONString()); - - if (!StringUtils.isEmpty(validatorStr)) - map.put("Validator", validatorStr); - if (!StringUtils.isEmpty(conditionalFormatStr)) - map.put("ConditionalFormat", conditionalFormatStr); - - return map; - } - - public static class TestCase extends Assert - { - private PropertyDescriptor _pd; - private DomainPropertyImpl _dp; - - @Test - public void testUpdateDomainPropertyFromDescriptor() - { - Container c = ContainerManager.ensureContainer("/_DomainPropertyImplTest", TestContext.get().getUser()); - String domainURI = new Lsid("Junit", "DD", "Domain1").toString(); - Domain d = PropertyService.get().createDomain(c, domainURI, "Domain1"); - - resetProperties(d, domainURI, c); - - // verify no change - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertFalse(_dp.isDirty()); - assertFalse(_dp._schemaChanged); - - // change a property - _pd.setPHI(PHI.Restricted); - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertTrue(_dp.isDirty()); - assertFalse(_dp._schemaChanged); - assertTrue(_dp.getPHI() == _pd.getPHI()); - - // Issue #18738 change the schema outside of a schema reload and verify that the column - // change the schema but don't mark the property as "Schema Import" - // this will allow whatever type changes the UI allows (text -> multiline, for example) - resetProperties(d, domainURI, c); - _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#double"); - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertTrue(_dp.isDirty()); - assertTrue(_dp._schemaChanged); - assertFalse(_dp.isRecreateRequired()); - assertTrue(Strings.CS.equals(_dp.getRangeURI(), _pd.getRangeURI())); - - // setting schema import to true will enable the _schemaChanged flag to toggle - // so it should be set true here - resetProperties(d, domainURI, c); - _dp.setSchemaImport(true); - _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#double"); - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertTrue(_dp.isDirty()); - assertTrue(_dp._schemaChanged); - assertTrue(_dp.isRecreateRequired()); - assertTrue(Strings.CS.equals(_dp.getRangeURI(), _pd.getRangeURI())); - - // verify no change when setting value to the same value as it was - resetProperties(d, domainURI, c); - _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#int"); - _pd.setPHI(PHI.NotPHI); - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertFalse(_dp.isDirty()); - assertFalse(_dp._schemaChanged); - assertFalse(_dp.isRecreateRequired()); - - // verify Lookup is set to null with null schema - resetProperties(d, domainURI, c); - verifyLookup(null, "lkSchema", null, true); - - // verify Lookup is set to null with null query - resetProperties(d, domainURI, c); - verifyLookup(null, null, "lkQuery",true); - - // verify Lookup is set to null with invalid container - resetProperties(d, domainURI, c); - verifyLookup("bogus", null, "lkQuery",true); - - // verify Lookup is set with valid schema and query - resetProperties(d, domainURI, c); - verifyLookup(null, "lkSchema", "lkQuery",true); - - // verify Lookup is set with valid container, schema and query - resetProperties(d, domainURI, c); - verifyLookup(c.getId(), "lkSchema1", "lkQuery2",true); - - // no cleanup as we never persisted anything - } - - private void verifyLookup(String containerId, String schema, String query, Boolean expectedDirty) - { - _pd.setLookupContainer(containerId); - _pd.setLookupQuery(query); - _pd.setLookupSchema(schema); - OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); - assertTrue(_dp.isDirty() == expectedDirty); - assertFalse(_dp._schemaChanged); - - // verify the lookup object returned - Lookup l = _dp.getLookup(); - - if (l == null) - { - // lookup can be null if we specified a containerId that is invalid or - // we specified a valid containerId (including null) but schema or query is null - if (containerId != null && null == ContainerManager.getForId(containerId)) - assertTrue(true); - else if (query == null || schema == null) - assertTrue(true); - else - assertTrue(false); - } - else - { - if (containerId != null) - assertTrue(Strings.CS.equals(l.getContainer().getId(), _pd.getLookupContainer())); - - assertTrue(Strings.CS.equals(l.getQueryName(), _pd.getLookupQuery())); - assertTrue(Strings.CS.equals(l.getSchemaKey().toString(), _pd.getLookupSchema())); - } - } - - private void resetProperties(Domain d, String domainUri, Container c) - { - _pd = getPropertyDescriptor(c, domainUri); - _dp = (DomainPropertyImpl) d.addProperty(); - _pd.copyTo(_dp.getPropertyDescriptor()); - } - - - private PropertyDescriptor getPropertyDescriptor(Container c, String domainURI) - { - PropertyDescriptor pd = new PropertyDescriptor(); - pd.setPropertyURI(domainURI + ":column"); - pd.setName("column"); - pd.setLabel("label"); - pd.setConceptURI(null); - pd.setRangeURI("http://www.w3.org/2001/XMLSchema#int"); - pd.setContainer(c); - pd.setDescription("description"); - pd.setURL(StringExpressionFactory.createURL((String)null)); - pd.setURLTarget(null); - pd.setImportAliases(null); - pd.setRequired(false); - pd.setHidden(false); - pd.setShownInInsertView(true); - pd.setShownInUpdateView(true); - pd.setShownInDetailsView(true); - pd.setDimension(false); - pd.setMeasure(true); - pd.setRecommendedVariable(false); - pd.setDefaultScale(DefaultScaleType.LINEAR); - pd.setFormat(null); - pd.setMvEnabled(false); - pd.setLookupContainer(c.getId()); - pd.setLookupSchema("lkSchema"); - pd.setLookupQuery("lkQuery"); - pd.setFacetingBehaviorType(FacetingBehaviorType.AUTOMATIC); - pd.setPHI(PHI.NotPHI); - pd.setExcludeFromShifting(false); - return pd; - } - } - - -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api.property; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.data.BooleanFormat; +import org.labkey.api.data.ColumnRenderPropertiesImpl; +import org.labkey.api.data.ConditionalFormat; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DatabaseIdentifier; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.PHI; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.exp.ChangePropertyDescriptorException; +import org.labkey.api.exp.DomainDescriptor; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.IPropertyType; +import org.labkey.api.exp.property.IPropertyValidator; +import org.labkey.api.exp.property.Lookup; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.gwt.client.DefaultScaleType; +import org.labkey.api.gwt.client.DefaultValueType; +import org.labkey.api.gwt.client.FacetingBehaviorType; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.security.User; +import org.labkey.api.util.StringExpressionFactory; +import org.labkey.api.util.TestContext; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public class DomainPropertyImpl implements DomainProperty +{ + private final DomainImpl _domain; + + PropertyDescriptor _pd; + PropertyDescriptor _pdOld; + boolean _deleted; + + private boolean _schemaChanged; + private boolean _schemaImport; + private List _validators; + private List _formats; + private String _defaultValue; + + public DomainPropertyImpl(DomainImpl type, PropertyDescriptor pd) + { + this(type, pd, null); + } + + public DomainPropertyImpl(DomainImpl type, PropertyDescriptor pd, List formats) + { + _domain = type; + _pd = pd.clone(); + _formats = formats; + } + + @Override + public int getPropertyId() + { + return _pd.getPropertyId(); + } + + @Override + public Container getContainer() + { + return _pd.getContainer(); + } + + @Override + public String getPropertyURI() + { + return _pd.getPropertyURI(); + } + + @Override + public String getName() + { + return _pd.getName(); + } + + @Override + public String getDescription() + { + return _pd.getDescription(); + } + + @Override + public String getFormat() + { + return _pd.getFormat(); + } + + @Override + public String getLabel() + { + return _pd.getLabel(); + } + + @Override + public String getConceptURI() + { + return _pd.getConceptURI(); + } + + @Override + public Domain getDomain() + { + return _domain; + } + + @Override + public IPropertyType getType() + { + return PropertyService.get().getType(getContainer(), _pd.getRangeURI()); + } + + @Override + public boolean isRequired() + { + return _pd.isRequired(); + } + + @Override + public boolean isHidden() + { + return _pd.isHidden(); + } + + @Override + public boolean isDeleted() + { + return _deleted; + } + + @Override + public boolean isShownInInsertView() + { + return _pd.isShownInInsertView(); + } + + @Override + public boolean isShownInDetailsView() + { + return _pd.isShownInDetailsView(); + } + + @Override + public boolean isShownInUpdateView() + { + return _pd.isShownInUpdateView(); + } + + @Override + public boolean isShownInLookupView() + { + return _pd.isShownInLookupView(); + } + + @Override + public boolean isMeasure() + { + return _pd.isMeasure(); + } + + @Override + public boolean isDimension() + { + return _pd.isDimension(); + } + + @Override + public boolean isRecommendedVariable() + { + return _pd.isRecommendedVariable(); + } + + @Override + public DefaultScaleType getDefaultScale() + { + return _pd.getDefaultScale(); + } + + @Override + public PHI getPHI() + { + return _pd.getPHI(); + } + + @Override + public String getRedactedText() { return _pd.getRedactedText(); } + + @Override + public boolean isExcludeFromShifting() + { + return _pd.isExcludeFromShifting(); + } + + @Override + public boolean isMvEnabled() + { + return _pd.isMvEnabled(); + } + + @Override + public boolean isMvEnabledForDrop() + { + if (null != _pdOld) + return _pdOld.isMvEnabled(); // if we need to drop/recreate we care about the old one + return _pd.isMvEnabled(); + } + + @Override + public void delete() + { + _deleted = true; + } + + @Override + public void setSchemaImport(boolean isSchemaImport) + { + // if this flag is set True then the column is dropped and recreated by its Domain if there is a type change + _schemaImport = isSchemaImport; + } + + @Override + public void setName(String name) + { + if (Strings.CS.equals(name, getName())) + return; + edit().setName(name); + } + + @Override + public void setDescription(String description) + { + if (Strings.CS.equals(description, getDescription())) + return; + edit().setDescription(description); + } + + @Override + public void setType(IPropertyType domain) + { + edit().setRangeURI(domain.getTypeURI()); + } + + @Override + public void setPropertyURI(String uri) + { + if (Strings.CS.equals(uri, getPropertyURI())) + return; + edit().setPropertyURI(uri); + } + + @Override + public void setRangeURI(String rangeURI) + { + if (Strings.CS.equals(rangeURI, getRangeURI())) + return; + editSchema().setRangeURI(rangeURI); + } + + @Override + public String getRangeURI() + { + return _pd.getRangeURI(); + } + + @Override + public void setFormat(String s) + { + if (Strings.CS.equals(s, getFormat())) + return; + edit().setFormat(s); + } + + @Override + public void setLabel(String caption) + { + if (Strings.CS.equals(caption, getLabel())) + return; + edit().setLabel(caption); + } + + @Override + public void setConceptURI(String conceptURI) + { + if (Strings.CS.equals(conceptURI, getConceptURI())) + return; + edit().setConceptURI(conceptURI); + } + + @Override + public void setRequired(boolean required) + { + if (required == isRequired()) + return; + edit().setRequired(required); + } + + @Override + public void setHidden(boolean hidden) + { + if (hidden == isHidden()) + return; + edit().setHidden(hidden); + } + + @Override + public void setShownInDetailsView(boolean shown) + { + if (shown == isShownInDetailsView()) + return; + edit().setShownInDetailsView(shown); + } + + @Override + public void setShownInInsertView(boolean shown) + { + if (shown == isShownInInsertView()) + return; + edit().setShownInInsertView(shown); + } + + @Override + public void setShownInUpdateView(boolean shown) + { + if (shown == isShownInUpdateView()) + return; + edit().setShownInUpdateView(shown); + } + + @Override + public void setShownInLookupView(boolean shown) + { + if (shown == isShownInLookupView()) + return; + edit().setShownInLookupView(shown); + } + + @Override + public void setMeasure(boolean isMeasure) + { + // UNDONE: isMeasure() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. + if (!isEdited() && isMeasure == isMeasure()) + return; + edit().setMeasure(isMeasure); + } + + @Override + public void setDimension(boolean isDimension) + { + // UNDONE: isDimension() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. + if (!isEdited() && isDimension == isDimension()) + return; + edit().setDimension(isDimension); + } + + @Override + public void setRecommendedVariable(boolean isRecommendedVariable) + { + if (!isEdited() && isRecommendedVariable == isRecommendedVariable()) + return; + edit().setRecommendedVariable(isRecommendedVariable); + } + + @Override + public void setDefaultScale(DefaultScaleType defaultScale) + { + if (!isEdited() && getDefaultScale() == defaultScale) + return; + + edit().setDefaultScale(defaultScale); + } + + @Override + public void setPhi(PHI phi) + { + if (!isEdited() && getPHI() == phi) + return; + edit().setPHI(phi); + } + + @Override + public void setRedactedText(String redactedText) + { + if (!isEdited() && ((getRedactedText() != null && getRedactedText().equals(redactedText)) + || (getRedactedText() == null && redactedText == null))) + return; + edit().setRedactedText(redactedText); + } + + @Override + public void setExcludeFromShifting(boolean isExcludeFromShifting) + { + // UNDONE: isExcludeFromShifting() has side-effect due to calling isNumeric()->getSqlTypeInt() which relies on rangeURI which might not be set yet. + if (!isEdited() && isExcludeFromShifting == isExcludeFromShifting()) + return; + edit().setExcludeFromShifting(isExcludeFromShifting); + } + + @Override + public void setMvEnabled(boolean mv) + { + if (mv == isMvEnabled()) + return; + edit().setMvEnabled(mv); + } + + @Override + public void setScale(int scale) + { + if (scale == getScale()) + return; + edit().setScale(scale); + } + + /** Need the string version of this method because it's called by reflection and must match by name */ + public void setImportAliases(String aliases) + { + if (Strings.CS.equals(aliases, getImportAliases())) + return; + edit().setImportAliases(aliases); + } + + /** Need the string version of this method because it's called by reflection and must match by name */ + public String getImportAliases() + { + return _pd.getImportAliases(); + } + + @Override + public void setImportAliasSet(Set aliases) + { + String current = getImportAliases(); + String newAliases = ColumnRenderPropertiesImpl.convertToString(aliases); + if (Strings.CS.equals(current, newAliases)) + return; + edit().setImportAliasesSet(aliases); + } + + @Override + public Set getImportAliasSet() + { + return _pd.getImportAliasSet(); + } + + @Override + public void setURL(String url) + { + if (Strings.CS.equals(getURL(), url)) + return; + + if (null == url) + edit().setURL(null); + else + edit().setURL(StringExpressionFactory.createURL(url)); + } + + @Override + public String getURL() + { + return _pd.getURL() == null ? null : _pd.getURL().toString(); + } + + @Override + public void setURLTarget(String urlTarget) + { + if (Strings.CS.equals(getURLTarget(), urlTarget)) + return; + edit().setURLTarget(urlTarget); + } + + @Override + public String getURLTarget() + { + return _pd.getURLTarget(); + } + + private boolean isEdited() + { + return null != _pdOld; + } + + private PropertyDescriptor editSchema() + { + PropertyDescriptor pd = edit(); + _schemaChanged = true; + _pd.clearPropertyType(); + return pd; + } + + public boolean isRecreateRequired() + { + return _schemaChanged && _schemaImport; + } + + public void markAsNew() + { + assert isRecreateRequired() && !isNew(); + _pd.setPropertyId(0); + } + + private PropertyDescriptor edit() + { + if (_pdOld == null) + { + _pdOld = _pd; + _pd = _pdOld.clone(); + } + return _pd; + } + + @Override + public PropertyType getPropertyType() + { + return _pd.getPropertyType(); + } + + @Override + public JdbcType getJdbcType() + { + return _pd.getPropertyType().getJdbcType(); + } + + @Override + public int getScale() + { + return _pd.getScale(); + } + + @Override + public String getInputType() + { + return _pd.getPropertyType().getInputType(); + } + + @Override + public DefaultValueType getDefaultValueTypeEnum() + { + return _pd.getDefaultValueTypeEnum(); + } + + @Override + public void setDefaultValueTypeEnum(DefaultValueType defaultValueType) + { + _pd.setDefaultValueTypeEnum(defaultValueType); + } + + public String getDefaultValueType() + { + return _pd.getDefaultValueType(); + } + + @Override + public void setDefaultValueType(String defaultValueTypeName) + { + if (getDefaultValueType() != null && getDefaultValueType().equals(defaultValueTypeName)) + return; + + if (getDefaultValueType() == null && defaultValueTypeName == null) + return; // if both are null, don't call edit(), with marks property as dirty + + edit().setDefaultValueType(defaultValueTypeName); + } + + @Override + public void setDefaultValue(String value) + { + _defaultValue = value; + } + + public String getDefaultValue() + { + return _defaultValue; + } + + @Override + public Lookup getLookup() + { + return _pd.getLookup(); + } + + @Override + public void setLookup(Lookup lookup) + { + Lookup current = getLookup(); + + if (current == lookup) + return; + + // current will return null if the schema or query is null so check + // for this case in the passed in lookup + if (current == null) + if (lookup.getQueryName() == null || lookup.getSchemaKey() == null) + return; + + if (current != null && current.equals(lookup)) + return; + + if (lookup == null) + { + edit().setLookupContainer(null); + edit().setLookupSchema(null); + edit().setLookupQuery(null); + return; + } + if (lookup.getContainer() == null) + { + edit().setLookupContainer(null); + } + else + { + edit().setLookupContainer(lookup.getContainer().getId()); + } + edit().setLookupQuery(lookup.getQueryName()); + edit().setLookupSchema(Objects.toString(lookup.getSchemaKey(),null)); + } + + @Override + public void setScannable(boolean scannable) + { + if (scannable != isScannable()) + edit().setScannable(scannable); + } + + @Override + public void setOldPropertyDescriptor(PropertyDescriptor oldPropertyDescriptor) + { + if (isEdited()) + return; + + _pdOld = oldPropertyDescriptor.clone(); + } + + @Override + public boolean isScannable() + { + return _pd.isScannable(); + } + + @Override + public void setPrincipalConceptCode(String code) + { + if (!Strings.CS.equals(code, getPrincipalConceptCode())) + edit().setPrincipalConceptCode(code); + } + + @Override + public String getPrincipalConceptCode() + { + return _pd.getPrincipalConceptCode(); + } + + @Override + public String getSourceOntology() + { + return _pd.getSourceOntology(); + } + + @Override + public void setSourceOntology(String sourceOntology) + { + if (!Strings.CS.equals(sourceOntology, getSourceOntology())) + edit().setSourceOntology(sourceOntology); + } + + @Override + public String getConceptSubtree() + { + return _pd.getConceptSubtree(); + } + + @Override + public void setConceptSubtree(String path) + { + if (!Strings.CS.equals(path, getConceptSubtree())) + edit().setConceptSubtree(path); + } + + @Override + public String getConceptImportColumn() + { + return _pd.getConceptImportColumn(); + } + + @Override + public void setConceptImportColumn(String conceptImportColumn) + { + if (!Strings.CS.equals(conceptImportColumn, getConceptImportColumn())) + edit().setConceptImportColumn(conceptImportColumn); + } + + @Override + public String getConceptLabelColumn() + { + return _pd.getConceptLabelColumn(); + } + + @Override + public void setConceptLabelColumn(String conceptLabelColumn) + { + if (!Strings.CS.equals(conceptLabelColumn, getConceptLabelColumn())) + edit().setConceptLabelColumn(conceptLabelColumn); + } + + @Override + public void setDerivationDataScope(String scope) + { + if (!Strings.CS.equals(scope, getDerivationDataScope())) + edit().setDerivationDataScope(scope); + } + + @Override + public String getDerivationDataScope() + { + return _pd.getDerivationDataScope(); + } + + @Override + public PropertyDescriptor getPropertyDescriptor() + { + return _pd; + } + + @Override + public List getConditionalFormats() + { + return ensureConditionalFormats(); + } + + public boolean isNew() + { + return _pd.getPropertyId() == 0; + } + + // Scenario to swap property descriptors on study upload to or from a system property, instead of updating the + // current property descriptor. Avoids overwriting a system property. + public boolean isSystemPropertySwap() + { + if (_pd.getPropertyId() == 0 && _pd.getPropertyURI() != null && _pdOld != null && _pdOld.getPropertyURI() != null + && !_pd.getPropertyURI().equals(_pdOld.getPropertyURI())) + { + return SystemProperty.getProperties().stream().anyMatch(sp -> + sp.getPropertyURI().equals(_pd.getPropertyURI()) || sp.getPropertyURI().equals(_pdOld.getPropertyURI())); + } + + return false; + } + + public boolean isDirty() + { + if (_pdOld != null) return true; + + for (PropertyValidatorImpl v : ensureValidators()) + { + if (v.isDirty() || v.isNew()) + return true; + } + return false; + } + + public void delete(User user) + { + DomainPropertyManager.get().removeValidatorsForPropertyDescriptor(getContainer(), getPropertyId()); + DomainPropertyManager.get().deleteConditionalFormats(getPropertyId()); + + DomainKind kind = getDomain().getDomainKind(); + if (null != kind) + kind.deletePropertyDescriptor(getDomain(), user, _pd); + OntologyManager.removePropertyDescriptorFromDomain(this); + } + + public void save(User user, DomainDescriptor dd, int sortOrder) throws ChangePropertyDescriptorException + { + if (isSystemPropertySwap()) + { + _pd = OntologyManager.insertOrUpdatePropertyDescriptor(_pd, dd, sortOrder); + OntologyManager.removePropertyDescriptorFromDomain(new DomainPropertyImpl((DomainImpl) getDomain(), _pdOld)); + } + else if (isNew()) + { + _pd = OntologyManager.insertOrUpdatePropertyDescriptor(_pd, dd, sortOrder); + } + else if (_pdOld != null) + { + PropertyType oldType = _pdOld.getPropertyType(); + PropertyType newType = _pd.getPropertyType(); + boolean changedType = false; + if (oldType.getJdbcType() != newType.getJdbcType()) + { + if (newType.getJdbcType().isText() || + (oldType.getJdbcType().isInteger() && newType.getJdbcType().isNumeric())) + { + changedType = true; + if (newType.getJdbcType().isText()) + { + // Remove any previously set formatting string as it won't apply to a text field + _pd.setFormat(null); + } + } + else if (newType.getJdbcType().isDateOrTime() && oldType.getJdbcType().isDateOrTime()) + { + changedType = true; + _pd.setFormat(null); + } + else if (newType == PropertyType.MULTI_CHOICE || oldType == PropertyType.MULTI_CHOICE) + { + changedType = true; + _pd.setFormat(null); + } + else + { + throw new ChangePropertyDescriptorException("Cannot convert an instance of " + oldType.getJdbcType() + " to " + newType.getJdbcType() + "."); + } + } + + // Issue 44711: Prevent attachment and file field types from being converted to a different type + if (PropertyType.FILE_LINK.getInputType().equalsIgnoreCase(oldType.getInputType()) && oldType != newType) + throw new ChangePropertyDescriptorException("Cannot convert an instance of " + oldType.name() + " to " + newType.name() + "."); + + OntologyManager.validatePropertyDescriptor(_pd); + Table.update(user, OntologyManager.getTinfoPropertyDescriptor(), _pd, _pdOld.getPropertyId()); + OntologyManager.ensurePropertyDomain(_pd, dd, sortOrder); + + boolean hasProvisioner = null != getDomain().getDomainKind() && null != getDomain().getDomainKind().getStorageSchemaName() && dd.getStorageTableName() != null; + SqlDialect dialect = OntologyManager.getExpSchema().getSqlDialect(); + + if (hasProvisioner) + { + boolean mvAdded = !_pdOld.isMvEnabled() && _pd.isMvEnabled(); + boolean mvDropped = _pdOld.isMvEnabled() && !_pd.isMvEnabled(); + boolean propRenamed = !_pdOld.getName().equals(_pd.getName()); + boolean propResized = _pd.isStringType() && _pdOld.getScale() != _pd.getScale(); + + // Drop first, so rename doesn't have to worry about it + if (mvDropped) + ((StorageProvisionerImpl)StorageProvisioner.get()).dropMvIndicator(this, _pdOld); + + if (propRenamed) + StorageProvisionerImpl.get().renameProperty(this.getDomain(), this, _pdOld, mvDropped); + + if (changedType) + { + var domainKind = _domain.getDomainKind(); + if (domainKind == null) + throw new ChangePropertyDescriptorException("Cannot change property type for domain, unknown domain kind."); + + StorageProvisionerImpl.get().changePropertyType(this.getDomain(), this); + if (_pdOld.getJdbcType() == JdbcType.BOOLEAN && _pd.getJdbcType().isText()) + { + updateBooleanValue( + new SQLFragment().appendIdentifier(domainKind.getStorageSchemaName()).append(".").appendIdentifier(_domain.getStorageTableName()), + _pd.getLegalSelectName(dialect), _pdOld.getFormat(), null); // GitHub Issue #647 + } + + TableInfo table = domainKind.getTableInfo(user, getContainer(), _domain, ContainerFilter.getUnsafeEverythingFilter()); + if (table != null && _pdOld.getPropertyType() != null) + QueryChangeListener.QueryPropertyChange.handleColumnTypeChange(_pdOld, _pd, SchemaKey.fromString(table.getUserSchema().getName()), table.getName(), user, getContainer()); + } + else if (propResized) + StorageProvisionerImpl.get().resizeProperty(this.getDomain(), this, _pdOld.getScale()); + + if (mvAdded) + StorageProvisionerImpl.get().addMvIndicator(this); + } + else if (changedType) + { + if (oldType.getJdbcType().isDateOrTime() && newType.getJdbcType().isText()) + { + new SqlExecutor(OntologyManager.getExpSchema()).execute( + new SQLFragment("UPDATE "). + append(OntologyManager.getTinfoObjectProperty()). + append(" SET StringValue = DateTimeValue, DateTimeValue = NULL WHERE PropertyId = ?"). + add(_pdOld.getPropertyId())); + } + else if (!oldType.getJdbcType().isText() && newType.getJdbcType().isText()) + { + new SqlExecutor(OntologyManager.getExpSchema()).execute( + new SQLFragment("UPDATE "). + append(OntologyManager.getTinfoObjectProperty()). + append(" SET StringValue = FloatValue, FloatValue = NULL WHERE PropertyId = ?"). + add(_pdOld.getPropertyId())); + } + else if (oldType.getJdbcType().isDateOrTime() && newType.getJdbcType().isDateOrTime()) + { + String sqlTypeName = dialect.getSqlTypeName(newType.getJdbcType()); + String update = String.format("CAST(DateTimeValue AS %s)", sqlTypeName); + if (newType.getJdbcType() == JdbcType.TIME) + update = dialect.getDateTimeToTimeCast("DateTimeValue"); + SQLFragment sqlFragment = new SQLFragment("UPDATE ") + .append(OntologyManager.getTinfoObjectProperty()) + .append(" SET DateTimeValue = ") + .append(update) + .append(" WHERE PropertyId = ?") + .add(_pdOld.getPropertyId()); + new SqlExecutor(OntologyManager.getExpSchema()).execute(sqlFragment); + } + else //noinspection StatementWithEmptyBody + if (oldType.getJdbcType().isInteger() && newType.getJdbcType().isReal()) + { + // Since exp.ObjectProperty stores these types in the same column, there's nothing for us to do + } + else + { + throw new ChangePropertyDescriptorException("Cannot convert from " + oldType.getJdbcType() + " to " + newType.getJdbcType() + " for non-provisioned table"); + } + } + + if (changedType && _pdOld.getJdbcType() == JdbcType.BOOLEAN && _pd.getJdbcType().isText()) + { + updateBooleanValue(OntologyManager.getTinfoObjectProperty().getSQLName(), dialect.makeDatabaseIdentifier("StringValue"), _pdOld.getFormat(), new SQLFragment("PropertyId = ?", _pdOld.getPropertyId())); + } + } + else + { + OntologyManager.ensurePropertyDomain(_pd, _domain._dd, sortOrder); + } + + _pdOld = null; + _schemaChanged = false; + _schemaImport = false; + + for (PropertyValidatorImpl validator : ensureValidators()) + { + if (validator.isDeleted()) + DomainPropertyManager.get().removePropertyValidator(this, validator); + else + DomainPropertyManager.get().savePropertyValidator(user, this, validator); + } + + DomainPropertyManager.get().saveConditionalFormats(user, getPropertyDescriptor(), ensureConditionalFormats()); + } + + /** + * Format values in columns that were just converted from booleans to strings with the DB's default type conversion. + * Postgres will now have 'true' and 'false', and SQLServer will have '0' and '1'. Use the format string to use the + * preferred format, and standardize on 'true' and 'false' in the absence of an explicitly configured format. + */ + private void updateBooleanValue(SQLFragment schemaTable, DatabaseIdentifier column, String formatString, @Nullable SQLFragment whereClause) + { + BooleanFormat f = BooleanFormat.getInstance(formatString); + String trueValue = StringUtils.trimToNull(f.format(true)); + String falseValue = StringUtils.trimToNull(f.format(false)); + String nullValue = StringUtils.trimToNull(f.format(null)); + SQLFragment sql = new SQLFragment("UPDATE ").append(schemaTable).append(" SET "). + appendIdentifier(column).append(" = CASE WHEN "). + appendIdentifier(column).append(" IN ('1', 'true') THEN ? WHEN "). + appendIdentifier(column).append(" IN ('0', 'false') THEN ? ELSE ? END"); + sql.add(trueValue); + sql.add(falseValue); + sql.add(nullValue); + if (whereClause != null) + { + sql.append(" WHERE "); + sql.append(whereClause); + } + new SqlExecutor(OntologyManager.getExpSchema()).execute(sql); + } + + @Override + @NotNull + public List getValidators() + { + return Collections.unmodifiableList(ensureValidators()); + } + + @Override + public void addValidator(IPropertyValidator validator) + { + if (validator != null) + { + if (0 != validator.getPropertyId() && getPropertyId() != validator.getPropertyId()) + throw new IllegalStateException(); + + // Ensure validator is a valid kind (ex. urn:lsid:labkey.com:PropertyValidator:length is no longer valid) + if ( null != PropertyService.get().getValidatorKind(validator.getTypeURI()) ) + { + PropertyValidator impl = new PropertyValidator(); + impl.copy(validator); + impl.setPropertyId(getPropertyId()); + ensureValidators().add(new PropertyValidatorImpl(impl)); + } + } + } + + @Override + public void removeValidator(IPropertyValidator validator) + { + int idx = ensureValidators().indexOf(validator); + if (idx != -1) + { + PropertyValidatorImpl impl = ensureValidators().get(idx); + impl.delete(); + } + } + + @Override + public void removeValidator(long validatorId) + { + if (validatorId == 0) return; + + for (PropertyValidatorImpl imp : ensureValidators()) + { + if (imp.getRowId() == validatorId) + { + imp.delete(); + break; + } + } + } + + @Override + public void copyFrom(DomainProperty propSrc, Container targetContainer) + { + setDescription(propSrc.getDescription()); + setFormat(propSrc.getFormat()); + setLabel(propSrc.getLabel()); + setName(propSrc.getName()); + setDescription(propSrc.getDescription()); + setConceptURI(propSrc.getConceptURI()); + setType(propSrc.getType()); + setDimension(propSrc.isDimension()); + setMeasure(propSrc.isMeasure()); + setRecommendedVariable(propSrc.isRecommendedVariable()); + setDefaultScale(propSrc.getDefaultScale()); + setRequired(propSrc.isRequired()); + setExcludeFromShifting(propSrc.isExcludeFromShifting()); + setFacetingBehavior(propSrc.getFacetingBehavior()); + setImportAliasSet(propSrc.getImportAliasSet()); + setPhi(propSrc.getPHI()); + setURL(propSrc.getURL()); + setURLTarget(propSrc.getURLTarget()); + setHidden(propSrc.isHidden()); + setShownInDetailsView(propSrc.isShownInDetailsView()); + setShownInInsertView(propSrc.isShownInInsertView()); + setShownInUpdateView(propSrc.isShownInUpdateView()); + setShownInLookupView(propSrc.isShownInLookupView()); + setMvEnabled(propSrc.isMvEnabled()); + setDefaultValueTypeEnum(propSrc.getDefaultValueTypeEnum()); + setScale(propSrc.getScale()); + setScannable(propSrc.isScannable()); + + setPrincipalConceptCode(propSrc.getPrincipalConceptCode()); + setSourceOntology(propSrc.getSourceOntology()); + setConceptSubtree(propSrc.getConceptSubtree()); + setConceptImportColumn(propSrc.getConceptImportColumn()); + setConceptLabelColumn(propSrc.getConceptLabelColumn()); + setDerivationDataScope(propSrc.getDerivationDataScope()); + + // check to see if we're moving a lookup column to another container: + Lookup lookup = propSrc.getLookup(); + if (lookup != null && !getContainer().equals(targetContainer)) + { + // we need to update the lookup properties if the lookup container is either the source or the destination container + if (lookup.getContainer() == null) + lookup.setContainer(propSrc.getContainer()); + else if (lookup.getContainer().equals(targetContainer)) + lookup.setContainer(null); + } + setLookup(lookup); + } + + @Override + public void setConditionalFormats(List formats) + { + String newVal = ConditionalFormat.toStringVal(formats); + String oldVal = ConditionalFormat.toStringVal(getConditionalFormats()); + + if (!Objects.equals(newVal, oldVal)) + edit(); + + _formats = formats; + } + + private List ensureValidators() + { + if (_validators == null) + { + _validators = new ArrayList<>(); + for (PropertyValidator validator : DomainPropertyManager.get().getValidators(this)) + { + _validators.add(new PropertyValidatorImpl(validator)); + } + } + return _validators; + } + + private List ensureConditionalFormats() + { + if (_formats == null) + { + _formats = new ArrayList<>(); + _formats.addAll(DomainPropertyManager.get().getConditionalFormats(this)); + } + return _formats; + } + + public PropertyDescriptor getOldProperty() + { + return _pdOld; + } + + @Override + public FacetingBehaviorType getFacetingBehavior() + { + return _pd.getFacetingBehaviorType(); + } + + @Override + public void setFacetingBehavior(FacetingBehaviorType type) + { + if (getFacetingBehavior() == type) + return; + + edit().setFacetingBehaviorType(type); + } + + @Override + public int hashCode() + { + return _pd.hashCode(); + } + + @Override + public boolean equals(Object obj) + { + if (obj == this) + return true; + if (!(obj instanceof DomainPropertyImpl)) + return false; + // once a domain property has been edited, it no longer equals any other domain property: + if (_pdOld != null || ((DomainPropertyImpl) obj)._pdOld != null) + return false; + return (_pd.equals(((DomainPropertyImpl) obj)._pd)); + } + + @Override + public String toString() + { + return super.toString() + _pd.getPropertyURI(); + } + + public Map getAuditRecordMap(@Nullable String validatorStr, @Nullable String conditionalFormatStr) + { + Map map = new LinkedHashMap<>(); + if (!StringUtils.isEmpty(getName())) + map.put("Name", getName()); + if (!StringUtils.isEmpty(getLabel())) + map.put("Label", getLabel()); + if (null != getPropertyType()) + { + if (org.labkey.api.gwt.client.ui.PropertyType.expFlag.getURI().equals(getConceptURI())) + map.put("Type", "Flag"); + else + map.put("Type", getPropertyType().getXarName()); + } + if (getPropertyType().getJdbcType().isText()) + map.put("Scale", getScale()); + if (!StringUtils.isEmpty(getDescription())) + map.put("Description", getDescription()); + if (!StringUtils.isEmpty(getFormat())) + map.put("Format", getFormat()); + if (!StringUtils.isEmpty(getURL())) + map.put("URL", getURL()); + if (!StringUtils.isEmpty(getURLTarget())) + map.put("URLTarget", getURLTarget()); + if (getPHI() != null) + map.put("PHI", getPHI().getLabel()); + if (getDefaultScale() != null) + map.put("DefaultScale", getDefaultScale().getLabel()); + map.put("Required", isRequired()); + map.put("Hidden", isHidden()); + map.put("MvEnabled", isMvEnabled()); + map.put("Measure", isMeasure()); + map.put("Dimension", isDimension()); + map.put("ShownInInsert", isShownInInsertView()); + map.put("ShownInDetails", isShownInDetailsView()); + map.put("ShownInUpdate", isShownInUpdateView()); + map.put("ShownInLookupView", isShownInLookupView()); + map.put("RecommendedVariable", isRecommendedVariable()); + map.put("ExcludedFromShifting", isExcludeFromShifting()); + map.put("Scannable", isScannable()); + if (!StringUtils.isEmpty(getDerivationDataScope())) + map.put("DerivationDataScope", getDerivationDataScope()); + String importAliasStr = StringUtils.join(getImportAliasSet(), ","); + if (!StringUtils.isEmpty(importAliasStr)) + map.put("ImportAliases", importAliasStr); + if (getDefaultValueTypeEnum() != null) + map.put("DefaultValueType", getDefaultValueTypeEnum().getLabel()); + if (getLookup() != null) + map.put("Lookup", getLookup().toJSONString()); + + if (!StringUtils.isEmpty(validatorStr)) + map.put("Validator", validatorStr); + if (!StringUtils.isEmpty(conditionalFormatStr)) + map.put("ConditionalFormat", conditionalFormatStr); + + return map; + } + + public static class TestCase extends Assert + { + private PropertyDescriptor _pd; + private DomainPropertyImpl _dp; + + @Test + public void testUpdateDomainPropertyFromDescriptor() + { + Container c = ContainerManager.ensureContainer("/_DomainPropertyImplTest", TestContext.get().getUser()); + String domainURI = new Lsid("Junit", "DD", "Domain1").toString(); + Domain d = PropertyService.get().createDomain(c, domainURI, "Domain1"); + + resetProperties(d, domainURI, c); + + // verify no change + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertFalse(_dp.isDirty()); + assertFalse(_dp._schemaChanged); + + // change a property + _pd.setPHI(PHI.Restricted); + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertTrue(_dp.isDirty()); + assertFalse(_dp._schemaChanged); + assertTrue(_dp.getPHI() == _pd.getPHI()); + + // Issue #18738 change the schema outside of a schema reload and verify that the column + // change the schema but don't mark the property as "Schema Import" + // this will allow whatever type changes the UI allows (text -> multiline, for example) + resetProperties(d, domainURI, c); + _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#double"); + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertTrue(_dp.isDirty()); + assertTrue(_dp._schemaChanged); + assertFalse(_dp.isRecreateRequired()); + assertTrue(Strings.CS.equals(_dp.getRangeURI(), _pd.getRangeURI())); + + // setting schema import to true will enable the _schemaChanged flag to toggle + // so it should be set true here + resetProperties(d, domainURI, c); + _dp.setSchemaImport(true); + _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#double"); + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertTrue(_dp.isDirty()); + assertTrue(_dp._schemaChanged); + assertTrue(_dp.isRecreateRequired()); + assertTrue(Strings.CS.equals(_dp.getRangeURI(), _pd.getRangeURI())); + + // verify no change when setting value to the same value as it was + resetProperties(d, domainURI, c); + _pd.setRangeURI("http://www.w3.org/2001/XMLSchema#int"); + _pd.setPHI(PHI.NotPHI); + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertFalse(_dp.isDirty()); + assertFalse(_dp._schemaChanged); + assertFalse(_dp.isRecreateRequired()); + + // verify Lookup is set to null with null schema + resetProperties(d, domainURI, c); + verifyLookup(null, "lkSchema", null, true); + + // verify Lookup is set to null with null query + resetProperties(d, domainURI, c); + verifyLookup(null, null, "lkQuery",true); + + // verify Lookup is set to null with invalid container + resetProperties(d, domainURI, c); + verifyLookup("bogus", null, "lkQuery",true); + + // verify Lookup is set with valid schema and query + resetProperties(d, domainURI, c); + verifyLookup(null, "lkSchema", "lkQuery",true); + + // verify Lookup is set with valid container, schema and query + resetProperties(d, domainURI, c); + verifyLookup(c.getId(), "lkSchema1", "lkQuery2",true); + + // no cleanup as we never persisted anything + } + + private void verifyLookup(String containerId, String schema, String query, Boolean expectedDirty) + { + _pd.setLookupContainer(containerId); + _pd.setLookupQuery(query); + _pd.setLookupSchema(schema); + OntologyManager.updateDomainPropertyFromDescriptor(_dp, _pd); + assertTrue(_dp.isDirty() == expectedDirty); + assertFalse(_dp._schemaChanged); + + // verify the lookup object returned + Lookup l = _dp.getLookup(); + + if (l == null) + { + // lookup can be null if we specified a containerId that is invalid or + // we specified a valid containerId (including null) but schema or query is null + if (containerId != null && null == ContainerManager.getForId(containerId)) + assertTrue(true); + else if (query == null || schema == null) + assertTrue(true); + else + assertTrue(false); + } + else + { + if (containerId != null) + assertTrue(Strings.CS.equals(l.getContainer().getId(), _pd.getLookupContainer())); + + assertTrue(Strings.CS.equals(l.getQueryName(), _pd.getLookupQuery())); + assertTrue(Strings.CS.equals(l.getSchemaKey().toString(), _pd.getLookupSchema())); + } + } + + private void resetProperties(Domain d, String domainUri, Container c) + { + _pd = getPropertyDescriptor(c, domainUri); + _dp = (DomainPropertyImpl) d.addProperty(); + _pd.copyTo(_dp.getPropertyDescriptor()); + } + + + private PropertyDescriptor getPropertyDescriptor(Container c, String domainURI) + { + PropertyDescriptor pd = new PropertyDescriptor(); + pd.setPropertyURI(domainURI + ":column"); + pd.setName("column"); + pd.setLabel("label"); + pd.setConceptURI(null); + pd.setRangeURI("http://www.w3.org/2001/XMLSchema#int"); + pd.setContainer(c); + pd.setDescription("description"); + pd.setURL(StringExpressionFactory.createURL((String)null)); + pd.setURLTarget(null); + pd.setImportAliases(null); + pd.setRequired(false); + pd.setHidden(false); + pd.setShownInInsertView(true); + pd.setShownInUpdateView(true); + pd.setShownInDetailsView(true); + pd.setDimension(false); + pd.setMeasure(true); + pd.setRecommendedVariable(false); + pd.setDefaultScale(DefaultScaleType.LINEAR); + pd.setFormat(null); + pd.setMvEnabled(false); + pd.setLookupContainer(c.getId()); + pd.setLookupSchema("lkSchema"); + pd.setLookupQuery("lkQuery"); + pd.setFacetingBehaviorType(FacetingBehaviorType.AUTOMATIC); + pd.setPHI(PHI.NotPHI); + pd.setExcludeFromShifting(false); + return pd; + } + } + + +} diff --git a/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java b/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java index 4da70cab723..4d20461cb46 100644 --- a/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java +++ b/query/src/org/labkey/query/QuerySnapshotQueryChangeListener.java @@ -1,141 +1,141 @@ -/* - * Copyright (c) 2013-2017 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.query; - -import org.apache.logging.log4j.LogManager; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.snapshot.QuerySnapshotDefinition; -import org.labkey.api.security.User; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * QueryChangeListener for query snapshots. - * - * User: cnathe - * Date: 4/19/13 - */ -public class QuerySnapshotQueryChangeListener implements QueryChangeListener -{ - @Override - public void queryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) - { - } - - @Override - public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) - { - if (property.equals(QueryProperty.Name)) - { - _updateQuerySnapshotQueryNameChange(user, container, schema, changes); - } - if (property.equals(QueryProperty.SchemaName)) - { - _updateQuerySnapshotSchemaNameChange(user, container, changes); - } - } - - @Override - public void queryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) - { - } - - @Override - public Collection queryDependents(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) - { - // UNDONE - return Collections.emptyList(); - } - - private void _updateQuerySnapshotQueryNameChange(User user, Container container, SchemaKey schemaKey, Collection> changes) - { - // most property updates only care about the query name old value string and new value string - Map queryNameChangeMap = new HashMap<>(); - for (QueryPropertyChange qpc : changes) - { - String oldVal = (String)qpc.getOldValue(); - String newVal = (String)qpc.getNewValue(); - if (oldVal != null && !oldVal.equals(newVal)) - queryNameChangeMap.put((String)qpc.getOldValue(), (String)qpc.getNewValue()); - } - - for (QuerySnapshotDefinition qsd : QueryService.get().getQuerySnapshotDefs(container, schemaKey.toString())) - { - try - { - // update QueryTableName (stored in query.QuerySnapshotDef) - boolean changed = false; - String queryTableName = qsd.getQueryTableName(); - if (null != queryTableName && queryNameChangeMap.containsKey(queryTableName)) - { - qsd.setQueryTableName(queryNameChangeMap.get(queryTableName)); - changed = true; - } - String snapshotName = qsd.getName(); - if (null != snapshotName && queryNameChangeMap.containsKey(snapshotName)) - { - qsd.setName(queryNameChangeMap.get(snapshotName)); - changed = true; - } - if (changed) - qsd.save(qsd.getModifiedBy()); - } - catch (Exception e) - { - LogManager.getLogger(QuerySnapshotQueryChangeListener.class).error("An error occurred upgrading query snapshot properties: ", e); - } - } - } - - private void _updateQuerySnapshotSchemaNameChange(User user, Container container, Collection> changes) - { - Map schemaNameChangeMap = new HashMap<>(); - for (QueryPropertyChange qpc : changes) - { - if (qpc.getOldValue().equals(qpc.getNewValue())) - continue; - schemaNameChangeMap.put((String)qpc.getOldValue(), (String)qpc.getNewValue()); - } - - if (schemaNameChangeMap.isEmpty()) - return; - - for (String oldSchema : schemaNameChangeMap.keySet()) - { - String newSchema = schemaNameChangeMap.get(oldSchema); - for (QuerySnapshotDefinition qsd : QueryService.get().getQuerySnapshotDefs(container, oldSchema)) - { - qsd.setSchema(newSchema); - try - { - qsd.save(qsd.getModifiedBy()); - } - catch (Exception e) - { - LogManager.getLogger(QuerySnapshotQueryChangeListener.class).error("An error occurred upgrading query snapshot properties: ", e); - } - } - } - } -} +/* + * Copyright (c) 2013-2017 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.query; + +import org.apache.logging.log4j.LogManager; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.snapshot.QuerySnapshotDefinition; +import org.labkey.api.security.User; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * QueryChangeListener for query snapshots. + * + * User: cnathe + * Date: 4/19/13 + */ +public class QuerySnapshotQueryChangeListener implements QueryChangeListener +{ + @Override + public void queryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) + { + } + + @Override + public void queryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, String queryName, @NotNull QueryProperty property, @NotNull Collection> changes) + { + if (property.equals(QueryProperty.Name)) + { + _updateQuerySnapshotQueryNameChange(user, container, schema, changes); + } + if (property.equals(QueryProperty.SchemaName)) + { + _updateQuerySnapshotSchemaNameChange(user, container, changes); + } + } + + @Override + public void queryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) + { + } + + @Override + public Collection queryDependents(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) + { + // UNDONE + return Collections.emptyList(); + } + + private void _updateQuerySnapshotQueryNameChange(User user, Container container, SchemaKey schemaKey, Collection> changes) + { + // most property updates only care about the query name old value string and new value string + Map queryNameChangeMap = new HashMap<>(); + for (QueryPropertyChange qpc : changes) + { + String oldVal = (String)qpc.getOldValue(); + String newVal = (String)qpc.getNewValue(); + if (oldVal != null && !oldVal.equals(newVal)) + queryNameChangeMap.put((String)qpc.getOldValue(), (String)qpc.getNewValue()); + } + + for (QuerySnapshotDefinition qsd : QueryService.get().getQuerySnapshotDefs(container, schemaKey.toString())) + { + try + { + // update QueryTableName (stored in query.QuerySnapshotDef) + boolean changed = false; + String queryTableName = qsd.getQueryTableName(); + if (null != queryTableName && queryNameChangeMap.containsKey(queryTableName)) + { + qsd.setQueryTableName(queryNameChangeMap.get(queryTableName)); + changed = true; + } + String snapshotName = qsd.getName(); + if (null != snapshotName && queryNameChangeMap.containsKey(snapshotName)) + { + qsd.setName(queryNameChangeMap.get(snapshotName)); + changed = true; + } + if (changed) + qsd.save(qsd.getModifiedBy()); + } + catch (Exception e) + { + LogManager.getLogger(QuerySnapshotQueryChangeListener.class).error("An error occurred upgrading query snapshot properties: ", e); + } + } + } + + private void _updateQuerySnapshotSchemaNameChange(User user, Container container, Collection> changes) + { + Map schemaNameChangeMap = new HashMap<>(); + for (QueryPropertyChange qpc : changes) + { + if (qpc.getOldValue().equals(qpc.getNewValue())) + continue; + schemaNameChangeMap.put((String)qpc.getOldValue(), (String)qpc.getNewValue()); + } + + if (schemaNameChangeMap.isEmpty()) + return; + + for (String oldSchema : schemaNameChangeMap.keySet()) + { + String newSchema = schemaNameChangeMap.get(oldSchema); + for (QuerySnapshotDefinition qsd : QueryService.get().getQuerySnapshotDefs(container, oldSchema)) + { + qsd.setSchema(newSchema); + try + { + qsd.save(qsd.getModifiedBy()); + } + catch (Exception e) + { + LogManager.getLogger(QuerySnapshotQueryChangeListener.class).error("An error occurred upgrading query snapshot properties: ", e); + } + } + } + } +} diff --git a/query/src/org/labkey/query/persist/QueryManager.java b/query/src/org/labkey/query/persist/QueryManager.java index d9d5bda4a51..c041d8c9a2d 100644 --- a/query/src/org/labkey/query/persist/QueryManager.java +++ b/query/src/org/labkey/query/persist/QueryManager.java @@ -1,1177 +1,1177 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.query.persist; - -import org.apache.commons.collections4.Bag; -import org.apache.commons.collections4.bag.HashBag; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerType; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.FilterInfo; -import org.labkey.api.data.JsonWriter; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.ontology.Concept; -import org.labkey.api.ontology.OntologyService; -import org.labkey.api.query.AliasedColumn; -import org.labkey.api.query.CustomView; -import org.labkey.api.query.CustomViewChangeListener; -import org.labkey.api.query.CustomViewInfo; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryChangeListener.QueryPropertyChange; -import org.labkey.api.query.QueryDefinition; -import org.labkey.api.query.QueryParseException; -import org.labkey.api.query.QueryParseWarning; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.view.NotFoundException; -import org.labkey.query.ExternalSchema; -import org.labkey.query.ExternalSchemaDocumentProvider; -import org.springframework.jdbc.BadSqlGrammarException; - -import java.net.URISyntaxException; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.Function; -import java.util.stream.Collectors; - - -public class QueryManager -{ - private static final Logger _log = LogManager.getLogger(QueryManager.class); - private static final QueryManager instance = new QueryManager(); - private static final String SCHEMA_NAME = "query"; - private static final List QUERY_LISTENERS = new CopyOnWriteArrayList<>(); - private static final List VIEW_LISTENERS = new CopyOnWriteArrayList<>(); - - public static final int FLAG_INHERITABLE = 0x01; - public static final int FLAG_HIDDEN = 0x02; - public static final int FLAG_SNAPSHOT = 0x04; - - public static QueryManager get() - { - return instance; - } - - /** - * @param customQuery whether to look for custom queries or modified metadata on built-in tables - */ - public QueryDef getQueryDef(Container container, String schema, String name, boolean customQuery) - { - return QueryDefCache.getQueryDef(container, schema, name, customQuery); - } - - /** - * @param customQuery whether to look for custom queries or modified metadata on built-in tables - */ - public List getQueryDefs(Container container, @Nullable String schema, boolean inheritableOnly, boolean includeSnapshots, boolean customQuery) - { - return QueryDefCache.getQueryDefs(container, schema, inheritableOnly, includeSnapshots, customQuery); - } - - public Collection getQuerySnapshots(@Nullable Container container, @Nullable String schemaName) - { - return QuerySnapshotCache.getQuerySnapshotDefs(container, schemaName); - } - - public QuerySnapshotDef getQuerySnapshotDef(@NotNull Container container, @NotNull String schemaName, @NotNull String snapshotName) - { - return QuerySnapshotCache.getQuerySnapshotDef(container, schemaName, snapshotName); - } - - public QueryDef insert(User user, QueryDef queryDef) - { - QueryDef def = Table.insert(user, getTableInfoQueryDef(), queryDef); - QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); - return def; - } - - public QueryDef update(User user, QueryDef queryDef) - { - QueryDef def = Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); - QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); - return def; - } - - public void renameQuery(User user, Container container, String schema, String oldName, String newName) - { - QueryDef queryDef = getQueryDef(container, schema, oldName, false); - if (queryDef != null) - { - queryDef.setName(newName); - QueryDef def = Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); - QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); - } - } - - public void renameSchema(User user, Container container, String oldSchema, String newSchema) - { - List queryDefs = getQueryDefs(container, oldSchema, false, false, false); - for (QueryDef queryDef : queryDefs) - { - queryDef.setSchema(newSchema); - Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); - } - QueryDefCache.uncache(ContainerManager.getForId(container.getId())); - } - - public void delete(QueryDef queryDef) - { - Table.delete(getTableInfoQueryDef(), queryDef.getQueryDefId()); - QueryDefCache.uncache(ContainerManager.getForId(queryDef.getContainerId())); - } - - public void delete(QuerySnapshotDef querySnapshotDef) - { - Table.delete(getTableInfoQuerySnapshotDef(), querySnapshotDef.getRowId()); - QuerySnapshotCache.uncache(querySnapshotDef); - if (querySnapshotDef.getQueryDefId() != null) - { - Table.delete(getTableInfoQueryDef(), querySnapshotDef.getQueryDefId()); - QueryDefCache.uncache(querySnapshotDef.lookupContainer()); - } - } - - public QuerySnapshotDef insert(User user, QueryDef queryDef, QuerySnapshotDef snapshotDef) - { - if (queryDef != null && snapshotDef.getQueryTableName() == null) - { - QueryDef def = insert(user, queryDef); - snapshotDef.setQueryDefId(def.getQueryDefId()); - } - snapshotDef = Table.insert(user, getTableInfoQuerySnapshotDef(), snapshotDef); - QuerySnapshotCache.uncache(snapshotDef); - return snapshotDef; - } - - public QuerySnapshotDef update(User user, QueryDef queryDef, QuerySnapshotDef snapshotDef) - { - if (queryDef != null && snapshotDef.getQueryTableName() == null) - update(user, queryDef); - snapshotDef = Table.update(user, getTableInfoQuerySnapshotDef(), snapshotDef, snapshotDef.getRowId()); - QuerySnapshotCache.uncache(snapshotDef); - return snapshotDef; - } - - // Does not use the cache... but only used at save time - public QuerySnapshotDef getQuerySnapshotDef(int id) - { - return new TableSelector(getTableInfoQuerySnapshotDef()).getObject(id, QuerySnapshotDef.class); - } - - public CstmView getCustomView(Container container, int id) - { - CstmView view = CustomViewCache.getCstmView(container, id); - _log.debug(view); - return view; - } - - public CstmView getCustomView(Container container, String entityId) - { - CstmView view = CustomViewCache.getCstmViewByEntityId(container, entityId); - _log.debug(view); - return view; - } - - /** - * Get all shared custom views that are applicable. - * If inheritable is true, custom views from parent and Shared container are included. - * - */ - public List getAllSharedCstmViews(Container container, String schemaName, String queryName, boolean inheritable) - { - return getAllCstmViews(container, schemaName, queryName, null, inheritable, true); - } - - /** - * Get all custom views that are applicable for this user including shared custom views. - * If inheritable is true, custom views from parent and Shared container are included. - * - * @param container The current container. - * @param schemaName The schema name or null for all schemas. - * @param queryName The query name or null for all queries in the schema. - * @param owner The owner or null for all views (shared or owned by someone.) - * @param inheritable If true, look up container hierarchy and in Shared project for custom views. - * @param sharedOnly If true, ignore the user parameter and only include shared custom views. - * @return List of custom views entities in priority order. - */ - public List getAllCstmViews(Container container, String schemaName, String queryName, @Nullable User owner, boolean inheritable, boolean sharedOnly) - { - List views = new ArrayList<>(); - - getCstmViewsInContainer(views, container, schemaName, queryName, owner, false, sharedOnly); - if (!container.isContainerFor(ContainerType.DataType.customQueryViews)) - { - getCstmViewsInContainer(views, container.getContainerFor(ContainerType.DataType.customQueryViews), schemaName, queryName, owner, false, sharedOnly); - } - - if (!inheritable) - return views; - - Container containerCur = container.getParent(); - while (containerCur != null && !containerCur.isRoot()) - { - getCstmViewsInContainer(views, containerCur, schemaName, queryName, owner, true, sharedOnly); - containerCur = containerCur.getParent(); - } - - // look in the shared project - getCstmViewsInContainer(views, ContainerManager.getSharedContainer(), schemaName, queryName, owner, true, sharedOnly); - - return views; - } - - private void getCstmViewsInContainer(List views, Container container, String schemaName, String queryName, @Nullable User user, boolean inheritable, boolean sharedOnly) - { - if (sharedOnly) - { - // Get only shared custom views - views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, true)); - } - else - { - if (user != null) - { - // Custom views owned by the user first, then add shared custom views - views.addAll(getCstmViews(container, schemaName, queryName, null, user, inheritable, false)); - views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, true)); - } - else - { - // Get all custom views regardless of owner - views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, false)); - } - } - } - - public List getCstmViews(Container container, @Nullable String schemaName, @Nullable String queryName, @Nullable String viewName, @Nullable User user, boolean inheritableOnly, boolean sharedOnly) - { - return CustomViewCache.getCstmViews(container, schemaName, queryName, viewName, user, inheritableOnly, sharedOnly); - } - - public CstmView update(User user, CstmView view) - { - CstmView cstmView = Table.update(user, getTableInfoCustomView(), view, view.getCustomViewId()); - CustomViewCache.uncache(ContainerManager.getForId(cstmView.getContainerId())); - - return cstmView; - } - - public CstmView insert(User user, CstmView view) - { - CstmView cstmView = Table.insert(user, getTableInfoCustomView(), view); - CustomViewCache.uncache(ContainerManager.getForId(cstmView.getContainerId())); - - return cstmView; - } - - public void delete(CstmView view) - { - Table.delete(getTableInfoCustomView(), view.getCustomViewId()); - CustomViewCache.uncache(ContainerManager.getForId(view.getContainerId())); - } - - @Nullable - public ExternalSchemaDef getExternalSchemaDef(Container c, int rowId) - { - return ExternalSchemaDefCache.getSchemaDef(c, rowId, ExternalSchemaDef.class); - } - - @NotNull - public List getExternalSchemaDefs(@Nullable Container container) - { - return ExternalSchemaDefCache.getSchemaDefs(container, ExternalSchemaDef.class); - } - - @Nullable - public ExternalSchemaDef getExternalSchemaDef(Container container, @Nullable String userSchemaName) - { - return ExternalSchemaDefCache.getSchemaDef(container, userSchemaName, ExternalSchemaDef.class); - } - - @Nullable - public LinkedSchemaDef getLinkedSchemaDef(Container c, int rowId) - { - return ExternalSchemaDefCache.getSchemaDef(c, rowId, LinkedSchemaDef.class); - } - - @NotNull - public List getLinkedSchemaDefs(@Nullable Container c) - { - return ExternalSchemaDefCache.getSchemaDefs(c, LinkedSchemaDef.class); - } - - @Nullable - public LinkedSchemaDef getLinkedSchemaDef(Container c, @Nullable String userSchemaName) - { - return ExternalSchemaDefCache.getSchemaDef(c, userSchemaName, LinkedSchemaDef.class); - } - - public void delete(@NotNull AbstractExternalSchemaDef def) - { - Container c = def.lookupContainer(); - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(getTableInfoExternalSchema().getColumn("ExternalSchemaId"), def.getExternalSchemaId()); - Table.delete(getTableInfoExternalSchema(), filter); - updateExternalSchemas(def.lookupContainer()); - } - - public LinkedSchemaDef insertLinkedSchema(User user, LinkedSchemaDef def) - { - LinkedSchemaDef newDef = Table.insert(user, getTableInfoExternalSchema(), def); - updateExternalSchemas(def.lookupContainer()); - return newDef; - } - - public void deleteLinkedSchema(Container container, String userSchemaName) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromString("UserSchemaName"), userSchemaName); - filter.addCondition(FieldKey.fromString("SchemaType"), AbstractExternalSchemaDef.SchemaType.linked); - Table.delete(getTableInfoExternalSchema(), filter); - updateExternalSchemas(container); - } - - // Uncaches and re-indexes all external schemas in a container. Called any time an external schema or linked schema - // changes in any way (insert/update/delete). - public void updateExternalSchemas(Container c) - { - QueryService.get().updateLastModified(); - if (null != c) - { - ExternalSchemaDefCache.uncache(c); - ExternalSchemaDocumentProvider.getInstance().enumerateDocuments(SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified), null); - } - } - - public void reloadAllExternalSchemas(Container c) - { - getExternalSchemaDefs(c).forEach(this::reloadExternalSchema); - } - - public void reloadExternalSchema(ExternalSchemaDef def) - { - ExternalSchema.uncache(def); - } - - public boolean canInherit(int flag) - { - return (flag & FLAG_INHERITABLE) != 0; - } - - public int setCanInherit(int flag, boolean f) - { - if (f) - { - return flag | FLAG_INHERITABLE; - } - else - { - return flag & ~FLAG_INHERITABLE; - } - } - - public boolean isHidden(int flag) - { - return (flag & FLAG_HIDDEN) != 0; - } - - public int setIsHidden(int flag, boolean f) - { - if (f) - { - return flag | FLAG_HIDDEN; - } - else - { - return flag & ~FLAG_HIDDEN; - } - } - - public boolean isSnapshot(int flag) - { - return (flag & FLAG_SNAPSHOT) != 0; - } - - public int setIsSnapshot(int flag, boolean f) - { - if (f) - { - return flag | FLAG_SNAPSHOT; - } - else - { - return flag & ~FLAG_SNAPSHOT; - } - } - - public String getDbSchemaName() - { - return SCHEMA_NAME; - } - - public DbSchema getDbSchema() - { - return DbSchema.get(SCHEMA_NAME, DbSchemaType.Module); - } - - public TableInfo getTableInfoQueryDef() - { - return getDbSchema().getTable("QueryDef"); - } - - public TableInfo getTableInfoQuerySnapshotDef() - { - return getDbSchema().getTable("QuerySnapshotDef"); - } - - public TableInfo getTableInfoCustomView() - { - return getDbSchema().getTable("CustomView"); - } - - public TableInfo getTableInfoExternalSchema() - { - return getDbSchema().getTable("ExternalSchema"); - } - - public TableInfo getTableInfoOlapDef() - { - return getDbSchema().getTable("OlapDef"); - } - - public void containerDeleted(Container c) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - Table.delete(getTableInfoQuerySnapshotDef(), filter); - QuerySnapshotCache.uncache(c); - Table.delete(getTableInfoCustomView(), filter); - CustomViewCache.uncache(c); - Table.delete(getTableInfoQueryDef(), filter); - QueryDefCache.uncache(c); - Table.delete(getTableInfoExternalSchema(), filter); - ExternalSchemaDefCache.uncache(c); - Table.delete(getTableInfoOlapDef(), filter); - } - - public void addQueryListener(QueryChangeListener listener) - { - QUERY_LISTENERS.add(listener); - } - - public void removeQueryListener(QueryChangeListener listener) - { - QUERY_LISTENERS.remove(listener); - } - - public void fireQueryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) - { - QueryService.get().updateLastModified(); - for (QueryChangeListener l : QUERY_LISTENERS) - l.queryCreated(user, container, scope, schema, queries); - } - - public void fireQueryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @Nullable String queryName, @NotNull QueryChangeListener.QueryProperty property, @NotNull Collection> changes) - { - QueryService.get().updateLastModified(); - assert checkChanges(property, changes); - for (QueryChangeListener l : QUERY_LISTENERS) - l.queryChanged(user, container, scope, schema, queryName, property, changes); - } - - // Checks all changes have the correct property and type. - private boolean checkChanges(QueryChangeListener.QueryProperty property, Collection> changes) - { - if (property == null) - { - _log.error("Null property not allowed."); - return false; - } - - boolean valid = true; - for (QueryPropertyChange change : changes) - { - if (change.getProperty() != property) - { - _log.error(String.format("Property '%s' doesn't match change property '%s'", property, change.getProperty())); - valid = false; - } - if (change.getOldValue() != null && !property.getPropertyClass().isInstance(change.getOldValue())) - { - _log.error(String.format("Old value '%s' isn't an instance of property '%s' class '%s'", change.getOldValue(), property, property.getPropertyClass())); - valid = false; - } - if (change.getNewValue() != null && !property.getPropertyClass().isInstance(change.getNewValue())) - { - _log.error(String.format("New value '%s' isn't an instance of property '%s' class '%s'", change.getNewValue(), property, property.getPropertyClass())); - valid = false; - } - } - return valid; - } - - public void fireQueryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries) - { - QueryService.get().updateLastModified(); - for (QueryChangeListener l : QUERY_LISTENERS) - l.queryDeleted(user, container, scope, schema, queries); - } - - public Collection getQueryDependents(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries) - { - ArrayList dependents = new ArrayList<>(); - for (QueryChangeListener l : QUERY_LISTENERS) - dependents.addAll(l.queryDependents(user, container, scope, schema, queries)); - return dependents; - } - - public void addCustomViewListener(CustomViewChangeListener listener) - { - VIEW_LISTENERS.add(listener); - } - - public void removeCustomViewListener(CustomViewChangeListener listener) - { - VIEW_LISTENERS.remove(listener); - } - - public void fireViewCreated(CustomView view) - { - QueryService.get().updateLastModified(); - for (CustomViewChangeListener l : VIEW_LISTENERS) - l.viewCreated(view); - } - - public void fireViewChanged(CustomView view) - { - QueryService.get().updateLastModified(); - for (CustomViewChangeListener l : VIEW_LISTENERS) - l.viewChanged(view); - } - - public void fireViewDeleted(CustomView view) - { - QueryService.get().updateLastModified(); - for (CustomViewChangeListener l : VIEW_LISTENERS) - l.viewDeleted(view); - } - - public Collection getViewDepedents(CustomView view) - { - ArrayList dependents = new ArrayList<>(); - for (CustomViewChangeListener l : VIEW_LISTENERS) - dependents.addAll(l.viewDependents(view)); - return dependents; - } - - static public final ContainerManager.ContainerListener CONTAINER_LISTENER = new ContainerManager.ContainerListener() - { - @Override - public void containerDeleted(Container c, User user) - { - QueryManager.get().containerDeleted(c); - } - }; - - public boolean validateQuery(SchemaKey schemaPath, String queryName, User user, Container container, @NotNull List errors, - @NotNull List warnings) - { - UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); - if (null == schema) - throw new IllegalArgumentException("Could not find the schema '" + schemaPath.toDisplayString() + "'!"); - - TableInfo table = schema.getTable(queryName); - if (null == table) - throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schemaPath.toDisplayString() + "'!"); - - return validateQuery(table, true, errors, warnings); - } - - public boolean validateQuery(TableInfo table, boolean testAllColumns, @NotNull List errors, - @NotNull List warnings) - { - errors.addAll(table.getWarnings()); - - Collection params = table.getNamedParameters(); - Map parameters = new HashMap<>(); - for (QueryService.ParameterDecl p : params) - { - if (!p.isRequired()) - continue; - parameters.put(p.getName(), null); - } - - TableSelector selector; - - // Note this check had been inverted for years, but was fixed in 14.1. Previously, testAllColumns == true meant - // the default column list was computed but discarded, and testAllColumns == false was completely broken - if (testAllColumns) - { - selector = new TableSelector(table); - } - else - { - List defVisCols = table.getDefaultVisibleColumns(); - Map colMap = QueryService.get().getColumns(table, defVisCols); - List cols = new ArrayList<>(colMap.values()); - - selector = new TableSelector(table, cols, null, null); - } - - // set forDisplay to mimic the behavior one would get in the UI - // try to execute with a rowcount of 0 (will throw SQLException to client if it fails) - selector.setForDisplay(true).setNamedParameters(parameters).setMaxRows(Table.NO_ROWS); - - //noinspection EmptyTryBlock,UnusedDeclaration - try (ResultSet rs = selector.getResultSet()) - { - } - catch (SQLException e) - { - errors.add(new QueryParseException(e.getMessage(), e, 0, 0)); - } - catch (BadSqlGrammarException e) - { - errors.add(new QueryParseException(e.getSQLException().getMessage(), e, 0, 0)); - } - - UserSchema schema = table.getUserSchema(); - if (schema != null) - { - QueryDefinition queryDef = schema.getQueryDef(table.getName()); - if (queryDef != null) - { - queryDef.validateQuery(schema, errors, warnings); - } - } - - OntologyService os = OntologyService.get(); - if (null != os) - { - for (var col : table.getColumns()) - { - String code = col.getPrincipalConceptCode(); - if (null != code) - { - Concept concept = os.resolveCode(code); - if (null == concept) - warnings.add(new QueryParseException("Concept not found: " + code, null, 0, 0)); - } - } - } - - return errors.isEmpty(); - } - - /** - * Experimental. The goal is to provide a more thorough validation of query metadata, including warnings of potentially - * invalid conditions, like autoincrement columns set userEditable=true. - */ - public boolean validateQueryMetadata(SchemaKey schemaPath, String queryName, User user, Container container, - @NotNull List errors, @NotNull List warnings) - { - Set columns = new HashSet<>(); - UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); - if (null == schema) - throw new IllegalArgumentException("Could not find the schema '" + schemaPath.getName() + "'!"); - - TableInfo table = schema.getTable(queryName); - if (null == table) - throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schemaPath.getName() + "'!"); - - if (table.isPublic() && table.getPublicSchemaName() != null && !schemaPath.toString().equalsIgnoreCase(table.getPublicSchemaName())) - warnings.add(new QueryParseWarning("(metadata) TableInfo.getPublicSchemaName() does not match: set to '" + table.getPublicSchemaName() + "', expected '" + schemaPath + "'", null, 0,0)); - - try - { - //validate foreign keys and other metadata warnings - columns.addAll(table.getColumns()); - columns.addAll(QueryService.get().getColumns(table, table.getDefaultVisibleColumns()).values()); - } - catch(QueryParseException e) - { - errors.add(e); - } - - for (ColumnInfo col : columns) - { - validateColumn(col, user, container, table, errors, warnings); - } - - return errors.isEmpty(); - } - - /** - * Experimental. See validateQueryMetadata() - */ - private boolean validateColumn(ColumnInfo col, User user, Container container, @Nullable TableInfo parentTable, - @NotNull List errors, @NotNull List warnings) - { - if(parentTable == null) - parentTable = col.getParentTable(); - - String publicSchema = col.getParentTable().getPublicSchemaName() != null ? col.getParentTable().getPublicSchemaName() : col.getParentTable().getSchema().toString(); - String publicQuery = col.getParentTable().getPublicName() != null ? col.getParentTable().getPublicName() : col.getParentTable().getName(); - String errorBase = "(metadata) for column '" + col.getFieldKey() + "' in " + publicSchema + "." + publicQuery + ": "; - - validateFk(col, user, container, parentTable, errors, warnings, errorBase); - - Set specialCols = new CaseInsensitiveHashSet(); - specialCols.add("LSID"); - specialCols.add("entityId"); - specialCols.add("container"); - specialCols.add("created"); - specialCols.add("createdby"); - specialCols.add("modified"); - specialCols.add("modifiedby"); - - if(specialCols.contains(col.getName())) - { - if(col.isUserEditable()) - warnings.add(new QueryParseWarning(errorBase + " column is user editable, which is not expected based on its name", null, 0,0)); - if(col.isShownInInsertView()) - warnings.add(new QueryParseWarning(errorBase + " column has shownInInsertView set to true, which is not expected based on its name", null, 0, 0)); - if(col.isShownInUpdateView()) - warnings.add(new QueryParseWarning(errorBase + " column has shownInUpdateView set to true, which is not expected based on its name", null, 0, 0)); - } - - if(col.isAutoIncrement() && col.isUserEditable()) - warnings.add(new QueryParseException(errorBase + " column is autoIncrement, but has userEditable set to true", null, 0, 0)); - if(col.isAutoIncrement() && col.isShownInInsertView()) - warnings.add(new QueryParseWarning(errorBase + " column is autoIncrement, but has shownInInsertView set to true", null, 0, 0)); - if(col.isAutoIncrement() && col.isShownInUpdateView()) - warnings.add(new QueryParseWarning(errorBase + " column is autoIncrement, but has shownInUpdateView set to true", null, 0, 0)); - - try - { - if (StringUtils.isNotBlank(col.getDisplayWidth()) && Integer.parseInt(col.getDisplayWidth()) > 200 && !"textarea".equalsIgnoreCase(col.getInputType())) - { - if (col.isUserEditable() && col.getJdbcType() != null && col.getJdbcType().getJavaClass() == String.class) - warnings.add(new QueryParseWarning(errorBase + " column has a displayWidth > 200, but does not use a textarea as the inputType", null, 0, 0)); - } - } - catch (NumberFormatException e) - { - warnings.add(new QueryParseWarning(errorBase + " column has invalid value for displayWidth: '" + col.getDisplayWidth() + "'", null, 0, 0)); - } - return errors.isEmpty(); - } - - /** - * Experimental. See validateQueryMetadata() - */ - private boolean validateFk(ColumnInfo col, User user, Container container, TableInfo parentTable, - @NotNull List errors, @NotNull List warnings, - String errorBase) - - { - //NOTE: this is the same code that writes JSON to the client - JSONObject o = JsonWriter.getLookupInfo(col, false); - if (o == null) - return true; - - boolean isPublic = o.getBoolean("isPublic"); - SchemaKey schemaPath = SchemaKey.fromString(o.optString("schemaName")); - String queryName = o.optString("queryName"); - if (queryName == null) - { - // Likely a lookup that targets something not exposed via a UserSchema. Bail out without further validation - return true; - } - String displayColumn = o.optString("displayColumn"); - String keyColumn = o.optString("keyColumn"); - String containerPath = o.optString("containerPath"); - - Container lookupContainer = containerPath == null ? container : ContainerManager.getForPath(containerPath); - if (lookupContainer == null) - { - warnings.add(new QueryParseWarning(errorBase + " Unable to find container" + containerPath, null, 0, 0)); - } - - //String publicSchema = col.getParentTable().getPublicSchemaName() != null ? col.getParentTable().getPublicSchemaName() : col.getParentTable().getSchema().toString(); - //String publicQuery = col.getParentTable().getPublicName() != null ? col.getParentTable().getPublicName() : col.getParentTable().getName(); - if (col.getFk() == null) - return errors.isEmpty(); - - if (!isPublic) - { - warnings.add(new QueryParseWarning(errorBase + " has a lookup to a non-public table: " + (schemaPath == null ? "" : schemaPath.toDisplayString()) + "." + queryName, null, 0, 0)); - return errors.isEmpty(); - } - - UserSchema userSchema = QueryService.get().getUserSchema(user, lookupContainer, schemaPath); - if (userSchema == null) - { - warnings.add(new QueryParseWarning(errorBase + " unable to find the user schema: " + schemaPath.toDisplayString(), null, 0, 0)); - return errors.isEmpty(); - } - - TableInfo fkTable = userSchema.getTable(queryName); - if(fkTable == null) - { - warnings.add(new QueryParseWarning(errorBase + " has a lookup to a table that does not exist: " + schemaPath.toDisplayString() + "." + queryName, null, 0, 0)); - return errors.isEmpty(); - } - - //a FK can have a table non-visible to the client, so long as public is set to false - if (fkTable.isPublic()){ - String fkt = schemaPath.toDisplayString() + "." + queryName; - - QueryManager.get().validateQuery(schemaPath, queryName, user, lookupContainer, errors, warnings); - if (displayColumn != null) - { - FieldKey displayFieldKey = FieldKey.fromString(displayColumn); - Map cols = QueryService.get().getColumns(fkTable, Collections.singleton(displayFieldKey)); - if (!cols.containsKey(displayFieldKey)) - { - warnings.add(new QueryParseWarning(errorBase + " reports a foreign key with displayColumn of " + displayColumn + " in the table " + schemaPath.toDisplayString() + "." + queryName + ", but the column does not exist", null, 0, 0)); - } - else - { - ColumnInfo ci = cols.get(displayFieldKey); - if (!displayColumn.equals(ci.getFieldKey().toString())) - { - warnings.add(new QueryParseWarning(errorBase + ", the lookup to " + schemaPath.toDisplayString() + "." + queryName + "' did not match the expected case, which was '" + ci.getFieldKey().toString() + "'. Actual: '" + displayColumn + "'", null, 0, 0)); - } - } - } - - if (keyColumn != null) - { - FieldKey keyFieldKey = FieldKey.fromString(keyColumn); - Map cols = QueryService.get().getColumns(fkTable, Collections.singleton(keyFieldKey)); - if (!cols.containsKey(keyFieldKey)) - { - warnings.add(new QueryParseException(errorBase + " reports a foreign key with keyColumn of " + keyColumn + " in the table " + schemaPath.toDisplayString() + "." + queryName + ", but the column does not exist", null, 0, 0)); - } - else - { - ColumnInfo ci = cols.get(keyFieldKey); - if (!keyColumn.equals(ci.getFieldKey().toString())) - { - warnings.add(new QueryParseWarning(errorBase + ", the lookup to " + schemaPath.toDisplayString() + "." + queryName + "' did not match the expected case, which was '" + ci.getFieldKey().toString() + "'. Actual: '" + keyColumn + "'", null, 0, 0)); - } - } - } - else - { - warnings.add(new QueryParseWarning(errorBase + ", there is a lookup where the keyColumn is blank", null, 0, 0)); - } - } - - return errors.isEmpty(); - } - - /** - * Experimental. The goal is to provide a more thorough validation of saved views, including errors like invalid - * column names or case errors (which cause problems for case-sensitive js) - */ - public boolean validateQueryViews(SchemaKey schemaPath, String queryName, User user, Container container, - @NotNull List errors, @NotNull List warnings) throws QueryParseException - { - UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); - if (null == schema) - throw new IllegalArgumentException("Could not find the schema '" + schemaPath.getName() + "'!"); - - TableInfo table = schema.getTable(queryName); - if (null == table) - throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schema.getSchemaName() + "'!"); - - //validate views - try - { - List views = QueryService.get().getCustomViews(user, container, null, schema.getSchemaName(), queryName, true); - for (CustomView v : views) - { - validateViewColumns(user, container, v, "columns", v.getColumns(), table, errors, warnings); - - if (!StringUtils.isEmpty(v.getFilterAndSort())) - { - try - { - CustomViewInfo.FilterAndSort fs = CustomViewInfo.FilterAndSort.fromString(v.getFilterAndSort()); - List filterCols = new ArrayList<>(); - for (FilterInfo f : fs.getFilter()) - { - filterCols.add(f.getField()); - } - validateViewColumns(user, container, v, "filter", filterCols, table, errors, warnings); - - List sortCols = new ArrayList<>(); - for (Sort.SortField f : fs.getSort()) - { - sortCols.add(f.getFieldKey()); - } - validateViewColumns(user, container, v, "sort", sortCols, table, errors, warnings); - - } - catch (URISyntaxException e) - { - warnings.add(new QueryParseWarning("unable to process the filter/sort section of view: " + v.getName(), null, 0, 0)); - } - } - } - } - catch (NotFoundException e) - { - errors.add(new QueryParseException("Cannot get views: ", e, 0, 0)); - } - - - return errors.isEmpty(); - } - - private void validateViewColumns(User user, Container container, CustomView v, String identifier, List viewCols, TableInfo sourceTable, - @NotNull List errors, @NotNull List warnings) throws QueryParseException - { - //verify columns match, accounting for case - Map colMap = QueryService.get().getColumns(sourceTable, viewCols); - - for (FieldKey f : viewCols) - { - boolean found = false; - boolean matchCase = false; - FieldKey fk = null; - ColumnInfo c = colMap.get(f); - if(c != null) - { - found = true; - fk = c.getFieldKey(); - if(c instanceof AliasedColumn) - fk = ((AliasedColumn)c).getColumn().getFieldKey(); - - if(fk.toString().equals(f.toString())) - { - matchCase = true; - } - } - - if (!found){ - warnings.add(new QueryParseWarning("In the saved view '" + (v.getName() == null ? "default" : v.getName()) + "', in the " + identifier + " section, the column '" + f.toString() + "' in " + v.getSchemaName() + "." + v.getQueryName() + " could not be matched to a column", null, 0, 0)); - continue; - } - - if (!matchCase){ - warnings.add(new QueryParseWarning("In the saved view '" + (v.getName() == null ? "default" : v.getName()) + "', in the " + identifier + " section, the column '" + f + "' in " + v.getSchemaName() + "." + v.getQueryName() + "' did not match the expected case, which was '" + fk + "'", null, 0, 0)); - } - - //queryErrors.addAll(validateColumn(c, user, container)); - } - } - - public static void registerUsageMetrics(String moduleName) - { - UsageMetricsService svc = UsageMetricsService.get(); - if (null != svc) - { - svc.registerUsageMetrics(moduleName, () -> { - Bag bag = DbScope.getDbScopes().stream() - .filter(scope -> !scope.isLabKeyScope()).map(DbScope::getDatabaseProductName) - .collect(Collectors.toCollection(HashBag::new)); - - Map statsMap = bag.uniqueSet().stream() - .collect(Collectors.toMap(Function.identity(), bag::getCount)); - - return Map.of("externalDatasources", statsMap, - "customViewCounts", - Map.of( - "DataClasses", getSchemaCustomViewCounts("exp.data"), - "SampleTypes", getSchemaCustomViewCounts("samples"), - "Assays", getSchemaCustomViewCounts("assay"), - "Inventory", getSchemaCustomViewCounts("inventory") - ), - "customViewWithLineageColumn", getLineageCustomViewMetrics(), - "queryDefWithCalculatedFieldsCounts", getCalculatedFieldsCountsMetric() - ); - }); - } - } - - private static Map getCalculatedFieldsCountsMetric() - { - DbSchema dbSchema = CoreSchema.getInstance().getSchema(); - return new SqlSelector(dbSchema, - new SQLFragment("SELECT \"schema\", COUNT(*) AS count FROM (\n" + - " SELECT CASE WHEN \"schema\" LIKE 'assay.%' THEN 'assay' ELSE \"schema\" END AS \"schema\" FROM query.querydef WHERE metadata LIKE '%(), (x, m) -> { - x.put(m.get("schema").toString(), m.get("count")); - return x; - }); - } - - private static Map getSchemaCustomViewCounts(String schema) - { - DbSchema dbSchema = DbSchema.get("query"); - TableInfo customView = dbSchema.getTable("customview"); - var schemaField = customView.getColumn("schema").getSelectIdentifier(); - SQLFragment schemaClause; - if (schema.equalsIgnoreCase("assay")) - schemaClause = new SQLFragment("C.").appendIdentifier(schemaField).append(" LIKE 'assay.%'"); - else - schemaClause = new SQLFragment("C.").appendIdentifier(schemaField).append(" = ").appendValue(schema); - return Map.of( - "defaultOverrides", new SqlSelector(dbSchema, - new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags < 2 AND C.name IS NULL")).getObject(Long.class), // possibly inheritable, no hidden, not snapshot - "inheritable", new SqlSelector(dbSchema, - new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags = 1")).getObject(Long.class), // inheritable, not hidden, not snapshot - "namedViews", new SqlSelector(dbSchema, - new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags < 2 AND C.name IS NOT NULL")).getObject(Long.class), // possibly inheritable, no hidden, not snapshot - "shared", new SqlSelector(dbSchema, - new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.customviewowner IS NULL")).getObject(Long.class), - "identifyingFieldsViews", new SqlSelector(dbSchema, - new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.name = '~~identifyingfields~~'")).getObject(Long.class) - ); - } - - - private static Long percentile(double percentile, List sortedCounts) { - if (percentile <= 0.01) - return sortedCounts.get(0); - if (percentile >= 99.99) - return sortedCounts.get(sortedCounts.size() - 1); - return sortedCounts.get((int) Math.round(percentile / 100.0 * (sortedCounts.size() - 1))); - } - - /** - * customViewsCountWithLineageCol: total number of non-hidden saved custom views that has at least one input/output/ancestor column - * customViewsCountWithAncestorCol: total number of non-hidden saved custom views that has at least one ancestor column - * totalLineageColumnsInAllViews: total number of input/output/ancestor columns defined for all saved non-hidden custom views - * totalAncestorColumnsInAllViews: total number of ancestor columns defined for all saved non-hidden custom views - * lineageColumnsCountMin: the minimum count of input/output/ancestor columns in any view with such column - * lineageColumnsCount25: the 25 percentile count of input/output/ancestor columns in all views with such column - * lineageColumnsCount50: the 50 percentile / median count of input/output/ancestor columns in all views with such column - * lineageColumnsCount75: the 75 percentile count of input/output/ancestor columns in all views with such column - * lineageColumnsCountMax: the maximum count of input/output/ancestor columns in any view with such column - * lineageColumnsCountAvg: the average count of input/output/ancestor columns in any view with such column - * ancestorColumnsCountMin: the minimum count of ancestor columns in any view with ancestor columns - * ancestorColumnsCount25: the 25 percentile count of ancestor columns in all views with ancestor columns - * ancestorColumnsCount50: the 50 percentile / median count of ancestor columns in all views with ancestor columns - * ancestorColumnsCount75: the 75 percentile count of ancestor columns in all views with ancestor columns - * ancestorColumnsCountMax: the maximum count of ancestor columns in any view with ancestor columns - * ancestorColumnsCountAvg: the average count of ancestor columns in any view with ancestor columns - */ - private static Map getLineageCustomViewMetrics() - { - List ancestorColCounts = new ArrayList<>(); - List lineageColCounts = new ArrayList<>(); - final String ANCESTOR_PREFIX = "ancestors/"; - final String INPUT_PREFIX = "inputs/"; - final String OUTPUT_PREFIX = "outputs/"; - - Map metrics = new HashMap<>(); - - DbSchema schema = DbSchema.get("query", DbSchemaType.Module); - SqlDialect sqlDialect = schema.getSqlDialect(); - SQLFragment sql = new SQLFragment() - .append("SELECT columns FROM query.customview WHERE flags < 2 AND (columns LIKE ? OR columns LIKE ? OR columns LIKE ?)") - .add("%" + sqlDialect.encodeLikeOpSearchString("Ancestors%2F") + "%") - .add("%" + sqlDialect.encodeLikeOpSearchString("Inputs%2F") + "%") - .add("%" + sqlDialect.encodeLikeOpSearchString("Outputs%2F") + "%"); - List viewsColumnStrs = new SqlSelector(schema, sql).getArrayList(String.class); - - for (String columnStr : viewsColumnStrs) - { - long lineageColCount = 0L; - long ancestorColCount = 0L; - for (Map.Entry> entry : CustomViewInfo.decodeProperties(columnStr)) - { - String fieldName = entry.getKey().toString().toLowerCase(); - if (fieldName.startsWith(ANCESTOR_PREFIX)) - { - ancestorColCount++; - lineageColCount++; - } - else if (fieldName.startsWith(INPUT_PREFIX) || fieldName.startsWith(OUTPUT_PREFIX)) - { - lineageColCount++; - } - } - if (ancestorColCount > 0) - ancestorColCounts.add(ancestorColCount); - if (lineageColCount > 0) - lineageColCounts.add(lineageColCount); - } - - Collections.sort(lineageColCounts); - int lineageViewCount = lineageColCounts.size(); - metrics.put("customViewsCountWithLineageColumnsCount", lineageViewCount); - if (lineageViewCount != 0) - { - long totalLineageCols = lineageColCounts.stream().mapToLong(Long::longValue).sum(); - metrics.put("totalLineageColumnsInAllViews", totalLineageCols); - metrics.put("lineageColumnsCountMin", percentile(0, lineageColCounts)); - metrics.put("lineageColumnsCount25", percentile(25, lineageColCounts)); - metrics.put("lineageColumnsCount50", percentile(50, lineageColCounts)); - metrics.put("lineageColumnsCount75", percentile(75, lineageColCounts)); - metrics.put("lineageColumnsCountMax", percentile(100, lineageColCounts)); - metrics.put("lineageColumnsCountAvg", Math.round((float) totalLineageCols / lineageViewCount)); - } - - Collections.sort(ancestorColCounts); - int ancestorViewCount = ancestorColCounts.size(); - metrics.put("customViewsWithAncestorColumnsCounts", ancestorViewCount); - if (ancestorViewCount != 0) - { - long totalAncestorCols = ancestorColCounts.stream().mapToLong(Long::longValue).sum(); - metrics.put("totalAncestorColumnsInAllViews", totalAncestorCols); - metrics.put("ancestorColumnsCountMin", percentile(0, ancestorColCounts)); - metrics.put("ancestorColumnsCount25", percentile(25, ancestorColCounts)); - metrics.put("ancestorColumnsCount50", percentile(50, ancestorColCounts)); - metrics.put("ancestorColumnsCount75", percentile(75, ancestorColCounts)); - metrics.put("ancestorColumnsCountMax", percentile(100, ancestorColCounts)); - metrics.put("ancestorColumnsCountAvg", Math.round((float) totalAncestorCols / ancestorViewCount)); - } - - return metrics; - } - - -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.query.persist; + +import org.apache.commons.collections4.Bag; +import org.apache.commons.collections4.bag.HashBag; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerType; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.FilterInfo; +import org.labkey.api.data.JsonWriter; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.ontology.Concept; +import org.labkey.api.ontology.OntologyService; +import org.labkey.api.query.AliasedColumn; +import org.labkey.api.query.CustomView; +import org.labkey.api.query.CustomViewChangeListener; +import org.labkey.api.query.CustomViewInfo; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryChangeListener.QueryPropertyChange; +import org.labkey.api.query.QueryDefinition; +import org.labkey.api.query.QueryParseException; +import org.labkey.api.query.QueryParseWarning; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.view.NotFoundException; +import org.labkey.query.ExternalSchema; +import org.labkey.query.ExternalSchemaDocumentProvider; +import org.springframework.jdbc.BadSqlGrammarException; + +import java.net.URISyntaxException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Function; +import java.util.stream.Collectors; + + +public class QueryManager +{ + private static final Logger _log = LogManager.getLogger(QueryManager.class); + private static final QueryManager instance = new QueryManager(); + private static final String SCHEMA_NAME = "query"; + private static final List QUERY_LISTENERS = new CopyOnWriteArrayList<>(); + private static final List VIEW_LISTENERS = new CopyOnWriteArrayList<>(); + + public static final int FLAG_INHERITABLE = 0x01; + public static final int FLAG_HIDDEN = 0x02; + public static final int FLAG_SNAPSHOT = 0x04; + + public static QueryManager get() + { + return instance; + } + + /** + * @param customQuery whether to look for custom queries or modified metadata on built-in tables + */ + public QueryDef getQueryDef(Container container, String schema, String name, boolean customQuery) + { + return QueryDefCache.getQueryDef(container, schema, name, customQuery); + } + + /** + * @param customQuery whether to look for custom queries or modified metadata on built-in tables + */ + public List getQueryDefs(Container container, @Nullable String schema, boolean inheritableOnly, boolean includeSnapshots, boolean customQuery) + { + return QueryDefCache.getQueryDefs(container, schema, inheritableOnly, includeSnapshots, customQuery); + } + + public Collection getQuerySnapshots(@Nullable Container container, @Nullable String schemaName) + { + return QuerySnapshotCache.getQuerySnapshotDefs(container, schemaName); + } + + public QuerySnapshotDef getQuerySnapshotDef(@NotNull Container container, @NotNull String schemaName, @NotNull String snapshotName) + { + return QuerySnapshotCache.getQuerySnapshotDef(container, schemaName, snapshotName); + } + + public QueryDef insert(User user, QueryDef queryDef) + { + QueryDef def = Table.insert(user, getTableInfoQueryDef(), queryDef); + QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); + return def; + } + + public QueryDef update(User user, QueryDef queryDef) + { + QueryDef def = Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); + QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); + return def; + } + + public void renameQuery(User user, Container container, String schema, String oldName, String newName) + { + QueryDef queryDef = getQueryDef(container, schema, oldName, false); + if (queryDef != null) + { + queryDef.setName(newName); + QueryDef def = Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); + QueryDefCache.uncache(ContainerManager.getForId(def.getContainerId())); + } + } + + public void renameSchema(User user, Container container, String oldSchema, String newSchema) + { + List queryDefs = getQueryDefs(container, oldSchema, false, false, false); + for (QueryDef queryDef : queryDefs) + { + queryDef.setSchema(newSchema); + Table.update(user, getTableInfoQueryDef(), queryDef, queryDef.getQueryDefId()); + } + QueryDefCache.uncache(ContainerManager.getForId(container.getId())); + } + + public void delete(QueryDef queryDef) + { + Table.delete(getTableInfoQueryDef(), queryDef.getQueryDefId()); + QueryDefCache.uncache(ContainerManager.getForId(queryDef.getContainerId())); + } + + public void delete(QuerySnapshotDef querySnapshotDef) + { + Table.delete(getTableInfoQuerySnapshotDef(), querySnapshotDef.getRowId()); + QuerySnapshotCache.uncache(querySnapshotDef); + if (querySnapshotDef.getQueryDefId() != null) + { + Table.delete(getTableInfoQueryDef(), querySnapshotDef.getQueryDefId()); + QueryDefCache.uncache(querySnapshotDef.lookupContainer()); + } + } + + public QuerySnapshotDef insert(User user, QueryDef queryDef, QuerySnapshotDef snapshotDef) + { + if (queryDef != null && snapshotDef.getQueryTableName() == null) + { + QueryDef def = insert(user, queryDef); + snapshotDef.setQueryDefId(def.getQueryDefId()); + } + snapshotDef = Table.insert(user, getTableInfoQuerySnapshotDef(), snapshotDef); + QuerySnapshotCache.uncache(snapshotDef); + return snapshotDef; + } + + public QuerySnapshotDef update(User user, QueryDef queryDef, QuerySnapshotDef snapshotDef) + { + if (queryDef != null && snapshotDef.getQueryTableName() == null) + update(user, queryDef); + snapshotDef = Table.update(user, getTableInfoQuerySnapshotDef(), snapshotDef, snapshotDef.getRowId()); + QuerySnapshotCache.uncache(snapshotDef); + return snapshotDef; + } + + // Does not use the cache... but only used at save time + public QuerySnapshotDef getQuerySnapshotDef(int id) + { + return new TableSelector(getTableInfoQuerySnapshotDef()).getObject(id, QuerySnapshotDef.class); + } + + public CstmView getCustomView(Container container, int id) + { + CstmView view = CustomViewCache.getCstmView(container, id); + _log.debug(view); + return view; + } + + public CstmView getCustomView(Container container, String entityId) + { + CstmView view = CustomViewCache.getCstmViewByEntityId(container, entityId); + _log.debug(view); + return view; + } + + /** + * Get all shared custom views that are applicable. + * If inheritable is true, custom views from parent and Shared container are included. + * + */ + public List getAllSharedCstmViews(Container container, String schemaName, String queryName, boolean inheritable) + { + return getAllCstmViews(container, schemaName, queryName, null, inheritable, true); + } + + /** + * Get all custom views that are applicable for this user including shared custom views. + * If inheritable is true, custom views from parent and Shared container are included. + * + * @param container The current container. + * @param schemaName The schema name or null for all schemas. + * @param queryName The query name or null for all queries in the schema. + * @param owner The owner or null for all views (shared or owned by someone.) + * @param inheritable If true, look up container hierarchy and in Shared project for custom views. + * @param sharedOnly If true, ignore the user parameter and only include shared custom views. + * @return List of custom views entities in priority order. + */ + public List getAllCstmViews(Container container, String schemaName, String queryName, @Nullable User owner, boolean inheritable, boolean sharedOnly) + { + List views = new ArrayList<>(); + + getCstmViewsInContainer(views, container, schemaName, queryName, owner, false, sharedOnly); + if (!container.isContainerFor(ContainerType.DataType.customQueryViews)) + { + getCstmViewsInContainer(views, container.getContainerFor(ContainerType.DataType.customQueryViews), schemaName, queryName, owner, false, sharedOnly); + } + + if (!inheritable) + return views; + + Container containerCur = container.getParent(); + while (containerCur != null && !containerCur.isRoot()) + { + getCstmViewsInContainer(views, containerCur, schemaName, queryName, owner, true, sharedOnly); + containerCur = containerCur.getParent(); + } + + // look in the shared project + getCstmViewsInContainer(views, ContainerManager.getSharedContainer(), schemaName, queryName, owner, true, sharedOnly); + + return views; + } + + private void getCstmViewsInContainer(List views, Container container, String schemaName, String queryName, @Nullable User user, boolean inheritable, boolean sharedOnly) + { + if (sharedOnly) + { + // Get only shared custom views + views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, true)); + } + else + { + if (user != null) + { + // Custom views owned by the user first, then add shared custom views + views.addAll(getCstmViews(container, schemaName, queryName, null, user, inheritable, false)); + views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, true)); + } + else + { + // Get all custom views regardless of owner + views.addAll(getCstmViews(container, schemaName, queryName, null, null, inheritable, false)); + } + } + } + + public List getCstmViews(Container container, @Nullable String schemaName, @Nullable String queryName, @Nullable String viewName, @Nullable User user, boolean inheritableOnly, boolean sharedOnly) + { + return CustomViewCache.getCstmViews(container, schemaName, queryName, viewName, user, inheritableOnly, sharedOnly); + } + + public CstmView update(User user, CstmView view) + { + CstmView cstmView = Table.update(user, getTableInfoCustomView(), view, view.getCustomViewId()); + CustomViewCache.uncache(ContainerManager.getForId(cstmView.getContainerId())); + + return cstmView; + } + + public CstmView insert(User user, CstmView view) + { + CstmView cstmView = Table.insert(user, getTableInfoCustomView(), view); + CustomViewCache.uncache(ContainerManager.getForId(cstmView.getContainerId())); + + return cstmView; + } + + public void delete(CstmView view) + { + Table.delete(getTableInfoCustomView(), view.getCustomViewId()); + CustomViewCache.uncache(ContainerManager.getForId(view.getContainerId())); + } + + @Nullable + public ExternalSchemaDef getExternalSchemaDef(Container c, int rowId) + { + return ExternalSchemaDefCache.getSchemaDef(c, rowId, ExternalSchemaDef.class); + } + + @NotNull + public List getExternalSchemaDefs(@Nullable Container container) + { + return ExternalSchemaDefCache.getSchemaDefs(container, ExternalSchemaDef.class); + } + + @Nullable + public ExternalSchemaDef getExternalSchemaDef(Container container, @Nullable String userSchemaName) + { + return ExternalSchemaDefCache.getSchemaDef(container, userSchemaName, ExternalSchemaDef.class); + } + + @Nullable + public LinkedSchemaDef getLinkedSchemaDef(Container c, int rowId) + { + return ExternalSchemaDefCache.getSchemaDef(c, rowId, LinkedSchemaDef.class); + } + + @NotNull + public List getLinkedSchemaDefs(@Nullable Container c) + { + return ExternalSchemaDefCache.getSchemaDefs(c, LinkedSchemaDef.class); + } + + @Nullable + public LinkedSchemaDef getLinkedSchemaDef(Container c, @Nullable String userSchemaName) + { + return ExternalSchemaDefCache.getSchemaDef(c, userSchemaName, LinkedSchemaDef.class); + } + + public void delete(@NotNull AbstractExternalSchemaDef def) + { + Container c = def.lookupContainer(); + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(getTableInfoExternalSchema().getColumn("ExternalSchemaId"), def.getExternalSchemaId()); + Table.delete(getTableInfoExternalSchema(), filter); + updateExternalSchemas(def.lookupContainer()); + } + + public LinkedSchemaDef insertLinkedSchema(User user, LinkedSchemaDef def) + { + LinkedSchemaDef newDef = Table.insert(user, getTableInfoExternalSchema(), def); + updateExternalSchemas(def.lookupContainer()); + return newDef; + } + + public void deleteLinkedSchema(Container container, String userSchemaName) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromString("UserSchemaName"), userSchemaName); + filter.addCondition(FieldKey.fromString("SchemaType"), AbstractExternalSchemaDef.SchemaType.linked); + Table.delete(getTableInfoExternalSchema(), filter); + updateExternalSchemas(container); + } + + // Uncaches and re-indexes all external schemas in a container. Called any time an external schema or linked schema + // changes in any way (insert/update/delete). + public void updateExternalSchemas(Container c) + { + QueryService.get().updateLastModified(); + if (null != c) + { + ExternalSchemaDefCache.uncache(c); + ExternalSchemaDocumentProvider.getInstance().enumerateDocuments(SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified), null); + } + } + + public void reloadAllExternalSchemas(Container c) + { + getExternalSchemaDefs(c).forEach(this::reloadExternalSchema); + } + + public void reloadExternalSchema(ExternalSchemaDef def) + { + ExternalSchema.uncache(def); + } + + public boolean canInherit(int flag) + { + return (flag & FLAG_INHERITABLE) != 0; + } + + public int setCanInherit(int flag, boolean f) + { + if (f) + { + return flag | FLAG_INHERITABLE; + } + else + { + return flag & ~FLAG_INHERITABLE; + } + } + + public boolean isHidden(int flag) + { + return (flag & FLAG_HIDDEN) != 0; + } + + public int setIsHidden(int flag, boolean f) + { + if (f) + { + return flag | FLAG_HIDDEN; + } + else + { + return flag & ~FLAG_HIDDEN; + } + } + + public boolean isSnapshot(int flag) + { + return (flag & FLAG_SNAPSHOT) != 0; + } + + public int setIsSnapshot(int flag, boolean f) + { + if (f) + { + return flag | FLAG_SNAPSHOT; + } + else + { + return flag & ~FLAG_SNAPSHOT; + } + } + + public String getDbSchemaName() + { + return SCHEMA_NAME; + } + + public DbSchema getDbSchema() + { + return DbSchema.get(SCHEMA_NAME, DbSchemaType.Module); + } + + public TableInfo getTableInfoQueryDef() + { + return getDbSchema().getTable("QueryDef"); + } + + public TableInfo getTableInfoQuerySnapshotDef() + { + return getDbSchema().getTable("QuerySnapshotDef"); + } + + public TableInfo getTableInfoCustomView() + { + return getDbSchema().getTable("CustomView"); + } + + public TableInfo getTableInfoExternalSchema() + { + return getDbSchema().getTable("ExternalSchema"); + } + + public TableInfo getTableInfoOlapDef() + { + return getDbSchema().getTable("OlapDef"); + } + + public void containerDeleted(Container c) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + Table.delete(getTableInfoQuerySnapshotDef(), filter); + QuerySnapshotCache.uncache(c); + Table.delete(getTableInfoCustomView(), filter); + CustomViewCache.uncache(c); + Table.delete(getTableInfoQueryDef(), filter); + QueryDefCache.uncache(c); + Table.delete(getTableInfoExternalSchema(), filter); + ExternalSchemaDefCache.uncache(c); + Table.delete(getTableInfoOlapDef(), filter); + } + + public void addQueryListener(QueryChangeListener listener) + { + QUERY_LISTENERS.add(listener); + } + + public void removeQueryListener(QueryChangeListener listener) + { + QUERY_LISTENERS.remove(listener); + } + + public void fireQueryCreated(User user, Container container, ContainerFilter scope, SchemaKey schema, @NotNull Collection queries) + { + QueryService.get().updateLastModified(); + for (QueryChangeListener l : QUERY_LISTENERS) + l.queryCreated(user, container, scope, schema, queries); + } + + public void fireQueryChanged(User user, Container container, ContainerFilter scope, SchemaKey schema, @Nullable String queryName, @NotNull QueryChangeListener.QueryProperty property, @NotNull Collection> changes) + { + QueryService.get().updateLastModified(); + assert checkChanges(property, changes); + for (QueryChangeListener l : QUERY_LISTENERS) + l.queryChanged(user, container, scope, schema, queryName, property, changes); + } + + // Checks all changes have the correct property and type. + private boolean checkChanges(QueryChangeListener.QueryProperty property, Collection> changes) + { + if (property == null) + { + _log.error("Null property not allowed."); + return false; + } + + boolean valid = true; + for (QueryPropertyChange change : changes) + { + if (change.getProperty() != property) + { + _log.error(String.format("Property '%s' doesn't match change property '%s'", property, change.getProperty())); + valid = false; + } + if (change.getOldValue() != null && !property.getPropertyClass().isInstance(change.getOldValue())) + { + _log.error(String.format("Old value '%s' isn't an instance of property '%s' class '%s'", change.getOldValue(), property, property.getPropertyClass())); + valid = false; + } + if (change.getNewValue() != null && !property.getPropertyClass().isInstance(change.getNewValue())) + { + _log.error(String.format("New value '%s' isn't an instance of property '%s' class '%s'", change.getNewValue(), property, property.getPropertyClass())); + valid = false; + } + } + return valid; + } + + public void fireQueryDeleted(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries) + { + QueryService.get().updateLastModified(); + for (QueryChangeListener l : QUERY_LISTENERS) + l.queryDeleted(user, container, scope, schema, queries); + } + + public Collection getQueryDependents(User user, Container container, ContainerFilter scope, SchemaKey schema, Collection queries) + { + ArrayList dependents = new ArrayList<>(); + for (QueryChangeListener l : QUERY_LISTENERS) + dependents.addAll(l.queryDependents(user, container, scope, schema, queries)); + return dependents; + } + + public void addCustomViewListener(CustomViewChangeListener listener) + { + VIEW_LISTENERS.add(listener); + } + + public void removeCustomViewListener(CustomViewChangeListener listener) + { + VIEW_LISTENERS.remove(listener); + } + + public void fireViewCreated(CustomView view) + { + QueryService.get().updateLastModified(); + for (CustomViewChangeListener l : VIEW_LISTENERS) + l.viewCreated(view); + } + + public void fireViewChanged(CustomView view) + { + QueryService.get().updateLastModified(); + for (CustomViewChangeListener l : VIEW_LISTENERS) + l.viewChanged(view); + } + + public void fireViewDeleted(CustomView view) + { + QueryService.get().updateLastModified(); + for (CustomViewChangeListener l : VIEW_LISTENERS) + l.viewDeleted(view); + } + + public Collection getViewDepedents(CustomView view) + { + ArrayList dependents = new ArrayList<>(); + for (CustomViewChangeListener l : VIEW_LISTENERS) + dependents.addAll(l.viewDependents(view)); + return dependents; + } + + static public final ContainerManager.ContainerListener CONTAINER_LISTENER = new ContainerManager.ContainerListener() + { + @Override + public void containerDeleted(Container c, User user) + { + QueryManager.get().containerDeleted(c); + } + }; + + public boolean validateQuery(SchemaKey schemaPath, String queryName, User user, Container container, @NotNull List errors, + @NotNull List warnings) + { + UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); + if (null == schema) + throw new IllegalArgumentException("Could not find the schema '" + schemaPath.toDisplayString() + "'!"); + + TableInfo table = schema.getTable(queryName); + if (null == table) + throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schemaPath.toDisplayString() + "'!"); + + return validateQuery(table, true, errors, warnings); + } + + public boolean validateQuery(TableInfo table, boolean testAllColumns, @NotNull List errors, + @NotNull List warnings) + { + errors.addAll(table.getWarnings()); + + Collection params = table.getNamedParameters(); + Map parameters = new HashMap<>(); + for (QueryService.ParameterDecl p : params) + { + if (!p.isRequired()) + continue; + parameters.put(p.getName(), null); + } + + TableSelector selector; + + // Note this check had been inverted for years, but was fixed in 14.1. Previously, testAllColumns == true meant + // the default column list was computed but discarded, and testAllColumns == false was completely broken + if (testAllColumns) + { + selector = new TableSelector(table); + } + else + { + List defVisCols = table.getDefaultVisibleColumns(); + Map colMap = QueryService.get().getColumns(table, defVisCols); + List cols = new ArrayList<>(colMap.values()); + + selector = new TableSelector(table, cols, null, null); + } + + // set forDisplay to mimic the behavior one would get in the UI + // try to execute with a rowcount of 0 (will throw SQLException to client if it fails) + selector.setForDisplay(true).setNamedParameters(parameters).setMaxRows(Table.NO_ROWS); + + //noinspection EmptyTryBlock,UnusedDeclaration + try (ResultSet rs = selector.getResultSet()) + { + } + catch (SQLException e) + { + errors.add(new QueryParseException(e.getMessage(), e, 0, 0)); + } + catch (BadSqlGrammarException e) + { + errors.add(new QueryParseException(e.getSQLException().getMessage(), e, 0, 0)); + } + + UserSchema schema = table.getUserSchema(); + if (schema != null) + { + QueryDefinition queryDef = schema.getQueryDef(table.getName()); + if (queryDef != null) + { + queryDef.validateQuery(schema, errors, warnings); + } + } + + OntologyService os = OntologyService.get(); + if (null != os) + { + for (var col : table.getColumns()) + { + String code = col.getPrincipalConceptCode(); + if (null != code) + { + Concept concept = os.resolveCode(code); + if (null == concept) + warnings.add(new QueryParseException("Concept not found: " + code, null, 0, 0)); + } + } + } + + return errors.isEmpty(); + } + + /** + * Experimental. The goal is to provide a more thorough validation of query metadata, including warnings of potentially + * invalid conditions, like autoincrement columns set userEditable=true. + */ + public boolean validateQueryMetadata(SchemaKey schemaPath, String queryName, User user, Container container, + @NotNull List errors, @NotNull List warnings) + { + Set columns = new HashSet<>(); + UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); + if (null == schema) + throw new IllegalArgumentException("Could not find the schema '" + schemaPath.getName() + "'!"); + + TableInfo table = schema.getTable(queryName); + if (null == table) + throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schemaPath.getName() + "'!"); + + if (table.isPublic() && table.getPublicSchemaName() != null && !schemaPath.toString().equalsIgnoreCase(table.getPublicSchemaName())) + warnings.add(new QueryParseWarning("(metadata) TableInfo.getPublicSchemaName() does not match: set to '" + table.getPublicSchemaName() + "', expected '" + schemaPath + "'", null, 0,0)); + + try + { + //validate foreign keys and other metadata warnings + columns.addAll(table.getColumns()); + columns.addAll(QueryService.get().getColumns(table, table.getDefaultVisibleColumns()).values()); + } + catch(QueryParseException e) + { + errors.add(e); + } + + for (ColumnInfo col : columns) + { + validateColumn(col, user, container, table, errors, warnings); + } + + return errors.isEmpty(); + } + + /** + * Experimental. See validateQueryMetadata() + */ + private boolean validateColumn(ColumnInfo col, User user, Container container, @Nullable TableInfo parentTable, + @NotNull List errors, @NotNull List warnings) + { + if(parentTable == null) + parentTable = col.getParentTable(); + + String publicSchema = col.getParentTable().getPublicSchemaName() != null ? col.getParentTable().getPublicSchemaName() : col.getParentTable().getSchema().toString(); + String publicQuery = col.getParentTable().getPublicName() != null ? col.getParentTable().getPublicName() : col.getParentTable().getName(); + String errorBase = "(metadata) for column '" + col.getFieldKey() + "' in " + publicSchema + "." + publicQuery + ": "; + + validateFk(col, user, container, parentTable, errors, warnings, errorBase); + + Set specialCols = new CaseInsensitiveHashSet(); + specialCols.add("LSID"); + specialCols.add("entityId"); + specialCols.add("container"); + specialCols.add("created"); + specialCols.add("createdby"); + specialCols.add("modified"); + specialCols.add("modifiedby"); + + if(specialCols.contains(col.getName())) + { + if(col.isUserEditable()) + warnings.add(new QueryParseWarning(errorBase + " column is user editable, which is not expected based on its name", null, 0,0)); + if(col.isShownInInsertView()) + warnings.add(new QueryParseWarning(errorBase + " column has shownInInsertView set to true, which is not expected based on its name", null, 0, 0)); + if(col.isShownInUpdateView()) + warnings.add(new QueryParseWarning(errorBase + " column has shownInUpdateView set to true, which is not expected based on its name", null, 0, 0)); + } + + if(col.isAutoIncrement() && col.isUserEditable()) + warnings.add(new QueryParseException(errorBase + " column is autoIncrement, but has userEditable set to true", null, 0, 0)); + if(col.isAutoIncrement() && col.isShownInInsertView()) + warnings.add(new QueryParseWarning(errorBase + " column is autoIncrement, but has shownInInsertView set to true", null, 0, 0)); + if(col.isAutoIncrement() && col.isShownInUpdateView()) + warnings.add(new QueryParseWarning(errorBase + " column is autoIncrement, but has shownInUpdateView set to true", null, 0, 0)); + + try + { + if (StringUtils.isNotBlank(col.getDisplayWidth()) && Integer.parseInt(col.getDisplayWidth()) > 200 && !"textarea".equalsIgnoreCase(col.getInputType())) + { + if (col.isUserEditable() && col.getJdbcType() != null && col.getJdbcType().getJavaClass() == String.class) + warnings.add(new QueryParseWarning(errorBase + " column has a displayWidth > 200, but does not use a textarea as the inputType", null, 0, 0)); + } + } + catch (NumberFormatException e) + { + warnings.add(new QueryParseWarning(errorBase + " column has invalid value for displayWidth: '" + col.getDisplayWidth() + "'", null, 0, 0)); + } + return errors.isEmpty(); + } + + /** + * Experimental. See validateQueryMetadata() + */ + private boolean validateFk(ColumnInfo col, User user, Container container, TableInfo parentTable, + @NotNull List errors, @NotNull List warnings, + String errorBase) + + { + //NOTE: this is the same code that writes JSON to the client + JSONObject o = JsonWriter.getLookupInfo(col, false); + if (o == null) + return true; + + boolean isPublic = o.getBoolean("isPublic"); + SchemaKey schemaPath = SchemaKey.fromString(o.optString("schemaName")); + String queryName = o.optString("queryName"); + if (queryName == null) + { + // Likely a lookup that targets something not exposed via a UserSchema. Bail out without further validation + return true; + } + String displayColumn = o.optString("displayColumn"); + String keyColumn = o.optString("keyColumn"); + String containerPath = o.optString("containerPath"); + + Container lookupContainer = containerPath == null ? container : ContainerManager.getForPath(containerPath); + if (lookupContainer == null) + { + warnings.add(new QueryParseWarning(errorBase + " Unable to find container" + containerPath, null, 0, 0)); + } + + //String publicSchema = col.getParentTable().getPublicSchemaName() != null ? col.getParentTable().getPublicSchemaName() : col.getParentTable().getSchema().toString(); + //String publicQuery = col.getParentTable().getPublicName() != null ? col.getParentTable().getPublicName() : col.getParentTable().getName(); + if (col.getFk() == null) + return errors.isEmpty(); + + if (!isPublic) + { + warnings.add(new QueryParseWarning(errorBase + " has a lookup to a non-public table: " + (schemaPath == null ? "" : schemaPath.toDisplayString()) + "." + queryName, null, 0, 0)); + return errors.isEmpty(); + } + + UserSchema userSchema = QueryService.get().getUserSchema(user, lookupContainer, schemaPath); + if (userSchema == null) + { + warnings.add(new QueryParseWarning(errorBase + " unable to find the user schema: " + schemaPath.toDisplayString(), null, 0, 0)); + return errors.isEmpty(); + } + + TableInfo fkTable = userSchema.getTable(queryName); + if(fkTable == null) + { + warnings.add(new QueryParseWarning(errorBase + " has a lookup to a table that does not exist: " + schemaPath.toDisplayString() + "." + queryName, null, 0, 0)); + return errors.isEmpty(); + } + + //a FK can have a table non-visible to the client, so long as public is set to false + if (fkTable.isPublic()){ + String fkt = schemaPath.toDisplayString() + "." + queryName; + + QueryManager.get().validateQuery(schemaPath, queryName, user, lookupContainer, errors, warnings); + if (displayColumn != null) + { + FieldKey displayFieldKey = FieldKey.fromString(displayColumn); + Map cols = QueryService.get().getColumns(fkTable, Collections.singleton(displayFieldKey)); + if (!cols.containsKey(displayFieldKey)) + { + warnings.add(new QueryParseWarning(errorBase + " reports a foreign key with displayColumn of " + displayColumn + " in the table " + schemaPath.toDisplayString() + "." + queryName + ", but the column does not exist", null, 0, 0)); + } + else + { + ColumnInfo ci = cols.get(displayFieldKey); + if (!displayColumn.equals(ci.getFieldKey().toString())) + { + warnings.add(new QueryParseWarning(errorBase + ", the lookup to " + schemaPath.toDisplayString() + "." + queryName + "' did not match the expected case, which was '" + ci.getFieldKey().toString() + "'. Actual: '" + displayColumn + "'", null, 0, 0)); + } + } + } + + if (keyColumn != null) + { + FieldKey keyFieldKey = FieldKey.fromString(keyColumn); + Map cols = QueryService.get().getColumns(fkTable, Collections.singleton(keyFieldKey)); + if (!cols.containsKey(keyFieldKey)) + { + warnings.add(new QueryParseException(errorBase + " reports a foreign key with keyColumn of " + keyColumn + " in the table " + schemaPath.toDisplayString() + "." + queryName + ", but the column does not exist", null, 0, 0)); + } + else + { + ColumnInfo ci = cols.get(keyFieldKey); + if (!keyColumn.equals(ci.getFieldKey().toString())) + { + warnings.add(new QueryParseWarning(errorBase + ", the lookup to " + schemaPath.toDisplayString() + "." + queryName + "' did not match the expected case, which was '" + ci.getFieldKey().toString() + "'. Actual: '" + keyColumn + "'", null, 0, 0)); + } + } + } + else + { + warnings.add(new QueryParseWarning(errorBase + ", there is a lookup where the keyColumn is blank", null, 0, 0)); + } + } + + return errors.isEmpty(); + } + + /** + * Experimental. The goal is to provide a more thorough validation of saved views, including errors like invalid + * column names or case errors (which cause problems for case-sensitive js) + */ + public boolean validateQueryViews(SchemaKey schemaPath, String queryName, User user, Container container, + @NotNull List errors, @NotNull List warnings) throws QueryParseException + { + UserSchema schema = QueryService.get().getUserSchema(user, container, schemaPath); + if (null == schema) + throw new IllegalArgumentException("Could not find the schema '" + schemaPath.getName() + "'!"); + + TableInfo table = schema.getTable(queryName); + if (null == table) + throw new IllegalArgumentException("The query '" + queryName + "' was not found in the schema '" + schema.getSchemaName() + "'!"); + + //validate views + try + { + List views = QueryService.get().getCustomViews(user, container, null, schema.getSchemaName(), queryName, true); + for (CustomView v : views) + { + validateViewColumns(user, container, v, "columns", v.getColumns(), table, errors, warnings); + + if (!StringUtils.isEmpty(v.getFilterAndSort())) + { + try + { + CustomViewInfo.FilterAndSort fs = CustomViewInfo.FilterAndSort.fromString(v.getFilterAndSort()); + List filterCols = new ArrayList<>(); + for (FilterInfo f : fs.getFilter()) + { + filterCols.add(f.getField()); + } + validateViewColumns(user, container, v, "filter", filterCols, table, errors, warnings); + + List sortCols = new ArrayList<>(); + for (Sort.SortField f : fs.getSort()) + { + sortCols.add(f.getFieldKey()); + } + validateViewColumns(user, container, v, "sort", sortCols, table, errors, warnings); + + } + catch (URISyntaxException e) + { + warnings.add(new QueryParseWarning("unable to process the filter/sort section of view: " + v.getName(), null, 0, 0)); + } + } + } + } + catch (NotFoundException e) + { + errors.add(new QueryParseException("Cannot get views: ", e, 0, 0)); + } + + + return errors.isEmpty(); + } + + private void validateViewColumns(User user, Container container, CustomView v, String identifier, List viewCols, TableInfo sourceTable, + @NotNull List errors, @NotNull List warnings) throws QueryParseException + { + //verify columns match, accounting for case + Map colMap = QueryService.get().getColumns(sourceTable, viewCols); + + for (FieldKey f : viewCols) + { + boolean found = false; + boolean matchCase = false; + FieldKey fk = null; + ColumnInfo c = colMap.get(f); + if(c != null) + { + found = true; + fk = c.getFieldKey(); + if(c instanceof AliasedColumn) + fk = ((AliasedColumn)c).getColumn().getFieldKey(); + + if(fk.toString().equals(f.toString())) + { + matchCase = true; + } + } + + if (!found){ + warnings.add(new QueryParseWarning("In the saved view '" + (v.getName() == null ? "default" : v.getName()) + "', in the " + identifier + " section, the column '" + f.toString() + "' in " + v.getSchemaName() + "." + v.getQueryName() + " could not be matched to a column", null, 0, 0)); + continue; + } + + if (!matchCase){ + warnings.add(new QueryParseWarning("In the saved view '" + (v.getName() == null ? "default" : v.getName()) + "', in the " + identifier + " section, the column '" + f + "' in " + v.getSchemaName() + "." + v.getQueryName() + "' did not match the expected case, which was '" + fk + "'", null, 0, 0)); + } + + //queryErrors.addAll(validateColumn(c, user, container)); + } + } + + public static void registerUsageMetrics(String moduleName) + { + UsageMetricsService svc = UsageMetricsService.get(); + if (null != svc) + { + svc.registerUsageMetrics(moduleName, () -> { + Bag bag = DbScope.getDbScopes().stream() + .filter(scope -> !scope.isLabKeyScope()).map(DbScope::getDatabaseProductName) + .collect(Collectors.toCollection(HashBag::new)); + + Map statsMap = bag.uniqueSet().stream() + .collect(Collectors.toMap(Function.identity(), bag::getCount)); + + return Map.of("externalDatasources", statsMap, + "customViewCounts", + Map.of( + "DataClasses", getSchemaCustomViewCounts("exp.data"), + "SampleTypes", getSchemaCustomViewCounts("samples"), + "Assays", getSchemaCustomViewCounts("assay"), + "Inventory", getSchemaCustomViewCounts("inventory") + ), + "customViewWithLineageColumn", getLineageCustomViewMetrics(), + "queryDefWithCalculatedFieldsCounts", getCalculatedFieldsCountsMetric() + ); + }); + } + } + + private static Map getCalculatedFieldsCountsMetric() + { + DbSchema dbSchema = CoreSchema.getInstance().getSchema(); + return new SqlSelector(dbSchema, + new SQLFragment("SELECT \"schema\", COUNT(*) AS count FROM (\n" + + " SELECT CASE WHEN \"schema\" LIKE 'assay.%' THEN 'assay' ELSE \"schema\" END AS \"schema\" FROM query.querydef WHERE metadata LIKE '%(), (x, m) -> { + x.put(m.get("schema").toString(), m.get("count")); + return x; + }); + } + + private static Map getSchemaCustomViewCounts(String schema) + { + DbSchema dbSchema = DbSchema.get("query"); + TableInfo customView = dbSchema.getTable("customview"); + var schemaField = customView.getColumn("schema").getSelectIdentifier(); + SQLFragment schemaClause; + if (schema.equalsIgnoreCase("assay")) + schemaClause = new SQLFragment("C.").appendIdentifier(schemaField).append(" LIKE 'assay.%'"); + else + schemaClause = new SQLFragment("C.").appendIdentifier(schemaField).append(" = ").appendValue(schema); + return Map.of( + "defaultOverrides", new SqlSelector(dbSchema, + new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags < 2 AND C.name IS NULL")).getObject(Long.class), // possibly inheritable, no hidden, not snapshot + "inheritable", new SqlSelector(dbSchema, + new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags = 1")).getObject(Long.class), // inheritable, not hidden, not snapshot + "namedViews", new SqlSelector(dbSchema, + new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.flags < 2 AND C.name IS NOT NULL")).getObject(Long.class), // possibly inheritable, no hidden, not snapshot + "shared", new SqlSelector(dbSchema, + new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.customviewowner IS NULL")).getObject(Long.class), + "identifyingFieldsViews", new SqlSelector(dbSchema, + new SQLFragment("SELECT COUNT(*) FROM query.customview C WHERE ").append(schemaClause).append(" AND C.name = '~~identifyingfields~~'")).getObject(Long.class) + ); + } + + + private static Long percentile(double percentile, List sortedCounts) { + if (percentile <= 0.01) + return sortedCounts.get(0); + if (percentile >= 99.99) + return sortedCounts.get(sortedCounts.size() - 1); + return sortedCounts.get((int) Math.round(percentile / 100.0 * (sortedCounts.size() - 1))); + } + + /** + * customViewsCountWithLineageCol: total number of non-hidden saved custom views that has at least one input/output/ancestor column + * customViewsCountWithAncestorCol: total number of non-hidden saved custom views that has at least one ancestor column + * totalLineageColumnsInAllViews: total number of input/output/ancestor columns defined for all saved non-hidden custom views + * totalAncestorColumnsInAllViews: total number of ancestor columns defined for all saved non-hidden custom views + * lineageColumnsCountMin: the minimum count of input/output/ancestor columns in any view with such column + * lineageColumnsCount25: the 25 percentile count of input/output/ancestor columns in all views with such column + * lineageColumnsCount50: the 50 percentile / median count of input/output/ancestor columns in all views with such column + * lineageColumnsCount75: the 75 percentile count of input/output/ancestor columns in all views with such column + * lineageColumnsCountMax: the maximum count of input/output/ancestor columns in any view with such column + * lineageColumnsCountAvg: the average count of input/output/ancestor columns in any view with such column + * ancestorColumnsCountMin: the minimum count of ancestor columns in any view with ancestor columns + * ancestorColumnsCount25: the 25 percentile count of ancestor columns in all views with ancestor columns + * ancestorColumnsCount50: the 50 percentile / median count of ancestor columns in all views with ancestor columns + * ancestorColumnsCount75: the 75 percentile count of ancestor columns in all views with ancestor columns + * ancestorColumnsCountMax: the maximum count of ancestor columns in any view with ancestor columns + * ancestorColumnsCountAvg: the average count of ancestor columns in any view with ancestor columns + */ + private static Map getLineageCustomViewMetrics() + { + List ancestorColCounts = new ArrayList<>(); + List lineageColCounts = new ArrayList<>(); + final String ANCESTOR_PREFIX = "ancestors/"; + final String INPUT_PREFIX = "inputs/"; + final String OUTPUT_PREFIX = "outputs/"; + + Map metrics = new HashMap<>(); + + DbSchema schema = DbSchema.get("query", DbSchemaType.Module); + SqlDialect sqlDialect = schema.getSqlDialect(); + SQLFragment sql = new SQLFragment() + .append("SELECT columns FROM query.customview WHERE flags < 2 AND (columns LIKE ? OR columns LIKE ? OR columns LIKE ?)") + .add("%" + sqlDialect.encodeLikeOpSearchString("Ancestors%2F") + "%") + .add("%" + sqlDialect.encodeLikeOpSearchString("Inputs%2F") + "%") + .add("%" + sqlDialect.encodeLikeOpSearchString("Outputs%2F") + "%"); + List viewsColumnStrs = new SqlSelector(schema, sql).getArrayList(String.class); + + for (String columnStr : viewsColumnStrs) + { + long lineageColCount = 0L; + long ancestorColCount = 0L; + for (Map.Entry> entry : CustomViewInfo.decodeProperties(columnStr)) + { + String fieldName = entry.getKey().toString().toLowerCase(); + if (fieldName.startsWith(ANCESTOR_PREFIX)) + { + ancestorColCount++; + lineageColCount++; + } + else if (fieldName.startsWith(INPUT_PREFIX) || fieldName.startsWith(OUTPUT_PREFIX)) + { + lineageColCount++; + } + } + if (ancestorColCount > 0) + ancestorColCounts.add(ancestorColCount); + if (lineageColCount > 0) + lineageColCounts.add(lineageColCount); + } + + Collections.sort(lineageColCounts); + int lineageViewCount = lineageColCounts.size(); + metrics.put("customViewsCountWithLineageColumnsCount", lineageViewCount); + if (lineageViewCount != 0) + { + long totalLineageCols = lineageColCounts.stream().mapToLong(Long::longValue).sum(); + metrics.put("totalLineageColumnsInAllViews", totalLineageCols); + metrics.put("lineageColumnsCountMin", percentile(0, lineageColCounts)); + metrics.put("lineageColumnsCount25", percentile(25, lineageColCounts)); + metrics.put("lineageColumnsCount50", percentile(50, lineageColCounts)); + metrics.put("lineageColumnsCount75", percentile(75, lineageColCounts)); + metrics.put("lineageColumnsCountMax", percentile(100, lineageColCounts)); + metrics.put("lineageColumnsCountAvg", Math.round((float) totalLineageCols / lineageViewCount)); + } + + Collections.sort(ancestorColCounts); + int ancestorViewCount = ancestorColCounts.size(); + metrics.put("customViewsWithAncestorColumnsCounts", ancestorViewCount); + if (ancestorViewCount != 0) + { + long totalAncestorCols = ancestorColCounts.stream().mapToLong(Long::longValue).sum(); + metrics.put("totalAncestorColumnsInAllViews", totalAncestorCols); + metrics.put("ancestorColumnsCountMin", percentile(0, ancestorColCounts)); + metrics.put("ancestorColumnsCount25", percentile(25, ancestorColCounts)); + metrics.put("ancestorColumnsCount50", percentile(50, ancestorColCounts)); + metrics.put("ancestorColumnsCount75", percentile(75, ancestorColCounts)); + metrics.put("ancestorColumnsCountMax", percentile(100, ancestorColCounts)); + metrics.put("ancestorColumnsCountAvg", Math.round((float) totalAncestorCols / ancestorViewCount)); + } + + return metrics; + } + + +} From cbb199b1539b7fec8bb08c7364870da393a79378 Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 23 Jan 2026 10:49:37 -0800 Subject: [PATCH 3/4] Port Is empty filter changes from fb_mvtc_empty branch --- api/src/org/labkey/api/data/CompareType.java | 95 +++++++++++++++++++ api/src/org/labkey/api/data/SimpleFilter.java | 2 +- .../labkey/api/data/dialect/SqlDialect.java | 6 ++ .../core/dialect/PostgreSql92Dialect.java | 6 ++ .../org/labkey/query/QueryServiceImpl.java | 2 + query/src/org/labkey/query/sql/Method.java | 30 ++++++ 6 files changed, 140 insertions(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/data/CompareType.java b/api/src/org/labkey/api/data/CompareType.java index d9165ae67da..954a35739fa 100644 --- a/api/src/org/labkey/api/data/CompareType.java +++ b/api/src/org/labkey/api/data/CompareType.java @@ -827,6 +827,39 @@ protected Collection getCollectionParam(Object value) * * */ + + + public static final CompareType ARRAY_IS_EMPTY = new CompareType("Is Empty", "arrayisempty", "ARRAYISEMPTY", false, null, OperatorType.ARRAYISEMPTY) + { + @Override + public ArrayIsEmptyClause createFilterClause(@NotNull FieldKey fieldKey, Object value) + { + return new ArrayIsEmptyClause(fieldKey); + } + + @Override + public boolean meetsCriteria(ColumnRenderProperties col, Object value, Object[] filterValues) + { + throw new UnsupportedOperationException("Conditional formatting not yet supported for Multi Choices"); + } + }; + + + public static final CompareType ARRAY_IS_NOT_EMPTY = new CompareType("Is Not Empty", "arrayisnotempty", "ARRAYISNOTEMPTY", false, null, OperatorType.ARRAYISNOTEMPTY) + { + @Override + public ArrayIsEmptyClause createFilterClause(@NotNull FieldKey fieldKey, Object value) + { + return new ArrayIsNotEmptyClause(fieldKey); + } + + @Override + public boolean meetsCriteria(ColumnRenderProperties col, Object value, Object[] filterValues) + { + throw new UnsupportedOperationException("Conditional formatting not yet supported for Multi Choices"); + } + }; + public static final CompareType ARRAY_CONTAINS_ALL = new CompareType("Contains All", "arraycontainsall", "ARRAYCONTAINSALL", true, null, OperatorType.ARRAYCONTAINSALL) { @Override @@ -981,6 +1014,68 @@ public Pair getSqlFragments(Map columnMap, SqlDialect dialect) + { + ColumnInfo colInfo = columnMap != null ? columnMap.get(_fieldKey) : null; + var alias = SimpleFilter.getAliasForColumnFilter(dialect, colInfo, _fieldKey); + + SQLFragment columnFragment = new SQLFragment().appendIdentifier(alias); + + SQLFragment sql = dialect.array_is_empty(columnFragment); + if (!_negated) + return sql; + return new SQLFragment(" NOT (").append(sql).append(")"); + } + + @Override + public String getLabKeySQLWhereClause(Map columnMap) + { + return "array_is_empty(" + getLabKeySQLColName(_fieldKey) + ")"; + } + + @Override + public void appendFilterText(StringBuilder sb, ColumnNameFormatter formatter) + { + sb.append("is empty"); + } + + } + + private static class ArrayIsNotEmptyClause extends ArrayIsEmptyClause + { + + public ArrayIsNotEmptyClause(@NotNull FieldKey fieldKey) + { + super(fieldKey, CompareType.ARRAY_IS_NOT_EMPTY, true); + } + + @Override + public String getLabKeySQLWhereClause(Map columnMap) + { + return "NOT array_is_empty(" + getLabKeySQLColName(_fieldKey) + ")"; + } + + @Override + public void appendFilterText(StringBuilder sb, ColumnNameFormatter formatter) + { + sb.append("is not empty"); + } + + } + private static class ArrayContainsAllClause extends ArrayClause { diff --git a/api/src/org/labkey/api/data/SimpleFilter.java b/api/src/org/labkey/api/data/SimpleFilter.java index cdedb78ce2c..1cb5380207e 100644 --- a/api/src/org/labkey/api/data/SimpleFilter.java +++ b/api/src/org/labkey/api/data/SimpleFilter.java @@ -620,7 +620,7 @@ public static abstract class MultiValuedFilterClause extends CompareType.Abstrac public MultiValuedFilterClause(@NotNull FieldKey fieldKey, CompareType comparison, Collection params, boolean negated) { super(fieldKey); - params = new ArrayList<>(params); // possibly immutable + params = params == null ? new ArrayList<>() : new ArrayList<>(params); // possibly immutable if (params.contains(null)) //params.size() == 0 || { _includeNull = true; diff --git a/api/src/org/labkey/api/data/dialect/SqlDialect.java b/api/src/org/labkey/api/data/dialect/SqlDialect.java index 85cc60ec933..77e7430d148 100644 --- a/api/src/org/labkey/api/data/dialect/SqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/SqlDialect.java @@ -2200,6 +2200,12 @@ public SQLFragment array_construct(SQLFragment[] elements) throw new UnsupportedOperationException(getClass().getSimpleName() + " does not implement"); } + public SQLFragment array_is_empty(SQLFragment a) + { + assert !supportsArrays(); + throw new UnsupportedOperationException(getClass().getSimpleName() + " does not implement"); + } + // element a is in array b public SQLFragment element_in_array(SQLFragment a, SQLFragment b) { diff --git a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java index a25da4256ad..daca9198496 100644 --- a/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java +++ b/core/src/org/labkey/core/dialect/PostgreSql92Dialect.java @@ -1157,6 +1157,12 @@ public SQLFragment array_construct(SQLFragment[] elements) return ret; } + @Override + public SQLFragment array_is_empty(SQLFragment a) + { + return new SQLFragment("(cardinality(").append(a).append(")=0)"); + } + @Override public SQLFragment array_all_in_array(SQLFragment a, SQLFragment b) { diff --git a/query/src/org/labkey/query/QueryServiceImpl.java b/query/src/org/labkey/query/QueryServiceImpl.java index 0d49c024b5c..e361786fa82 100644 --- a/query/src/org/labkey/query/QueryServiceImpl.java +++ b/query/src/org/labkey/query/QueryServiceImpl.java @@ -309,6 +309,8 @@ public void moduleChanged(Module module) CompareType.NONBLANK, CompareType.MV_INDICATOR, CompareType.NO_MV_INDICATOR, + CompareType.ARRAY_IS_EMPTY, + CompareType.ARRAY_IS_NOT_EMPTY, CompareType.ARRAY_CONTAINS_ALL, CompareType.ARRAY_CONTAINS_ANY, CompareType.ARRAY_CONTAINS_NONE, diff --git a/query/src/org/labkey/query/sql/Method.java b/query/src/org/labkey/query/sql/Method.java index 77214c9d348..b6377660675 100644 --- a/query/src/org/labkey/query/sql/Method.java +++ b/query/src/org/labkey/query/sql/Method.java @@ -1573,6 +1573,34 @@ public SQLFragment getSQL(SqlDialect dialect, SQLFragment[] arguments) } } + public static class ArrayIsEmptyMethod extends Method + { + ArrayIsEmptyMethod(String name) + { + super(name, JdbcType.BOOLEAN, 1, 1); + } + + @Override + public MethodInfo getMethodInfo() + { + return new AbstractMethodInfo(JdbcType.BOOLEAN) + { + @Override + public JdbcType getJdbcType(JdbcType[] args) + { + if (1 == args.length && args[0] != JdbcType.ARRAY) + throw new QueryParseException(_name + " requires an argument of type ARRAY", null, -1, -1); + return super.getJdbcType(args); + } + + @Override + public SQLFragment getSQL(SqlDialect dialect, SQLFragment[] arguments) + { + return dialect.array_is_empty(arguments[0]); + } + }; + } + } final static Map postgresMethods = Collections.synchronizedMap(new CaseInsensitiveHashMap<>()); @@ -1650,6 +1678,8 @@ private static void addPostgresArrayMethods() // not array_equals() because arrays are ordered, this is an unordered comparison postgresMethods.put("array_is_same", new ArrayOperatorMethod("array_is_same", SqlDialect::array_same_array)); // Use "NOT array_is_same()" instead of something clumsy like "array_is_not_same()" + + postgresMethods.put("array_is_empty", new ArrayIsEmptyMethod("array_is_empty")); } From 98f19f25d8132e11d9925e940b54aa32e0d92c8a Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 23 Jan 2026 16:29:42 -0800 Subject: [PATCH 4/4] fix display --- core/package-lock.json | 8 ++++---- core/package.json | 2 +- experiment/package-lock.json | 8 ++++---- experiment/package.json | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/package-lock.json b/core/package-lock.json index ef72ba6e0e2..7897498f8de 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.14.0-fb-mvtc-convert.1", + "@labkey/components": "7.14.0-fb-mvtc-convert.3", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3547,9 +3547,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.14.0-fb-mvtc-convert.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.14.0-fb-mvtc-convert.1.tgz", - "integrity": "sha512-LDwQkXH1oAsDTn0C+Ep6JvIl4sqXE2c5v+a/veezi96ltzWgXd7gOPrUubFRbLO0xXG3nn8QzGWdu0G7Zd0kWg==", + "version": "7.14.0-fb-mvtc-convert.3", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.14.0-fb-mvtc-convert.3.tgz", + "integrity": "sha512-zWFCmFERVct/gyKuOMZTylR7CEBgpcsOnU8A3sjfX/gXyxJ/JgC5BsFSOe92zOyMYRmMkNoBa+m+9QwTqAom6Q==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/core/package.json b/core/package.json index 7107e778af0..d9b435baf3e 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.14.0-fb-mvtc-convert.1", + "@labkey/components": "7.14.0-fb-mvtc-convert.3", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index 7a889cd4ac9..94a68fea867 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.14.0-fb-mvtc-convert.1" + "@labkey/components": "7.14.0-fb-mvtc-convert.3" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3314,9 +3314,9 @@ } }, "node_modules/@labkey/components": { - "version": "7.14.0-fb-mvtc-convert.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.14.0-fb-mvtc-convert.1.tgz", - "integrity": "sha512-LDwQkXH1oAsDTn0C+Ep6JvIl4sqXE2c5v+a/veezi96ltzWgXd7gOPrUubFRbLO0xXG3nn8QzGWdu0G7Zd0kWg==", + "version": "7.14.0-fb-mvtc-convert.3", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.14.0-fb-mvtc-convert.3.tgz", + "integrity": "sha512-zWFCmFERVct/gyKuOMZTylR7CEBgpcsOnU8A3sjfX/gXyxJ/JgC5BsFSOe92zOyMYRmMkNoBa+m+9QwTqAom6Q==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", diff --git a/experiment/package.json b/experiment/package.json index 52420d77e1c..d98c3e473ac 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.14.0-fb-mvtc-convert.1" + "@labkey/components": "7.14.0-fb-mvtc-convert.3" }, "devDependencies": { "@labkey/build": "8.7.0",