From 3bdcd83b5f812992caac27673b0489360b080977 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 13 Jul 2026 15:54:22 +0200 Subject: [PATCH 01/31] feat: add SqlBuilder capability flags for Doris continuous analytics --- .../hisp/dhis/db/sql/ClickHouseSqlBuilder.java | 10 ++++++++++ .../org/hisp/dhis/db/sql/DorisSqlBuilder.java | 10 ++++++++++ .../org/hisp/dhis/db/sql/PostgreSqlBuilder.java | 10 ++++++++++ .../java/org/hisp/dhis/db/sql/SqlBuilder.java | 15 +++++++++++++++ .../org/hisp/dhis/db/sql/DorisSqlBuilderTest.java | 11 +++++++++++ 5 files changed, 56 insertions(+) diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java index 771d348432c8..2ef03f4986fc 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/ClickHouseSqlBuilder.java @@ -219,6 +219,16 @@ public boolean requiresIndexesForAnalytics() { return false; } + @Override + public boolean supportsContinuousAnalytics() { + return false; + } + + @Override + public boolean requiresUniqueKeyAnalyticsTables() { + return false; + } + @Override public boolean supportsPercentileCont() { return false; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index 99aa77dd9b4e..570f896844cc 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -221,6 +221,16 @@ public boolean requiresIndexesForAnalytics() { return false; } + @Override + public boolean supportsContinuousAnalytics() { + return true; + } + + @Override + public boolean requiresUniqueKeyAnalyticsTables() { + return true; + } + @Override public boolean supportsPercentileCont() { return false; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java index b60a460abb83..aa7f87c42b24 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/PostgreSqlBuilder.java @@ -222,6 +222,16 @@ public boolean requiresIndexesForAnalytics() { return true; } + @Override + public boolean supportsContinuousAnalytics() { + return true; + } + + @Override + public boolean requiresUniqueKeyAnalyticsTables() { + return false; + } + @Override public boolean supportsPercentileCont() { return true; diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java index 6c2393c6d298..3f3dd5dd5b1c 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java @@ -211,6 +211,21 @@ public interface SqlBuilder { */ boolean requiresIndexesForAnalytics(); + /** + * @return true if the DBMS supports continuous (incremental latest-partition) analytics table + * updates. + */ + boolean supportsContinuousAnalytics(); + + /** + * @return true if analytics tables must be created with a real unique key for continuous + * analytics table updates to work correctly on this DBMS. When true, the natural key column + * of the affected analytics table types (e.g. {@code event}, {@code id}) is passed as the + * table's primary key so the DBMS can use it for both deduplication on insert and DELETE + * statements that join back to operational tables. + */ + boolean requiresUniqueKeyAnalyticsTables(); + /** * The percentile_cont function is a statistical function that calculates the continuous * percentile value for a given set of data. diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java index 312ae67d122e..aad3633cb7ef 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java @@ -32,6 +32,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.List; import org.hisp.dhis.db.model.Collation; @@ -136,6 +137,16 @@ void testSupportsVacuum() { assertFalse(sqlBuilder.supportsVacuum()); } + @Test + void testSupportsContinuousAnalytics() { + assertTrue(sqlBuilder.supportsContinuousAnalytics()); + } + + @Test + void testRequiresUniqueKeyAnalyticsTables() { + assertTrue(sqlBuilder.requiresUniqueKeyAnalyticsTables()); + } + // Utilities @Test From 4c82a38871000d4eeabb6eb39586123c66bda9c6 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 13 Jul 2026 17:50:00 +0200 Subject: [PATCH 02/31] feat: add primary-key-bearing AnalyticsTable constructor overloads Co-Authored-By: Claude Sonnet 5 --- .../analytics/table/model/AnalyticsTable.java | 50 +++++++++++++++++++ .../table/model/AnalyticsTableTest.java | 34 +++++++++++++ 2 files changed, 84 insertions(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTable.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTable.java index 4723736b5df0..aef7b396b34b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTable.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTable.java @@ -95,6 +95,34 @@ public AnalyticsTable( this.analyticsTableColumns = columns; } + /** + * Constructor. Sets the name to represent a staging table. Use when the table requires both a + * sort key and a real primary key (e.g. Doris continuous analytics, which needs a unique key + * column for deduplication and DELETE support). + * + * @param tableType the {@link AnalyticsTableType}. + * @param columns the list of {@link Column}. + * @param sortKey the sort key. + * @param primaryKey the primary key. + * @param logged the {@link Logged} property. + */ + public AnalyticsTable( + AnalyticsTableType tableType, + List columns, + List sortKey, + List primaryKey, + Logged logged) { + super( + toStaging(tableType.getTableName()), + toColumns(columns), + primaryKey, + sortKey, + List.of(), + logged); + this.tableType = tableType; + this.analyticsTableColumns = columns; + } + /** * Constructor. Sets the name to represent a staging table. * @@ -114,6 +142,28 @@ public AnalyticsTable( this.program = program; } + /** + * Constructor. Sets the name to represent a staging table. Use when the table requires a real + * primary key (e.g. Doris continuous analytics). + * + * @param tableType the {@link AnalyticsTableType}. + * @param columns the list of {@link Column}. + * @param primaryKey the primary key. + * @param logged the {@link Logged} property. + * @param program the {@link Program}. + */ + public AnalyticsTable( + AnalyticsTableType tableType, + List columns, + List primaryKey, + Logged logged, + Program program) { + super(toStaging(getTableName(tableType, program)), toColumns(columns), primaryKey, logged); + this.tableType = tableType; + this.analyticsTableColumns = columns; + this.program = program; + } + /** * Constructor. Sets the name to represent a staging table. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableTest.java index f3c7e8e91877..bae067263f88 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/model/AnalyticsTableTest.java @@ -215,4 +215,38 @@ void testEquals() { uniqueList.add(tableB); assertEquals(1, uniqueList.size()); } + + @Test + void testConstructorWithSortKeyAndPrimaryKey() { + List columns = + List.of( + AnalyticsTableColumn.builder().name("id").dataType(TEXT).build(), + AnalyticsTableColumn.builder().name("dx").dataType(CHARACTER_11).build()); + + AnalyticsTable table = + new AnalyticsTable( + AnalyticsTableType.DATA_VALUE, columns, List.of("dx"), List.of("id"), Logged.UNLOGGED); + + assertTrue(table.hasPrimaryKey()); + assertEquals(List.of("id"), table.getPrimaryKey()); + assertTrue(table.hasSortKey()); + assertEquals(List.of("dx"), table.getSortKey()); + } + + @Test + void testConstructorWithPrimaryKeyAndProgram() { + Program program = new Program("ProgramA", "DescriptionA"); + program.setUid("rft56ybgfek"); + + List columns = + List.of(AnalyticsTableColumn.builder().name("event").dataType(CHARACTER_11).build()); + + AnalyticsTable table = + new AnalyticsTable( + AnalyticsTableType.EVENT, columns, List.of("event"), Logged.UNLOGGED, program); + + assertTrue(table.hasPrimaryKey()); + assertEquals(List.of("event"), table.getPrimaryKey()); + assertEquals(program, table.getProgram()); + } } From 5a0099fc20ac95f950a0f9c4f357c7e21d9130b5 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 13 Jul 2026 18:06:22 +0200 Subject: [PATCH 03/31] feat: give DATA_VALUE/COMPLETENESS Doris analytics tables a unique key --- .../table/AbstractJdbcTableManager.java | 13 +- .../table/JdbcAnalyticsTableManager.java | 10 +- .../table/JdbcCompletenessTableManager.java | 10 +- .../JdbcValidationResultTableManager.java | 3 +- .../JdbcAnalyticsTableManagerDorisTest.java | 135 ++++++++++++++++++ 5 files changed, 162 insertions(+), 9 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index f7203395da98..48f9a4c6dd33 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -374,14 +374,16 @@ protected AnalyticsTable getRegularAnalyticsTable( AnalyticsTableUpdateParams params, List dataYears, List columns, - List sortKey) { + List sortKey, + List primaryKey) { Calendar calendar = PeriodType.getCalendar(); List years = ListUtils.mutableCopy(dataYears); Logged logged = analyticsTableSettings.getTableLogged(); Collections.sort(years); - AnalyticsTable table = new AnalyticsTable(getAnalyticsTableType(), columns, sortKey, logged); + AnalyticsTable table = + new AnalyticsTable(getAnalyticsTableType(), columns, sortKey, primaryKey, logged); for (Integer year : years) { List checks = getPartitionChecks(year, getEndDate(calendar, year)); @@ -402,7 +404,9 @@ protected AnalyticsTable getRegularAnalyticsTable( * @param columns the list of {@link AnalyticsTableColumn}. */ protected AnalyticsTable getLatestAnalyticsTable( - AnalyticsTableUpdateParams params, List columns) { + AnalyticsTableUpdateParams params, + List columns, + List primaryKey) { SystemSettings settings = settingsProvider.getCurrentSettings(); Date lastFullTableUpdate = settings.getLastSuccessfulAnalyticsTablesUpdate(); Date lastLatestPartitionUpdate = settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(); @@ -416,7 +420,8 @@ protected AnalyticsTable getLatestAnalyticsTable( Date endDate = params.getStartTime(); boolean hasUpdatedData = hasUpdatedLatestData(lastAnyTableUpdate, endDate); - AnalyticsTable table = new AnalyticsTable(getAnalyticsTableType(), columns, List.of(), logged); + AnalyticsTable table = + new AnalyticsTable(getAnalyticsTableType(), columns, List.of(), primaryKey, logged); if (hasUpdatedData) { table.addTablePartition( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index 6b625a8c4016..11aeed1e279c 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -168,6 +168,8 @@ public class JdbcAnalyticsTableManager extends AbstractJdbcTableManager { private static final List SORT_KEY = List.of("dx", "co"); + private static final List PRIMARY_KEY = List.of("id"); + public JdbcAnalyticsTableManager( IdentifiableObjectManager idObjectManager, OrganisationUnitService organisationUnitService, @@ -210,10 +212,14 @@ public AnalyticsTableType getAnalyticsTableType() { @Override @Transactional public List getAnalyticsTables(AnalyticsTableUpdateParams params) { + List primaryKey = + sqlBuilder.requiresUniqueKeyAnalyticsTables() ? PRIMARY_KEY : List.of(); + AnalyticsTable table = params.isLatestUpdate() - ? getLatestAnalyticsTable(params, getColumns(params)) - : getRegularAnalyticsTable(params, getDataYears(params), getColumns(params), SORT_KEY); + ? getLatestAnalyticsTable(params, getColumns(params), primaryKey) + : getRegularAnalyticsTable( + params, getDataYears(params), getColumns(params), SORT_KEY, primaryKey); return table.hasTablePartitions() ? List.of(table) : List.of(); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java index 3063acadabbb..269fb2488531 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java @@ -96,6 +96,8 @@ public class JdbcCompletenessTableManager extends AbstractJdbcTableManager { private static final List SORT_KEY = List.of("dx"); + private static final List PRIMARY_KEY = List.of("id"); + public JdbcCompletenessTableManager( IdentifiableObjectManager idObjectManager, OrganisationUnitService organisationUnitService, @@ -134,10 +136,14 @@ public AnalyticsTableType getAnalyticsTableType() { @Override @Transactional public List getAnalyticsTables(AnalyticsTableUpdateParams params) { + List primaryKey = + sqlBuilder.requiresUniqueKeyAnalyticsTables() ? PRIMARY_KEY : List.of(); + AnalyticsTable table = params.isLatestUpdate() - ? getLatestAnalyticsTable(params, getColumns()) - : getRegularAnalyticsTable(params, getDataYears(params), getColumns(), SORT_KEY); + ? getLatestAnalyticsTable(params, getColumns(), primaryKey) + : getRegularAnalyticsTable( + params, getDataYears(params), getColumns(), SORT_KEY, primaryKey); return table.hasTablePartitions() ? List.of(table) : List.of(); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java index ee1874b254d2..b5afcd31de4a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcValidationResultTableManager.java @@ -144,7 +144,8 @@ public List getAnalyticsTables(AnalyticsTableUpdateParams params params.isLatestUpdate() ? new AnalyticsTable( AnalyticsTableType.VALIDATION_RESULT, getColumns(), List.of(), Logged.LOGGED) - : getRegularAnalyticsTable(params, getDataYears(params), getColumns(), SORT_KEY); + : getRegularAnalyticsTable( + params, getDataYears(params), getColumns(), SORT_KEY, List.of()); return table.hasTablePartitions() ? List.of(table) : List.of(); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java new file mode 100644 index 000000000000..c28d137556b3 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.table; + +import static java.util.stream.Collectors.toUnmodifiableSet; +import static org.hisp.dhis.period.PeriodType.PERIOD_TYPES; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import java.util.Date; +import java.util.List; +import org.hisp.dhis.analytics.AnalyticsTableHookService; +import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; +import org.hisp.dhis.analytics.partition.PartitionManager; +import org.hisp.dhis.analytics.table.model.AnalyticsTable; +import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; +import org.hisp.dhis.category.CategoryService; +import org.hisp.dhis.common.IdentifiableObjectManager; +import org.hisp.dhis.configuration.Configuration; +import org.hisp.dhis.configuration.ConfigurationService; +import org.hisp.dhis.dataapproval.DataApprovalLevelService; +import org.hisp.dhis.dataelement.DataElementService; +import org.hisp.dhis.db.sql.DorisSqlBuilder; +import org.hisp.dhis.db.sql.SqlBuilder; +import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.period.PeriodDataProvider; +import org.hisp.dhis.resourcetable.ResourceTableService; +import org.hisp.dhis.setting.SystemSettings; +import org.hisp.dhis.setting.SystemSettingsProvider; +import org.joda.time.DateTime; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.springframework.jdbc.core.JdbcTemplate; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class JdbcAnalyticsTableManagerDorisTest { + @Mock private IdentifiableObjectManager idObjectManager; + + @Mock private OrganisationUnitService organisationUnitService; + + @Mock private CategoryService categoryService; + + @Mock private SystemSettingsProvider settingsProvider; + + @Mock private SystemSettings settings; + + @Mock private DataApprovalLevelService dataApprovalLevelService; + + @Mock private ResourceTableService resourceTableService; + + @Mock private AnalyticsTableHookService analyticsTableHookService; + + @Mock private PartitionManager partitionManager; + + @Mock private JdbcTemplate jdbcTemplate; + + @Mock private AnalyticsTableSettings analyticsTableSettings; + + @Mock private PeriodDataProvider periodDataProvider; + + @Mock private ConfigurationService configurationService; + + @Mock private Configuration configuration; + + @Mock private DataElementService dataElementService; + + @Spy private SqlBuilder sqlBuilder = new DorisSqlBuilder("dhis2", "driver"); + + @InjectMocks private JdbcAnalyticsTableManager subject; + + @BeforeEach + void setUp() { + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(settings.getLastSuccessfulResourceTablesUpdate()).thenReturn(new Date(0L)); + } + + @Test + void testGetRegularAnalyticsTableHasUniqueKeyOnId() { + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .startTime(new DateTime(2020, 1, 1, 0, 0).toDate()) + .build(); + + when(jdbcTemplate.queryForList( + org.mockito.Mockito.anyString(), org.mockito.Mockito.eq(Integer.class))) + .thenReturn(List.of(2020)); + when(configurationService.getConfiguration()).thenReturn(configuration); + when(configuration.getDataOutputPeriodTypes()) + .thenReturn(PERIOD_TYPES.stream().collect(toUnmodifiableSet())); + + List tables = subject.getAnalyticsTables(params); + + assertEquals(1, tables.size()); + AnalyticsTable table = tables.get(0); + assertTrue(table.hasPrimaryKey()); + assertEquals(List.of("id"), table.getPrimaryKey()); + assertTrue(sqlBuilder.createTable(table).contains("unique key (`id`)")); + } +} From f1aff453ef5616982e855dd8e1b36e8921a001dd Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 13 Jul 2026 18:43:49 +0200 Subject: [PATCH 04/31] feat: emit DELETE...USING for Doris in JdbcAnalyticsTableManager.removeUpdatedData --- .../table/JdbcAnalyticsTableManager.java | 51 ++++++++++++------- .../JdbcAnalyticsTableManagerDorisTest.java | 35 +++++++++++++ 2 files changed, 69 insertions(+), 17 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index 11aeed1e279c..ba2ba54d17d1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -259,23 +259,40 @@ public void preCreateTables(AnalyticsTableUpdateParams params) { public void removeUpdatedData(List tables) { AnalyticsTablePartition partition = getLatestTablePartition(tables); String sql = - replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - where ax.id in ( \ - select concat(des.dataelementuid,'-',ps.iso,'-',ous.organisationunituid,'-',dcs.categoryoptioncombouid,'-',acs.categoryoptioncombouid) as id \ - from ${datavalue} dv \ - inner join analytics_rs_dataelementstructure des on dv.dataelementid=des.dataelementid \ - inner join analytics_rs_periodstructure ps on dv.periodid=ps.periodid \ - inner join analytics_rs_orgunitstructure ous on dv.sourceid=ous.organisationunitid \ - inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ - inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ - where dv.lastupdated >= '${startDate}'and dv.lastupdated < '${endDate}');""", - Map.of( - "tableName", sqlBuilder.qualifyTable(getAnalyticsTableType().getTableName()), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))); + sqlBuilder.requiresUniqueKeyAnalyticsTables() + ? replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + using ${datavalue} dv \ + inner join analytics_rs_dataelementstructure des on dv.dataelementid=des.dataelementid \ + inner join analytics_rs_periodstructure ps on dv.periodid=ps.periodid \ + inner join analytics_rs_orgunitstructure ous on dv.sourceid=ous.organisationunitid \ + inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ + inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ + where ax.id = concat(des.dataelementuid,'-',ps.iso,'-',ous.organisationunituid,'-',dcs.categoryoptioncombouid,'-',acs.categoryoptioncombouid) \ + and dv.lastupdated >= '${startDate}' and dv.lastupdated < '${endDate}';""", + Map.of( + "tableName", quote(getAnalyticsTableType().getTableName()), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))) + : replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + where ax.id in ( \ + select concat(des.dataelementuid,'-',ps.iso,'-',ous.organisationunituid,'-',dcs.categoryoptioncombouid,'-',acs.categoryoptioncombouid) as id \ + from ${datavalue} dv \ + inner join analytics_rs_dataelementstructure des on dv.dataelementid=des.dataelementid \ + inner join analytics_rs_periodstructure ps on dv.periodid=ps.periodid \ + inner join analytics_rs_orgunitstructure ous on dv.sourceid=ous.organisationunitid \ + inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ + inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ + where dv.lastupdated >= '${startDate}'and dv.lastupdated < '${endDate}');""", + Map.of( + "tableName", sqlBuilder.qualifyTable(getAnalyticsTableType().getTableName()), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))); invokeTimeAndLog(sql, "Remove updated data values"); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index c28d137556b3..3fd1a7e1201a 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -35,8 +35,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; +import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map; import org.hisp.dhis.analytics.AnalyticsTableHookService; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; import org.hisp.dhis.analytics.partition.PartitionManager; @@ -132,4 +134,37 @@ void testGetRegularAnalyticsTableHasUniqueKeyOnId() { assertEquals(List.of("id"), table.getPrimaryKey()); assertTrue(sqlBuilder.createTable(table).contains("unique key (`id`)")); } + + @Test + void testRemoveUpdatedDataUsesUsingJoinNotSubquery() { + Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); + Date lastLatestPartitionUpdate = new DateTime(2019, 3, 1, 9, 0).toDate(); + Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); + + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().startTime(startTime).build().withLatestPartition(); + + List> queryResp = new ArrayList<>(); + queryResp.add(Map.of("dataelementid", 1)); + + when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + .thenReturn(lastLatestPartitionUpdate); + when(jdbcTemplate.queryForList(org.mockito.Mockito.anyString())).thenReturn(queryResp); + when(configurationService.getConfiguration()).thenReturn(configuration); + when(configuration.getDataOutputPeriodTypes()) + .thenReturn(PERIOD_TYPES.stream().collect(toUnmodifiableSet())); + + List tables = subject.getAnalyticsTables(params); + assertEquals(1, tables.size()); + + subject.removeUpdatedData(tables); + + org.mockito.ArgumentCaptor sql = org.mockito.ArgumentCaptor.forClass(String.class); + org.mockito.Mockito.verify(jdbcTemplate).execute(sql.capture()); + + assertTrue(sql.getValue().contains("using")); + assertTrue(sql.getValue().contains("ax.id =")); + assertTrue(!sql.getValue().contains("in (")); + } } From 8a832097d576d8237f67f721f04400ea3e27ec35 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 13 Jul 2026 19:33:46 +0200 Subject: [PATCH 05/31] feat: emit DELETE...USING for Doris in JdbcCompletenessTableManager.removeUpdatedData --- .../table/JdbcCompletenessTableManager.java | 50 +++-- ...JdbcCompletenessTableManagerDorisTest.java | 173 ++++++++++++++++++ 2 files changed, 206 insertions(+), 17 deletions(-) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java index 269fb2488531..8956f9ac03a2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java @@ -178,23 +178,39 @@ public boolean hasUpdatedLatestData(Date startDate, Date endDate) { public void removeUpdatedData(List tables) { AnalyticsTablePartition partition = getLatestTablePartition(tables); String sql = - replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - where ax.id in ( \ - select concat(ds.uid,'-',ps.iso,'-',ous.organisationunituid,'-',acs.categoryoptioncombouid) as id \ - from ${completedatasetregistration} cdr \ - inner join ${dataset} ds on cdr.datasetid=ds.datasetid \ - inner join analytics_rs_periodstructure ps on cdr.periodid=ps.periodid \ - inner join analytics_rs_orgunitstructure ous on cdr.sourceid=ous.organisationunitid \ - inner join analytics_rs_categorystructure acs on cdr.attributeoptioncomboid=acs.categoryoptioncomboid \ - where cdr.lastupdated >= '${startDate}' \ - and cdr.lastupdated < '${endDate}');""", - Map.of( - "tableName", quote(getAnalyticsTableType().getTableName()), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))); + sqlBuilder.requiresUniqueKeyAnalyticsTables() + ? replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + using ${completedatasetregistration} cdr \ + inner join ${dataset} ds on cdr.datasetid=ds.datasetid \ + inner join analytics_rs_periodstructure ps on cdr.periodid=ps.periodid \ + inner join analytics_rs_orgunitstructure ous on cdr.sourceid=ous.organisationunitid \ + inner join analytics_rs_categorystructure acs on cdr.attributeoptioncomboid=acs.categoryoptioncomboid \ + where ax.id = concat(ds.uid,'-',ps.iso,'-',ous.organisationunituid,'-',acs.categoryoptioncombouid) \ + and cdr.lastupdated >= '${startDate}' and cdr.lastupdated < '${endDate}';""", + Map.of( + "tableName", quote(getAnalyticsTableType().getTableName()), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))) + : replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + where ax.id in ( \ + select concat(ds.uid,'-',ps.iso,'-',ous.organisationunituid,'-',acs.categoryoptioncombouid) as id \ + from ${completedatasetregistration} cdr \ + inner join ${dataset} ds on cdr.datasetid=ds.datasetid \ + inner join analytics_rs_periodstructure ps on cdr.periodid=ps.periodid \ + inner join analytics_rs_orgunitstructure ous on cdr.sourceid=ous.organisationunitid \ + inner join analytics_rs_categorystructure acs on cdr.attributeoptioncomboid=acs.categoryoptioncomboid \ + where cdr.lastupdated >= '${startDate}' \ + and cdr.lastupdated < '${endDate}');""", + Map.of( + "tableName", quote(getAnalyticsTableType().getTableName()), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))); invokeTimeAndLog(sql, "Remove updated data values"); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java new file mode 100644 index 000000000000..2478ba7f3841 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.table; + +import static java.util.stream.Collectors.toUnmodifiableSet; +import static org.hisp.dhis.period.PeriodType.PERIOD_TYPES; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import org.hisp.dhis.analytics.AnalyticsTableHookService; +import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; +import org.hisp.dhis.analytics.partition.PartitionManager; +import org.hisp.dhis.analytics.table.model.AnalyticsTable; +import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; +import org.hisp.dhis.category.CategoryService; +import org.hisp.dhis.common.IdentifiableObjectManager; +import org.hisp.dhis.configuration.Configuration; +import org.hisp.dhis.configuration.ConfigurationService; +import org.hisp.dhis.dataapproval.DataApprovalLevelService; +import org.hisp.dhis.db.sql.DorisSqlBuilder; +import org.hisp.dhis.db.sql.SqlBuilder; +import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.period.PeriodDataProvider; +import org.hisp.dhis.resourcetable.ResourceTableService; +import org.hisp.dhis.setting.SystemSettings; +import org.hisp.dhis.setting.SystemSettingsProvider; +import org.joda.time.DateTime; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.springframework.jdbc.core.JdbcTemplate; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class JdbcCompletenessTableManagerDorisTest { + @Mock private IdentifiableObjectManager idObjectManager; + + @Mock private OrganisationUnitService organisationUnitService; + + @Mock private CategoryService categoryService; + + @Mock private SystemSettingsProvider settingsProvider; + + @Mock private SystemSettings settings; + + @Mock private DataApprovalLevelService dataApprovalLevelService; + + @Mock private ResourceTableService resourceTableService; + + @Mock private AnalyticsTableHookService analyticsTableHookService; + + @Mock private PartitionManager partitionManager; + + @Mock private JdbcTemplate jdbcTemplate; + + @Mock private AnalyticsTableSettings analyticsTableSettings; + + @Mock private PeriodDataProvider periodDataProvider; + + @Mock private ConfigurationService configurationService; + + @Mock private Configuration configuration; + + @Spy private SqlBuilder sqlBuilder = new DorisSqlBuilder("dhis2", "driver"); + + @InjectMocks private JdbcCompletenessTableManager subject; + + @BeforeEach + void setUp() { + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(settings.getLastSuccessfulResourceTablesUpdate()).thenReturn(new Date(0L)); + when(configurationService.getConfiguration()).thenReturn(configuration); + when(configuration.getDataOutputPeriodTypes()) + .thenReturn(PERIOD_TYPES.stream().collect(toUnmodifiableSet())); + } + + @Test + void testGetLatestAnalyticsTableHasUniqueKeyOnId() { + Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); + Date lastLatestPartitionUpdate = new DateTime(2019, 3, 1, 9, 0).toDate(); + Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); + + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().startTime(startTime).build().withLatestPartition(); + + List> queryResp = new ArrayList<>(); + queryResp.add(Map.of("datasetid", 1)); + + when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + .thenReturn(lastLatestPartitionUpdate); + when(jdbcTemplate.queryForList(anyString())).thenReturn(queryResp); + + List tables = subject.getAnalyticsTables(params); + + assertEquals(1, tables.size()); + AnalyticsTable table = tables.get(0); + assertTrue(table.hasPrimaryKey()); + assertEquals(List.of("id"), table.getPrimaryKey()); + assertTrue(sqlBuilder.createTable(table).contains("unique key (`id`)")); + } + + @Test + void testRemoveUpdatedDataUsesUsingJoinNotSubquery() { + Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); + Date lastLatestPartitionUpdate = new DateTime(2019, 3, 1, 9, 0).toDate(); + Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); + + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().startTime(startTime).build().withLatestPartition(); + + List> queryResp = new ArrayList<>(); + queryResp.add(Map.of("datasetid", 1)); + + when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + .thenReturn(lastLatestPartitionUpdate); + when(jdbcTemplate.queryForList(anyString())).thenReturn(queryResp); + + List tables = subject.getAnalyticsTables(params); + assertEquals(1, tables.size()); + + subject.removeUpdatedData(tables); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(jdbcTemplate).execute(sql.capture()); + + assertTrue(sql.getValue().contains("using")); + assertTrue(sql.getValue().contains("ax.id =")); + assertTrue(!sql.getValue().contains("in (")); + } +} From e1125673d9460f366b72976fa7564f2f970e75cd Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 13 Jul 2026 20:28:15 +0200 Subject: [PATCH 06/31] feat: give EVENT Doris analytics tables a unique key and DELETE...USING support --- .../table/JdbcEventAnalyticsTableManager.java | 110 ++++++++++++------ ...bcEventAnalyticsTableManagerDorisTest.java | 76 ++++++++++++ 2 files changed, 153 insertions(+), 33 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index d8c220b8b999..87442f2175e2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -111,6 +111,8 @@ public class JdbcEventAnalyticsTableManager extends AbstractEventJdbcTableManage static final String[] EXPORTABLE_EVENT_STATUSES = {"'COMPLETED'", "'ACTIVE'", "'SCHEDULE'"}; + private static final List PRIMARY_KEY = List.of("event"); + public JdbcEventAnalyticsTableManager( IdentifiableObjectManager idObjectManager, OrganisationUnitService organisationUnitService, @@ -195,7 +197,12 @@ private List getRegularAnalyticsTables( Collections.sort(yearsForPartitionTables); AnalyticsTable table = - new AnalyticsTable(getAnalyticsTableType(), getColumns(program), logged, program); + new AnalyticsTable( + getAnalyticsTableType(), + getColumns(program), + sqlBuilder.requiresUniqueKeyAnalyticsTables() ? PRIMARY_KEY : List.of(), + logged, + program); for (Integer year : yearsForPartitionTables) { List checks = getPartitionChecks(year, PartitionUtils.getEndDate(calendar, year)); @@ -251,7 +258,12 @@ private List getLatestAnalyticsTables(AnalyticsTableUpdateParams if (hasUpdatedData) { AnalyticsTable table = - new AnalyticsTable(getAnalyticsTableType(), getColumns(program), logged, program); + new AnalyticsTable( + getAnalyticsTableType(), + getColumns(program), + sqlBuilder.requiresUniqueKeyAnalyticsTables() ? PRIMARY_KEY : List.of(), + logged, + program); table.addTablePartition( List.of(), AnalyticsTablePartition.LATEST_PARTITION, startDate, endDate); tables.add(table); @@ -363,25 +375,42 @@ public void removeUpdatedData(List tables) { AnalyticsTablePartition partition = table.getLatestTablePartition(); String sql = null; Program program = table.getProgram(); + boolean uniqueKey = sqlBuilder.requiresUniqueKeyAnalyticsTables(); if (table.getProgram().isRegistration()) { sql = - replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - where ax.event in ( \ - select ev.uid \ - from ${trackerevent} ev \ - inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ - where en.programid = ${programId} \ - and ev.lastupdated >= '${startDate}' \ - and ev.lastupdated < '${endDate}');""", - Map.of( - "tableName", sqlBuilder.qualifyTable(table.getMainName()), - "programId", String.valueOf(program.getId()), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))); + uniqueKey + ? replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + using ${trackerevent} ev \ + inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ + where ax.event = ev.uid \ + and en.programid = ${programId} \ + and ev.lastupdated >= '${startDate}' \ + and ev.lastupdated < '${endDate}';""", + Map.of( + "tableName", quote(table.getMainName()), + "programId", String.valueOf(program.getId()), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))) + : replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + where ax.event in ( \ + select ev.uid \ + from ${trackerevent} ev \ + inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ + where en.programid = ${programId} \ + and ev.lastupdated >= '${startDate}' \ + and ev.lastupdated < '${endDate}');""", + Map.of( + "tableName", sqlBuilder.qualifyTable(table.getMainName()), + "programId", String.valueOf(program.getId()), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))); } else { if (isEmpty(program.getProgramStages()) || program.getProgramStages().size() > 1) { log.warn("Single event program {}, must have one stage", program.getUid()); @@ -392,21 +421,36 @@ where ax.event in ( \ ? program.getProgramStages().stream().toList().get(0).getId() : EMPTY); sql = - replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - where ax.event in ( \ - select ev.uid \ - from ${singleevent} ev \ - where ev.programstageid = ${programStageId} \ - and ev.lastupdated >= '${startDate}' \ - and ev.lastupdated < '${endDate}');""", - Map.of( - "tableName", sqlBuilder.qualifyTable(table.getMainName()), - "programStageId", String.valueOf(programStageId), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))); + uniqueKey + ? replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + using ${singleevent} ev \ + where ax.event = ev.uid \ + and ev.programstageid = ${programStageId} \ + and ev.lastupdated >= '${startDate}' \ + and ev.lastupdated < '${endDate}';""", + Map.of( + "tableName", quote(table.getMainName()), + "programStageId", String.valueOf(programStageId), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))) + : replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + where ax.event in ( \ + select ev.uid \ + from ${singleevent} ev \ + where ev.programstageid = ${programStageId} \ + and ev.lastupdated >= '${startDate}' \ + and ev.lastupdated < '${endDate}');""", + Map.of( + "tableName", sqlBuilder.qualifyTable(table.getMainName()), + "programStageId", String.valueOf(programStageId), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java index 99d26c4b8ebe..f871b1aa0362 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java @@ -49,12 +49,17 @@ import static org.hisp.dhis.test.TestBase.createProgramStage; import static org.hisp.dhis.test.TestBase.createProgramTrackedEntityAttribute; import static org.hisp.dhis.test.TestBase.createTrackedEntityAttribute; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.time.LocalDate; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.Set; import org.hisp.dhis.analytics.AggregationType; import org.hisp.dhis.analytics.AnalyticsTableType; @@ -83,15 +88,19 @@ import org.hisp.dhis.program.Program; import org.hisp.dhis.program.ProgramStage; import org.hisp.dhis.program.ProgramTrackedEntityAttribute; +import org.hisp.dhis.program.ProgramType; import org.hisp.dhis.resourcetable.ResourceTableService; import org.hisp.dhis.setting.SystemSettings; import org.hisp.dhis.setting.SystemSettingsProvider; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.jdbc.core.JdbcTemplate; @@ -291,4 +300,71 @@ void verifyGetTableWithDataElements() { private String toSelectExpression(String template, String uid) { return String.format(template, uid, uid, uid); } + + @Test + void testRegularAnalyticsTableHasUniqueKeyOnEvent() { + Program program = createProgram('A'); + program.setProgramType(ProgramType.WITH_REGISTRATION); + + when(configurationService.getConfiguration()).thenReturn(configuration); + when(configuration.getDataOutputPeriodTypes()) + .thenReturn(PERIOD_TYPES.stream().collect(toUnmodifiableSet())); + when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program)); + when(periodDataProvider.getAvailableYears(DATABASE)).thenReturn(List.of(2018, now().getYear())); + when(jdbcTemplate.queryForList(Mockito.anyString(), Mockito.eq(Integer.class))) + .thenReturn(List.of(2018)); + + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .lastYears(2) + .startTime(START_TIME) + .today(today) + .build(); + + List tables = subject.getAnalyticsTables(params); + + assertThat(tables, hasSize(1)); + AnalyticsTable table = tables.get(0); + assertTrue(table.hasPrimaryKey()); + assertEquals(List.of("event"), table.getPrimaryKey()); + assertTrue(sqlBuilder.createTable(table).contains("unique key (`event`)")); + } + + @Test + @DisplayName("removeUpdatedData emits DELETE...USING for a registration program on Doris") + void testRemoveUpdatedDataUsesUsingJoinForRegistration() { + Program program = createProgram('A'); + program.setProgramType(ProgramType.WITH_REGISTRATION); + + Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); + Date lastLatestPartitionUpdate = new DateTime(2019, 3, 1, 9, 0).toDate(); + Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); + + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().startTime(startTime).build().withLatestPartition(); + + List> queryResp = new ArrayList<>(); + queryResp.add(Map.of("eventid", 1)); + + when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + .thenReturn(lastLatestPartitionUpdate); + when(jdbcTemplate.queryForList(Mockito.anyString())).thenReturn(queryResp); + when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program)); + when(configurationService.getConfiguration()).thenReturn(configuration); + when(configuration.getDataOutputPeriodTypes()) + .thenReturn(PERIOD_TYPES.stream().collect(toUnmodifiableSet())); + + List tables = subject.getAnalyticsTables(params); + assertThat(tables, hasSize(1)); + + subject.removeUpdatedData(tables); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(jdbcTemplate).execute(sql.capture()); + + assertTrue(sql.getValue().contains("using")); + assertTrue(sql.getValue().contains("ax.event = ev.uid")); + assertTrue(!sql.getValue().contains("in (")); + } } From 3b732317c7b134b86e96211ef31d0c6063ecd900 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 13 Jul 2026 20:39:06 +0200 Subject: [PATCH 07/31] test: cover single-event Doris removeUpdatedData branch (review follow-up) --- ...bcEventAnalyticsTableManagerDorisTest.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java index f871b1aa0362..b6c45e7ef056 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java @@ -367,4 +367,46 @@ void testRemoveUpdatedDataUsesUsingJoinForRegistration() { assertTrue(sql.getValue().contains("ax.event = ev.uid")); assertTrue(!sql.getValue().contains("in (")); } + + @Test + @DisplayName("removeUpdatedData emits DELETE...USING for a single event program on Doris") + void testRemoveUpdatedDataUsesUsingJoinForSingleEvent() { + ProgramStage psA = new ProgramStage(); + psA.setId(123456); + + Program program = createProgram('A'); + program.setProgramType(ProgramType.WITHOUT_REGISTRATION); + program.setProgramStages(Set.of(psA)); + + Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); + Date lastLatestPartitionUpdate = new DateTime(2019, 3, 1, 9, 0).toDate(); + Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); + + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().startTime(startTime).build().withLatestPartition(); + + List> queryResp = new ArrayList<>(); + queryResp.add(Map.of("eventid", 1)); + + when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + .thenReturn(lastLatestPartitionUpdate); + when(jdbcTemplate.queryForList(Mockito.anyString())).thenReturn(queryResp); + when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program)); + when(configurationService.getConfiguration()).thenReturn(configuration); + when(configuration.getDataOutputPeriodTypes()) + .thenReturn(PERIOD_TYPES.stream().collect(toUnmodifiableSet())); + + List tables = subject.getAnalyticsTables(params); + assertThat(tables, hasSize(1)); + + subject.removeUpdatedData(tables); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(jdbcTemplate).execute(sql.capture()); + + assertTrue(sql.getValue().contains("using")); + assertTrue(sql.getValue().contains("ax.event =")); + assertTrue(!sql.getValue().contains("in (")); + } } From 94a3c852ced258bf40b02dbbe522d545e2a8a572 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 13 Jul 2026 20:56:25 +0200 Subject: [PATCH 08/31] feat: gate continuous-analytics delete/swap on supportsContinuousAnalytics --- .../table/AbstractJdbcTableManager.java | 7 ++ .../table/DefaultAnalyticsTableService.java | 2 +- .../table/AnalyticsTableServiceTest.java | 84 +++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 48f9a4c6dd33..f21900ecdde7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -246,6 +246,13 @@ public void swapTable(AnalyticsTableUpdateParams params, AnalyticsTable table) { if (!sqlBuilder.supportsDeclarativePartitioning()) { swappedPartitions.forEach( partition -> swapParentTable(partition, table.getName(), table.getMainName())); + } else if (sqlBuilder.supportsContinuousAnalytics()) { + // For databases with declarative partitioning that support continuous analytics (e.g. + // Doris): staging data must be inserted directly into the main table since there is no + // inheritance-based partition attachment. Stale rows are already removed by + // removeUpdatedData(). + String fromTable = sqlBuilder.qualifyTable(table.getName()); + jdbcTemplate.execute(sqlBuilder.insertIntoSelectFrom(table.fromStaging(), fromTable)); } dropTable(table); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index 94fb49fc34b9..cbc975b37a03 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -169,7 +169,7 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { clock.logTime("Analyzed tables"); } - if (params.isLatestUpdate()) { + if (params.isLatestUpdate() && sqlBuilder.supportsContinuousAnalytics()) { progress.startingStage( format("Removing updated and deleted data: '{}'", tableType), SKIP_STAGE); progress.runStage(() -> tableManager.removeUpdatedData(tables)); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java index cfae18cc1339..b7b11f8483b5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java @@ -32,15 +32,24 @@ import static org.hisp.dhis.db.model.DataType.DOUBLE; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.List; +import org.hisp.dhis.analytics.AnalyticsTableManager; import org.hisp.dhis.analytics.AnalyticsTableType; +import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; import org.hisp.dhis.analytics.table.model.AnalyticsTable; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; import org.hisp.dhis.analytics.table.model.AnalyticsTablePartition; +import org.hisp.dhis.dataelement.DataElementService; import org.hisp.dhis.db.model.Logged; import org.hisp.dhis.db.sql.SqlBuilder; +import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.resourcetable.ResourceTableService; +import org.hisp.dhis.scheduling.JobProgress; import org.hisp.dhis.setting.SystemSettings; import org.hisp.dhis.setting.SystemSettingsProvider; import org.joda.time.DateTime; @@ -49,12 +58,23 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; /** * @author Lars Helge Overland */ @ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) class AnalyticsTableServiceTest { + @Mock private AnalyticsTableManager tableManager; + + @Mock private OrganisationUnitService organisationUnitService; + + @Mock private DataElementService dataElementService; + + @Mock private ResourceTableService resourceTableService; + @Mock private SystemSettingsProvider settingsProvider; @Mock private SystemSettings settings; @@ -113,4 +133,68 @@ void testGetParallelJobsB() { assertEquals(8, tableService.getParallelJobs()); } + + @Test + void testRemoveUpdatedDataSkippedWhenContinuousAnalyticsNotSupported() { + AnalyticsTable table = latestPartitionTableFixture(); + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build() + .withLatestPartition(); + + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(tableManager.getAnalyticsTableType()).thenReturn(AnalyticsTableType.DATA_VALUE); + when(tableManager.validState()).thenReturn(true); + when(tableManager.getAnalyticsTables(params)).thenReturn(List.of(table)); + when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(false); + when(sqlBuilder.requiresIndexesForAnalytics()).thenReturn(false); + when(sqlBuilder.supportsAnalyze()).thenReturn(false); + when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(false); + + tableService.create(params, JobProgress.noop()); + + verify(tableManager, never()).removeUpdatedData(anyList()); + } + + @Test + void testRemoveUpdatedDataRunWhenContinuousAnalyticsSupported() { + AnalyticsTable table = latestPartitionTableFixture(); + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build() + .withLatestPartition(); + + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(tableManager.getAnalyticsTableType()).thenReturn(AnalyticsTableType.DATA_VALUE); + when(tableManager.validState()).thenReturn(true); + when(tableManager.getAnalyticsTables(params)).thenReturn(List.of(table)); + when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(false); + when(sqlBuilder.requiresIndexesForAnalytics()).thenReturn(false); + when(sqlBuilder.supportsAnalyze()).thenReturn(false); + when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); + + tableService.create(params, JobProgress.noop()); + + verify(tableManager).removeUpdatedData(List.of(table)); + } + + private AnalyticsTable latestPartitionTableFixture() { + List columns = + List.of( + AnalyticsTableColumn.builder() + .name("dx") + .dataType(TEXT) + .selectExpression("dx") + .build()); + AnalyticsTable table = + new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, List.of(), Logged.UNLOGGED); + table.addTablePartition( + List.of(), + AnalyticsTablePartition.LATEST_PARTITION, + new DateTime(2020, 1, 1, 0, 0).toDate(), + new DateTime(2020, 3, 1, 10, 0).toDate()); + return table; + } } From d44c2115caf7588098c5127b234428ed9e77333a Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 13 Jul 2026 21:30:08 +0200 Subject: [PATCH 09/31] test: cover swapTable() Doris insert-into-main branch and ClickHouse no-op negative case --- ...bcAnalyticsTableManagerClickHouseTest.java | 159 ++++++++++++++++++ .../JdbcAnalyticsTableManagerDorisTest.java | 40 +++++ 2 files changed, 199 insertions(+) create mode 100644 dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java new file mode 100644 index 000000000000..28e35b861b13 --- /dev/null +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.analytics.table; + +import static org.hisp.dhis.db.model.DataType.TEXT; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import java.util.Date; +import java.util.List; +import java.util.Map; +import org.hisp.dhis.analytics.AnalyticsTableHookService; +import org.hisp.dhis.analytics.AnalyticsTableType; +import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; +import org.hisp.dhis.analytics.partition.PartitionManager; +import org.hisp.dhis.analytics.table.model.AnalyticsTable; +import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; +import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; +import org.hisp.dhis.category.CategoryService; +import org.hisp.dhis.common.IdentifiableObjectManager; +import org.hisp.dhis.configuration.ConfigurationService; +import org.hisp.dhis.dataapproval.DataApprovalLevelService; +import org.hisp.dhis.dataelement.DataElementService; +import org.hisp.dhis.db.model.Logged; +import org.hisp.dhis.db.sql.ClickHouseSqlBuilder; +import org.hisp.dhis.db.sql.SqlBuilder; +import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.period.PeriodDataProvider; +import org.hisp.dhis.resourcetable.ResourceTableService; +import org.hisp.dhis.setting.SystemSettings; +import org.hisp.dhis.setting.SystemSettingsProvider; +import org.joda.time.DateTime; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.springframework.jdbc.core.JdbcTemplate; + +/** + * Covers the {@code swapTable()} branch for databases with declarative partitioning that do NOT + * support continuous analytics (e.g. ClickHouse): staging data must not be inserted into the main + * table, the staging table is simply dropped, same as before the Doris continuous-analytics + * pipeline was wired up. + */ +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class JdbcAnalyticsTableManagerClickHouseTest { + @Mock private IdentifiableObjectManager idObjectManager; + + @Mock private OrganisationUnitService organisationUnitService; + + @Mock private CategoryService categoryService; + + @Mock private SystemSettingsProvider settingsProvider; + + @Mock private SystemSettings settings; + + @Mock private DataApprovalLevelService dataApprovalLevelService; + + @Mock private ResourceTableService resourceTableService; + + @Mock private AnalyticsTableHookService analyticsTableHookService; + + @Mock private PartitionManager partitionManager; + + @Mock private JdbcTemplate jdbcTemplate; + + @Mock private AnalyticsTableSettings analyticsTableSettings; + + @Mock private PeriodDataProvider periodDataProvider; + + @Mock private ConfigurationService configurationService; + + @Mock private DataElementService dataElementService; + + @Spy private SqlBuilder sqlBuilder = new ClickHouseSqlBuilder("dhis2"); + + @InjectMocks private JdbcAnalyticsTableManager subject; + + @BeforeEach + void setUp() { + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(settings.getLastSuccessfulResourceTablesUpdate()).thenReturn(new Date(0L)); + } + + @Test + void testSwapTableDoesNotInsertStagingDataWhenContinuousAnalyticsNotSupported() { + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build() + .withLatestPartition(); + + List columns = + List.of( + AnalyticsTableColumn.builder() + .name("dx") + .dataType(TEXT) + .selectExpression("dx") + .build()); + AnalyticsTable table = + new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, List.of(), Logged.UNLOGGED); + + // Main table already exists, and params.isPartialUpdate() (via withLatestPartition()) plus + // AnalyticsTableType.DATA_VALUE.isLatestPartition()==true together push swapTable() into the + // skipMasterTable branch. ClickHouse supports declarative partitioning but not continuous + // analytics, so neither the reparenting branch nor the new insert-into-main branch should + // fire here. + when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) + .thenReturn(List.of(Map.of("table_name", "analytics"))); + + subject.swapTable(params, table); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + Mockito.verify(jdbcTemplate, Mockito.times(1)).execute(sql.capture()); + + String statement = sql.getValue(); + assertFalse( + statement.startsWith("insert into"), () -> "Unexpected insert statement: " + statement); + assertTrue( + statement.contains("drop table"), + () -> "Expected a drop-table statement, got: " + statement); + } +} diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index 3fd1a7e1201a..74273c837bcd 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -30,6 +30,7 @@ package org.hisp.dhis.analytics.table; import static java.util.stream.Collectors.toUnmodifiableSet; +import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.period.PeriodType.PERIOD_TYPES; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -40,9 +41,11 @@ import java.util.List; import java.util.Map; import org.hisp.dhis.analytics.AnalyticsTableHookService; +import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsTable; +import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; import org.hisp.dhis.common.IdentifiableObjectManager; @@ -50,6 +53,7 @@ import org.hisp.dhis.configuration.ConfigurationService; import org.hisp.dhis.dataapproval.DataApprovalLevelService; import org.hisp.dhis.dataelement.DataElementService; +import org.hisp.dhis.db.model.Logged; import org.hisp.dhis.db.sql.DorisSqlBuilder; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitService; @@ -167,4 +171,40 @@ void testRemoveUpdatedDataUsesUsingJoinNotSubquery() { assertTrue(sql.getValue().contains("ax.id =")); assertTrue(!sql.getValue().contains("in (")); } + + @Test + void testSwapTableInsertsStagingDataIntoMainTableWhenSkippingMasterTable() { + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build() + .withLatestPartition(); + + List columns = + List.of( + AnalyticsTableColumn.builder() + .name("dx") + .dataType(TEXT) + .selectExpression("dx") + .build()); + AnalyticsTable table = + new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, List.of(), Logged.UNLOGGED); + + // Main table already exists, and params.isPartialUpdate() (via withLatestPartition()) plus + // AnalyticsTableType.DATA_VALUE.isLatestPartition()==true together push swapTable() into the + // skipMasterTable branch. + when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) + .thenReturn(List.of(Map.of("table_name", "analytics"))); + + subject.swapTable(params, table); + + org.mockito.ArgumentCaptor sql = org.mockito.ArgumentCaptor.forClass(String.class); + org.mockito.Mockito.verify(jdbcTemplate, org.mockito.Mockito.times(2)).execute(sql.capture()); + + List statements = sql.getAllValues(); + assertTrue( + statements.stream() + .anyMatch(s -> s.startsWith("insert into `analytics`") && s.contains("analytics_temp")), + () -> "Expected an insert-into-main-from-staging statement, got: " + statements); + } } From 6f92575f67f5a730c9f24c04ed7313c6e098de68 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Tue, 14 Jul 2026 07:18:05 +0200 Subject: [PATCH 10/31] fix: Doris CREATE TABLE key-column ordering and swapTable() staging table catalog qualification - DorisSqlBuilder.createTable() now reorders the rendered column list so primary key columns lead, since Doris requires key columns to be an ordered leading prefix of the schema. Aggregate and completeness analytics tables (where id is not already column 0) would otherwise fail CREATE TABLE. - AbstractJdbcTableManager.swapTable()'s Doris insert-into-main branch now references the staging table via quote() instead of qualifyTable(), since the staging table is a native Doris table, not a federated one (same fix already applied to removeUpdatedData() in earlier commits). --- .../table/AbstractJdbcTableManager.java | 2 +- .../JdbcAnalyticsTableManagerDorisTest.java | 11 +++++- .../org/hisp/dhis/db/sql/DorisSqlBuilder.java | 35 ++++++++++++++++++- .../hisp/dhis/db/sql/DorisSqlBuilderTest.java | 35 +++++++++++++++++++ 4 files changed, 80 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index f21900ecdde7..3aff3d4cfed7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -251,7 +251,7 @@ public void swapTable(AnalyticsTableUpdateParams params, AnalyticsTable table) { // Doris): staging data must be inserted directly into the main table since there is no // inheritance-based partition attachment. Stale rows are already removed by // removeUpdatedData(). - String fromTable = sqlBuilder.qualifyTable(table.getName()); + String fromTable = sqlBuilder.quote(table.getName()); jdbcTemplate.execute(sqlBuilder.insertIntoSelectFrom(table.fromStaging(), fromTable)); } dropTable(table); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index 74273c837bcd..85c534d5d6c4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -204,7 +204,16 @@ void testSwapTableInsertsStagingDataIntoMainTableWhenSkippingMasterTable() { List statements = sql.getAllValues(); assertTrue( statements.stream() - .anyMatch(s -> s.startsWith("insert into `analytics`") && s.contains("analytics_temp")), + .anyMatch( + s -> + s.startsWith("insert into `analytics`") && s.contains("from `analytics_temp`")), () -> "Expected an insert-into-main-from-staging statement, got: " + statements); + // The staging table is a native Doris table, not a federated one, so the FROM reference must + // NOT be qualified with the federated catalog prefix that qualifyTable() would add. + assertTrue( + statements.stream().noneMatch(s -> s.contains("dhis2.public.")), + () -> + "Staging table reference must not carry the federated catalog prefix, got: " + + statements); } } diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index 570f896844cc..90722f0662b4 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -29,6 +29,7 @@ */ package org.hisp.dhis.db.sql; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.List; @@ -380,7 +381,7 @@ public String createTable(Table table) { // Columns if (table.hasColumns()) { - String columns = toCommaSeparated(table.getColumns(), this::toColumnString); + String columns = toCommaSeparated(getOrderedColumns(table), this::toColumnString); sql.append("(").append(columns).append(") engine = olap "); } @@ -457,6 +458,38 @@ private String generatePartitionClause(List partitions) { return partitionClause.substring(0, partitionClause.length() - 1) + ") "; } + /** + * Returns the table's columns ordered so that primary key columns appear first, in their declared + * order, followed by the remaining columns in their original order. Doris requires key columns + * (for both the unique key and duplicate key models) to be an ordered leading prefix of the table + * schema. + * + * @param table the {@link Table}. + * @return the ordered list of {@link Column}. + */ + private List getOrderedColumns(Table table) { + if (!table.hasPrimaryKey()) { + return table.getColumns(); + } + + List primaryKey = table.getPrimaryKey(); + List keyColumns = new ArrayList<>(); + List otherColumns = new ArrayList<>(); + + for (Column column : table.getColumns()) { + if (primaryKey.contains(column.getName())) { + keyColumns.add(column); + } else { + otherColumns.add(column); + } + } + + List orderedColumns = new ArrayList<>(keyColumns); + orderedColumns.addAll(otherColumns); + + return orderedColumns; + } + /** * Returns a column definition string. * diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java index aad3633cb7ef..1776466fd9c7 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java @@ -99,6 +99,24 @@ private Table getTableD() { return new Table("immunization", columns, List.of(), sortKey, List.of(), Logged.LOGGED); } + /** + * A table whose primary key column ("id") is not the first column, mirroring the aggregate + * (JdbcAnalyticsTableManager) and completeness (JdbcCompletenessTableManager) analytics tables, + * where "id" is not a leading column. + */ + private Table getTableE() { + List columns = + List.of( + new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL), + new Column("period", DataType.VARCHAR_50, Nullable.NOT_NULL), + new Column("id", DataType.BIGINT, Nullable.NOT_NULL), + new Column("value", DataType.DOUBLE)); + + List primaryKey = List.of("id"); + + return new Table("immunization", columns, primaryKey, Logged.LOGGED); + } + // Data types @Test @@ -238,6 +256,23 @@ distributed by hash(`id`) \ assertEquals(expected, sqlBuilder.createTable(table)); } + @Test + void testCreateTableOrdersPrimaryKeyColumnFirst() { + Table table = getTableE(); + + String expected = + """ + create table `immunization` (`id` bigint not null,`data` char(11) not null,\ + `period` varchar(200) not null,`value` double null) \ + engine = olap \ + unique key (`id`) \ + distributed by hash(`id`) \ + buckets 10 \ + properties ("replication_num" = "1");"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + @Test void testRenameTable() { Table table = getTableA(); From 758de6f6dc9c1a2b49de3ae54eebdb116041a850 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Tue, 14 Jul 2026 07:36:22 +0200 Subject: [PATCH 11/31] fix: composite unique key (id/event, year) for Doris Merge-on-Write partition requirement Doris's UNIQUE KEY (Merge-on-Write) model requires the partition column to be part of the key. Analytics tables are partitioned by year, so the key must be composite: (id, year) for aggregate/completeness, (event, year) for event. Confirmed working end-to-end against a real Doris 3.0.7 instance. Also fixes DorisSqlBuilder.getOrderedColumns(), which built the leading key-column block from natural column order rather than the primary key's declared order. For a composite key, Doris requires the leading schema columns to match the key clause's declared order exactly -- for the aggregate/completeness tables, year appears before id in natural column order (FIXED_COLS), which would have produced a mismatched schema. removeUpdatedData() DELETE...USING SQL is unchanged in all three managers -- it matches on id/event alone and was confirmed to still correctly find and delete rows regardless of partition. --- .../table/JdbcAnalyticsTableManager.java | 2 +- .../table/JdbcCompletenessTableManager.java | 2 +- .../table/JdbcEventAnalyticsTableManager.java | 2 +- .../JdbcAnalyticsTableManagerDorisTest.java | 4 +-- ...JdbcCompletenessTableManagerDorisTest.java | 4 +-- ...bcEventAnalyticsTableManagerDorisTest.java | 4 +-- .../org/hisp/dhis/db/sql/DorisSqlBuilder.java | 19 +++++----- .../hisp/dhis/db/sql/DorisSqlBuilderTest.java | 35 +++++++++++++++++++ 8 files changed, 53 insertions(+), 19 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index ba2ba54d17d1..8a8dbed3ef08 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -168,7 +168,7 @@ public class JdbcAnalyticsTableManager extends AbstractJdbcTableManager { private static final List SORT_KEY = List.of("dx", "co"); - private static final List PRIMARY_KEY = List.of("id"); + private static final List PRIMARY_KEY = List.of("id", "year"); public JdbcAnalyticsTableManager( IdentifiableObjectManager idObjectManager, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java index 8956f9ac03a2..5a5fc3c79e48 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java @@ -96,7 +96,7 @@ public class JdbcCompletenessTableManager extends AbstractJdbcTableManager { private static final List SORT_KEY = List.of("dx"); - private static final List PRIMARY_KEY = List.of("id"); + private static final List PRIMARY_KEY = List.of("id", "year"); public JdbcCompletenessTableManager( IdentifiableObjectManager idObjectManager, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 87442f2175e2..686e373d4c88 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -111,7 +111,7 @@ public class JdbcEventAnalyticsTableManager extends AbstractEventJdbcTableManage static final String[] EXPORTABLE_EVENT_STATUSES = {"'COMPLETED'", "'ACTIVE'", "'SCHEDULE'"}; - private static final List PRIMARY_KEY = List.of("event"); + private static final List PRIMARY_KEY = List.of("event", "year"); public JdbcEventAnalyticsTableManager( IdentifiableObjectManager idObjectManager, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index 85c534d5d6c4..d0e62cafb8e4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -135,8 +135,8 @@ void testGetRegularAnalyticsTableHasUniqueKeyOnId() { assertEquals(1, tables.size()); AnalyticsTable table = tables.get(0); assertTrue(table.hasPrimaryKey()); - assertEquals(List.of("id"), table.getPrimaryKey()); - assertTrue(sqlBuilder.createTable(table).contains("unique key (`id`)")); + assertEquals(List.of("id", "year"), table.getPrimaryKey()); + assertTrue(sqlBuilder.createTable(table).contains("unique key (`id`,`year`)")); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java index 2478ba7f3841..1da132d70d42 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java @@ -137,8 +137,8 @@ void testGetLatestAnalyticsTableHasUniqueKeyOnId() { assertEquals(1, tables.size()); AnalyticsTable table = tables.get(0); assertTrue(table.hasPrimaryKey()); - assertEquals(List.of("id"), table.getPrimaryKey()); - assertTrue(sqlBuilder.createTable(table).contains("unique key (`id`)")); + assertEquals(List.of("id", "year"), table.getPrimaryKey()); + assertTrue(sqlBuilder.createTable(table).contains("unique key (`id`,`year`)")); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java index b6c45e7ef056..afc6506c2eb8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java @@ -326,8 +326,8 @@ void testRegularAnalyticsTableHasUniqueKeyOnEvent() { assertThat(tables, hasSize(1)); AnalyticsTable table = tables.get(0); assertTrue(table.hasPrimaryKey()); - assertEquals(List.of("event"), table.getPrimaryKey()); - assertTrue(sqlBuilder.createTable(table).contains("unique key (`event`)")); + assertEquals(List.of("event", "year"), table.getPrimaryKey()); + assertTrue(sqlBuilder.createTable(table).contains("unique key (`event`,`year`)")); } @Test diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index 90722f0662b4..f2405e777daa 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -472,17 +472,16 @@ private List getOrderedColumns(Table table) { return table.getColumns(); } - List primaryKey = table.getPrimaryKey(); - List keyColumns = new ArrayList<>(); - List otherColumns = new ArrayList<>(); + Map columnsByName = + table.getColumns().stream().collect(Collectors.toMap(Column::getName, c -> c)); - for (Column column : table.getColumns()) { - if (primaryKey.contains(column.getName())) { - keyColumns.add(column); - } else { - otherColumns.add(column); - } - } + List keyColumns = + table.getPrimaryKey().stream().map(columnsByName::get).collect(Collectors.toList()); + + List otherColumns = + table.getColumns().stream() + .filter(c -> !table.getPrimaryKey().contains(c.getName())) + .collect(Collectors.toList()); List orderedColumns = new ArrayList<>(keyColumns); orderedColumns.addAll(otherColumns); diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java index 1776466fd9c7..eebeb6225fa4 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java @@ -117,6 +117,24 @@ private Table getTableE() { return new Table("immunization", columns, primaryKey, Logged.LOGGED); } + /** + * A table with a composite primary key ("id", "year") where the natural column order (`year` + * before `id`) does not match the declared key order (`id` before `year`) — mirrors the + * aggregate/completeness analytics tables, where `year` is a `FIXED_COLS` entry appended before + * the `id` column, but the Doris unique key clause must declare `id` first. + */ + private Table getTableF() { + List columns = + List.of( + new Column("year", DataType.INTEGER, Nullable.NOT_NULL), + new Column("id", DataType.BIGINT, Nullable.NOT_NULL), + new Column("data", DataType.CHARACTER_11, Nullable.NOT_NULL)); + + List primaryKey = List.of("id", "year"); + + return new Table("immunization", columns, primaryKey, Logged.LOGGED); + } + // Data types @Test @@ -273,6 +291,23 @@ distributed by hash(`id`) \ assertEquals(expected, sqlBuilder.createTable(table)); } + @Test + void testCreateTableOrdersCompositePrimaryKeyByDeclaredKeyOrder() { + Table table = getTableF(); + + String expected = + """ + create table `immunization` (`id` bigint not null,`year` int not null,\ + `data` char(11) not null) \ + engine = olap \ + unique key (`id`,`year`) \ + distributed by hash(`id`) \ + buckets 10 \ + properties ("replication_num" = "1");"""; + + assertEquals(expected, sqlBuilder.createTable(table)); + } + @Test void testRenameTable() { Table table = getTableA(); From e4a6c36a8279885f91bf0575bf221caf00e8dd06 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Tue, 14 Jul 2026 07:43:41 +0200 Subject: [PATCH 12/31] fix: narrow swapTable() Doris insert gate to isLatestUpdate(), not just isPartialUpdate() skipMasterTable is computed from params.isPartialUpdate() (lastYears != null || isLatestUpdate()), not isLatestUpdate() alone, so the new Doris insert-into-main branch was also firing for lastYears=1..N (a normal bounded-years rebuild), not just lastYears=0 (the continuous/latest-partition case this plan is scoped to). removeUpdatedData() never runs for lastYears=N>0, so the branch was merging staging data into main without ever purging rows deleted at source -- a real regression versus the pre-PR no-op-drop-only behavior for that path. Gate the insert branch on params.isLatestUpdate() as well, restoring the exact pre-PR (safe, no-op) behavior for lastYears=N>0 on Doris while keeping the insert working for lastYears=0. --- .../table/AbstractJdbcTableManager.java | 2 +- .../JdbcAnalyticsTableManagerDorisTest.java | 43 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 3aff3d4cfed7..f1c3737296c4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -246,7 +246,7 @@ public void swapTable(AnalyticsTableUpdateParams params, AnalyticsTable table) { if (!sqlBuilder.supportsDeclarativePartitioning()) { swappedPartitions.forEach( partition -> swapParentTable(partition, table.getName(), table.getMainName())); - } else if (sqlBuilder.supportsContinuousAnalytics()) { + } else if (params.isLatestUpdate() && sqlBuilder.supportsContinuousAnalytics()) { // For databases with declarative partitioning that support continuous analytics (e.g. // Doris): staging data must be inserted directly into the main table since there is no // inheritance-based partition attachment. Stale rows are already removed by diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index d0e62cafb8e4..1783b0848377 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -33,6 +33,7 @@ import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.period.PeriodType.PERIOD_TYPES; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; @@ -216,4 +217,46 @@ void testSwapTableInsertsStagingDataIntoMainTableWhenSkippingMasterTable() { "Staging table reference must not carry the federated catalog prefix, got: " + statements); } + + @Test + void testSwapTableDoesNotInsertStagingDataForBoundedYearsUpdate() { + // lastYears(1) (a normal bounded-years rebuild) makes isPartialUpdate()==true but + // isLatestUpdate()==false -- distinct from the "latest partition" (lastYears=0) case the + // insert-into-main branch is scoped to. removeUpdatedData() never runs for this case (it's + // gated on isLatestUpdate()), so merging staging data into main here would leave deleted rows + // behind forever. The insert branch must not fire. + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .lastYears(1) + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build(); + + List columns = + List.of( + AnalyticsTableColumn.builder() + .name("dx") + .dataType(TEXT) + .selectExpression("dx") + .build()); + AnalyticsTable table = + new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, List.of(), Logged.UNLOGGED); + + // Same skipMasterTable=true preconditions as the positive test: main table already exists, + // params.isPartialUpdate()==true (via lastYears(1)), and DATA_VALUE.isLatestPartition()==true. + when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) + .thenReturn(List.of(Map.of("table_name", "analytics"))); + + subject.swapTable(params, table); + + org.mockito.ArgumentCaptor sql = org.mockito.ArgumentCaptor.forClass(String.class); + org.mockito.Mockito.verify(jdbcTemplate, org.mockito.Mockito.times(1)).execute(sql.capture()); + + String statement = sql.getValue(); + assertFalse( + statement.startsWith("insert into"), + () -> "Unexpected insert statement for a bounded-years update: " + statement); + assertTrue( + statement.contains("drop table"), + () -> "Expected a drop-table statement only, got: " + statement); + } } From e27e9a41b0809d6aa2297d0b90022fcd9a72be18 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Tue, 14 Jul 2026 08:36:30 +0200 Subject: [PATCH 13/31] fix: bound the id key column's type on Doris (string not allowed as key column) Real Doris CI (Sierra Leone e2e run) failed with errCode = 2, "String Type should not be used in key column[id]" while creating the DATA_VALUE staging table -- aborting the whole analytics table generation job, including everything after DATA_VALUE in iteration order. The synthetic id column (concat(...) of several dimension UIDs) was defined as DataType.TEXT, which DorisSqlBuilder maps to the unbounded "string" type. Doris rejects unbounded string/text as a key column on any key model. event/dx don't have this problem -- they're already CHARACTER_11 (bounded char(11)). Give id a bounded VARCHAR_255 (Doris: varchar(1020)) type specifically when sqlBuilder.requiresUniqueKeyAnalyticsTables() is true (Doris only), leaving Postgres/ClickHouse's TEXT unchanged, in both JdbcAnalyticsTableManager and JdbcCompletenessTableManager. --- .../table/JdbcAnalyticsTableManager.java | 2 +- .../table/JdbcCompletenessTableManager.java | 3 +- .../JdbcAnalyticsTableManagerDorisTest.java | 15 ++++++++- .../table/JdbcAnalyticsTableManagerTest.java | 31 +++++++++++++++++++ ...JdbcCompletenessTableManagerDorisTest.java | 16 +++++++++- .../hisp/dhis/db/sql/DorisSqlBuilderTest.java | 5 +++ 6 files changed, 68 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index 8a8dbed3ef08..221b30fd587b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -554,7 +554,7 @@ private List getColumns(AnalyticsTableUpdateParams params) columns.add( AnalyticsTableColumn.builder() .name("id") - .dataType(TEXT) + .dataType(sqlBuilder.requiresUniqueKeyAnalyticsTables() ? VARCHAR_255 : TEXT) .selectExpression(idColAlias) .build()); columns.addAll(getDataElementGroupSetColumns()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java index 5a5fc3c79e48..d4907d3d32b6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManager.java @@ -41,6 +41,7 @@ import static org.hisp.dhis.db.model.DataType.INTEGER; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.db.model.DataType.TIMESTAMP; +import static org.hisp.dhis.db.model.DataType.VARCHAR_255; import static org.hisp.dhis.db.model.constraint.Nullable.NOT_NULL; import static org.hisp.dhis.util.DateUtils.SECONDS_PER_DAY; import static org.hisp.dhis.util.DateUtils.toLongDate; @@ -289,7 +290,7 @@ private List getColumns() { columns.add( AnalyticsTableColumn.builder() .name("id") - .dataType(TEXT) + .dataType(sqlBuilder.requiresUniqueKeyAnalyticsTables() ? VARCHAR_255 : TEXT) .selectExpression(idColAlias) .build()); columns.addAll(getOrganisationUnitGroupSetColumns()); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index 1783b0848377..9c1042ff52f1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -137,7 +137,20 @@ void testGetRegularAnalyticsTableHasUniqueKeyOnId() { AnalyticsTable table = tables.get(0); assertTrue(table.hasPrimaryKey()); assertEquals(List.of("id", "year"), table.getPrimaryKey()); - assertTrue(sqlBuilder.createTable(table).contains("unique key (`id`,`year`)")); + String createTableSql = sqlBuilder.createTable(table); + assertTrue(createTableSql.contains("unique key (`id`,`year`)")); + // Doris rejects unbounded string/text types as key columns on any key model (errCode = 2, + // "String Type should not be used in key column"). The id column (a concat(...) of several + // dimension UIDs) must be given a bounded type on Doris, unlike Postgres/ClickHouse where it + // stays TEXT. + assertTrue( + createTableSql.contains("`id` varchar(1020)"), + () -> "Expected id column to use a bounded varchar type, got: " + createTableSql); + assertFalse( + createTableSql.contains("`id` string"), + () -> + "id column must not use the unbounded string type as a key column, got: " + + createTableSql); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java index 9e7022310584..bbe95b3bf101 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java @@ -33,6 +33,7 @@ import static org.hisp.dhis.analytics.AnalyticsStringUtils.qualifyVariables; import static org.hisp.dhis.analytics.AnalyticsStringUtils.replaceQualify; import static org.hisp.dhis.db.model.DataType.INTEGER; +import static org.hisp.dhis.db.model.DataType.TEXT; import static org.hisp.dhis.db.model.Logged.LOGGED; import static org.hisp.dhis.db.model.Logged.UNLOGGED; import static org.hisp.dhis.period.PeriodType.PERIOD_TYPES; @@ -62,6 +63,7 @@ import org.hisp.dhis.configuration.Configuration; import org.hisp.dhis.configuration.ConfigurationService; import org.hisp.dhis.dataapproval.DataApprovalLevelService; +import org.hisp.dhis.db.model.Column; import org.hisp.dhis.db.model.Table; import org.hisp.dhis.db.sql.PostgreSqlBuilder; import org.hisp.dhis.db.sql.SqlBuilder; @@ -217,6 +219,35 @@ void testGetRegularAnalyticsTable() { partitionB.getYear().intValue(), new DateTime(partitionB.getStartDate()).getYear()); } + @Test + void testGetRegularAnalyticsTableIdColumnRemainsTextOnPostgres() { + // Doris needs a bounded type for the id key column (see JdbcAnalyticsTableManagerDorisTest), + // but that must not change Postgres behavior, since + // PostgreSqlBuilder.requiresUniqueKeyAnalyticsTables() + // is false and id is not used as a key column there. + Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); + List dataYears = List.of(2018, 2019); + + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().startTime(startTime).build(); + + when(analyticsTableSettings.getTableLogged()).thenReturn(UNLOGGED); + when(jdbcTemplate.queryForList(Mockito.anyString(), ArgumentMatchers.>any())) + .thenReturn(dataYears); + when(configurationService.getConfiguration()).thenReturn(configuration); + when(configuration.getDataOutputPeriodTypes()) + .thenReturn(PERIOD_TYPES.stream().collect(toUnmodifiableSet())); + + List tables = subject.getAnalyticsTables(params); + + assertEquals(1, tables.size()); + AnalyticsTable table = tables.get(0); + Column idColumn = + table.getColumns().stream().filter(c -> "id".equals(c.getName())).findFirst().orElseThrow(); + + assertEquals(TEXT, idColumn.getDataType()); + } + @Test void testGetRegularAnalyticsTableLogged() { Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java index 1da132d70d42..e7d9ecf14eea 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java @@ -32,6 +32,7 @@ import static java.util.stream.Collectors.toUnmodifiableSet; import static org.hisp.dhis.period.PeriodType.PERIOD_TYPES; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.verify; @@ -138,7 +139,20 @@ void testGetLatestAnalyticsTableHasUniqueKeyOnId() { AnalyticsTable table = tables.get(0); assertTrue(table.hasPrimaryKey()); assertEquals(List.of("id", "year"), table.getPrimaryKey()); - assertTrue(sqlBuilder.createTable(table).contains("unique key (`id`,`year`)")); + String createTableSql = sqlBuilder.createTable(table); + assertTrue(createTableSql.contains("unique key (`id`,`year`)")); + // Doris rejects unbounded string/text types as key columns on any key model (errCode = 2, + // "String Type should not be used in key column"). The id column (a concat(...) of several + // dimension UIDs) must be given a bounded type on Doris, unlike Postgres/ClickHouse where it + // stays TEXT. + assertTrue( + createTableSql.contains("`id` varchar(1020)"), + () -> "Expected id column to use a bounded varchar type, got: " + createTableSql); + assertFalse( + createTableSql.contains("`id` string"), + () -> + "id column must not use the unbounded string type as a key column, got: " + + createTableSql); } @Test diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java index eebeb6225fa4..cd59e9669050 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java @@ -142,6 +142,11 @@ void testDataTypes() { assertEquals("double", sqlBuilder.dataTypeDouble()); assertEquals("datetime(3)", sqlBuilder.dataTypeTimestamp()); assertEquals("json", sqlBuilder.dataTypeJson()); + // Doris' "string" type is unbounded and rejected as a key column on any key model (errCode = + // 2, "String Type should not be used in key column"). Callers that need a column to be a key + // (e.g. analytics table managers building a synthetic "id" key column) must use a bounded + // type (e.g. VARCHAR_255) instead of TEXT on Doris. + assertEquals("string", sqlBuilder.dataTypeText()); } @Test From d60e333bc71d35da2a922b57e2fdbc348b6048f6 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Tue, 14 Jul 2026 09:51:28 +0200 Subject: [PATCH 14/31] fix: DorisSqlBuilder.tableExists() checks wrong schema, breaking continuous-analytics history preservation tableExists() filtered on t.table_schema = 'public', but native Doris analytics tables live in whatever database the connection is on (e.g. 'dhis2'), not 'public' ('public' is only correct for the federated Postgres catalog). This meant tableExists() always returned false for real Doris tables, so swapTable()'s skipMasterTable was always false, so every continuous (lastYears=0) run silently replaced the entire main analytics table with just the latest-window data instead of inserting into the existing table - destroying history. Found via a live manual test against a real Doris instance: triggered lastYears=0, and confirmed the main table's row count and partition history matched exactly what a full replace (not a merge) would produce. Fixed by using database() instead of the hardcoded schema name, which correctly resolves to the connection's actual database. Re-verified live after the fix: history preserved, create/update/delete all correctly reflected, "skip master table: true" confirmed in logs. Co-Authored-By: Claude Sonnet 5 --- .../src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java | 2 +- .../src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index f2405e777daa..3abaf2e27c06 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -548,7 +548,7 @@ public String tableExists(String name) { return String.format( """ select t.table_name from information_schema.tables t \ - where t.table_schema = 'public' \ + where t.table_schema = database() \ and t.table_name = %s;""", singleQuote(name)); } diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java index cd59e9669050..fa7572703f78 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java @@ -370,7 +370,7 @@ void testTableExists() { """ select t.table_name \ from information_schema.tables t \ - where t.table_schema = 'public' \ + where t.table_schema = database() \ and t.table_name = 'immunization';"""; assertEquals(expected, sqlBuilder.tableExists("immunization")); From fcc1b5e830bf39f41c7567cff223042680b09dad Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Tue, 14 Jul 2026 20:29:58 +0200 Subject: [PATCH 15/31] fix: remove redundant public modifier on JUnit 5 setUp() [SonarCloud] JUnit 5 test lifecycle methods don't need public visibility; the rest of the class already uses package-private. --- .../dhis/analytics/table/JdbcAnalyticsTableManagerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java index bbe95b3bf101..ea2189d187c1 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerTest.java @@ -127,7 +127,7 @@ class JdbcAnalyticsTableManagerTest { @InjectMocks private JdbcAnalyticsTableManager subject; @BeforeEach - public void setUp() { + void setUp() { when(settingsProvider.getCurrentSettings()).thenReturn(settings); when(settings.getLastSuccessfulResourceTablesUpdate()).thenReturn(new Date(0L)); } From cbef739fae6978c4f9383d243da22ea47f1b4ef5 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Wed, 15 Jul 2026 15:56:44 +0200 Subject: [PATCH 16/31] Address SQ issue --- .../table/JdbcEventAnalyticsTableManager.java | 134 +++++++++--------- 1 file changed, 70 insertions(+), 64 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 686e373d4c88..d39c02d2017d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -373,91 +373,97 @@ private boolean hasNewTrackerEvents(Date startDate, Date endDate, Program progra public void removeUpdatedData(List tables) { for (AnalyticsTable table : tables) { AnalyticsTablePartition partition = table.getLatestTablePartition(); - String sql = null; Program program = table.getProgram(); boolean uniqueKey = sqlBuilder.requiresUniqueKeyAnalyticsTables(); + String sql = getSql(table, program, partition, uniqueKey); - if (table.getProgram().isRegistration()) { + if (isNotBlank(sql)) { + invokeTimeAndLog(sql, "Remove updated events for table: '{}'", table.getMainName()); + } + } + } + + private String getSql( + AnalyticsTable table, Program program, AnalyticsTablePartition partition, boolean uniqueKey) { + + String sql = null; + if (table.getProgram().isRegistration()) { + sql = + uniqueKey + ? replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + using ${trackerevent} ev \ + inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ + where ax.event = ev.uid \ + and en.programid = ${programId} \ + and ev.lastupdated >= '${startDate}' \ + and ev.lastupdated < '${endDate}';""", + Map.of( + "tableName", quote(table.getMainName()), + "programId", String.valueOf(program.getId()), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))) + : replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + where ax.event in ( \ + select ev.uid \ + from ${trackerevent} ev \ + inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ + where en.programid = ${programId} \ + and ev.lastupdated >= '${startDate}' \ + and ev.lastupdated < '${endDate}');""", + Map.of( + "tableName", sqlBuilder.qualifyTable(table.getMainName()), + "programId", String.valueOf(program.getId()), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))); + } else { + if (isEmpty(program.getProgramStages()) || program.getProgramStages().size() > 1) { + log.warn("Single event program {}, must have one stage", program.getUid()); + } else { + String programStageId = + String.valueOf( + program.isSingleProgramStage() + ? program.getProgramStages().stream().toList().get(0).getId() + : EMPTY); sql = uniqueKey ? replaceQualify( sqlBuilder, """ - delete from ${tableName} ax \ - using ${trackerevent} ev \ - inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ - where ax.event = ev.uid \ - and en.programid = ${programId} \ - and ev.lastupdated >= '${startDate}' \ - and ev.lastupdated < '${endDate}';""", + delete from ${tableName} ax \ + using ${singleevent} ev \ + where ax.event = ev.uid \ + and ev.programstageid = ${programStageId} \ + and ev.lastupdated >= '${startDate}' \ + and ev.lastupdated < '${endDate}';""", Map.of( "tableName", quote(table.getMainName()), - "programId", String.valueOf(program.getId()), + "programStageId", String.valueOf(programStageId), "startDate", toLongDate(partition.getStartDate()), "endDate", toLongDate(partition.getEndDate()))) : replaceQualify( sqlBuilder, """ - delete from ${tableName} ax \ - where ax.event in ( \ - select ev.uid \ - from ${trackerevent} ev \ - inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ - where en.programid = ${programId} \ - and ev.lastupdated >= '${startDate}' \ - and ev.lastupdated < '${endDate}');""", + delete from ${tableName} ax \ + where ax.event in ( \ + select ev.uid \ + from ${singleevent} ev \ + where ev.programstageid = ${programStageId} \ + and ev.lastupdated >= '${startDate}' \ + and ev.lastupdated < '${endDate}');""", Map.of( "tableName", sqlBuilder.qualifyTable(table.getMainName()), - "programId", String.valueOf(program.getId()), + "programStageId", String.valueOf(programStageId), "startDate", toLongDate(partition.getStartDate()), "endDate", toLongDate(partition.getEndDate()))); - } else { - if (isEmpty(program.getProgramStages()) || program.getProgramStages().size() > 1) { - log.warn("Single event program {}, must have one stage", program.getUid()); - } else { - String programStageId = - String.valueOf( - program.isSingleProgramStage() - ? program.getProgramStages().stream().toList().get(0).getId() - : EMPTY); - sql = - uniqueKey - ? replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - using ${singleevent} ev \ - where ax.event = ev.uid \ - and ev.programstageid = ${programStageId} \ - and ev.lastupdated >= '${startDate}' \ - and ev.lastupdated < '${endDate}';""", - Map.of( - "tableName", quote(table.getMainName()), - "programStageId", String.valueOf(programStageId), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))) - : replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - where ax.event in ( \ - select ev.uid \ - from ${singleevent} ev \ - where ev.programstageid = ${programStageId} \ - and ev.lastupdated >= '${startDate}' \ - and ev.lastupdated < '${endDate}');""", - Map.of( - "tableName", sqlBuilder.qualifyTable(table.getMainName()), - "programStageId", String.valueOf(programStageId), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))); - } - } - - if (isNotBlank(sql)) { - invokeTimeAndLog(sql, "Remove updated events for table: '{}'", table.getMainName()); } } + return sql; } @Override From 7c1177c50b225304488f4546779583fd60658157 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Thu, 30 Jul 2026 18:42:12 +0200 Subject: [PATCH 17/31] fix: avoid federated join inside Doris removeUpdatedData() DELETE The Doris branch of removeUpdatedData() joined a live federated Postgres datavalue table against local dimension tables inside a single DELETE ... USING statement. Live validation found this takes 40+ minutes (killed, incomplete) versus ~12 minutes for a full analytics rebuild on the same instance, defeating the purpose of continuous analytics. Split into: a plain federated SELECT (the same join shape already proven fast by populateTable) that materializes matching rows' natural dx/co/ao/pe/ou column values into a small native staging table, then a fully local DELETE against that staging table. No change to the main table's schema, key, or the Postgres/ClickHouse branch. Co-Authored-By: Claude Sonnet 5 --- .../table/JdbcAnalyticsTableManager.java | 132 +++++++++++++----- .../JdbcAnalyticsTableManagerDorisTest.java | 53 ++++++- 2 files changed, 146 insertions(+), 39 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java index 221b30fd587b..201ea4d6e1cf 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManager.java @@ -76,6 +76,7 @@ import org.hisp.dhis.common.ValueType; import org.hisp.dhis.configuration.ConfigurationService; import org.hisp.dhis.dataapproval.DataApprovalLevelService; +import org.hisp.dhis.db.model.Column; import org.hisp.dhis.db.model.Table; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.organisationunit.OrganisationUnitLevel; @@ -258,41 +259,104 @@ public void preCreateTables(AnalyticsTableUpdateParams params) { @Override public void removeUpdatedData(List tables) { AnalyticsTablePartition partition = getLatestTablePartition(tables); + + if (sqlBuilder.requiresUniqueKeyAnalyticsTables()) { + removeUpdatedDataViaStagingKeys(partition); + } else { + removeUpdatedDataViaSubquery(partition); + } + } + + /** + * Removes updated and deleted data on databases which require a unique key for analytics tables + * (Doris). The equivalent single {@code delete ... using } statement is avoided + * here because it joins a live federated table (the operational database's {@code datavalue}) + * against local tables inside a {@code delete}, which is dramatically slower than the same join + * shape used as a plain {@code select} (as {@link #populateTable} already does). Instead, the + * matching rows' natural key column values are first materialized into a small native staging + * table via that proven-fast federated {@code select}, and the actual delete runs entirely + * locally against that staging table. + */ + private void removeUpdatedDataViaStagingKeys(AnalyticsTablePartition partition) { + String keysTableName = getAnalyticsTableType().getTableName() + "_delete_keys"; + Table keysTable = + new Table( + keysTableName, + List.of( + new Column("dx", CHARACTER_11), + new Column("co", CHARACTER_11), + new Column("ao", CHARACTER_11), + new Column("pe", TEXT), + new Column("ou", CHARACTER_11), + new Column("year", INTEGER)), + List.of()); + + invokeTimeAndLog( + sqlBuilder.dropTableIfExistsCascade(keysTableName), "Drop delete-keys staging table (pre)"); + + invokeTimeAndLog(sqlBuilder.createTable(keysTable), "Create delete-keys staging table"); + + String insertSql = + replaceQualify( + sqlBuilder, + """ + insert into ${keysTableName} (dx,co,ao,pe,ou,year) \ + select des.dataelementuid, dcs.categoryoptioncombouid, acs.categoryoptioncombouid, ps.iso, ous.organisationunituid, ps.year \ + from ${datavalue} dv \ + inner join analytics_rs_dataelementstructure des on dv.dataelementid=des.dataelementid \ + inner join analytics_rs_periodstructure ps on dv.periodid=ps.periodid \ + inner join analytics_rs_orgunitstructure ous on dv.sourceid=ous.organisationunitid \ + inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ + inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ + where dv.lastupdated >= '${startDate}' and dv.lastupdated < '${endDate}';""", + Map.of( + "keysTableName", quote(keysTableName), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))); + + invokeTimeAndLog(insertSql, "Populate delete-keys staging table"); + + String deleteSql = + replace( + """ + delete from ${tableName} ax \ + using ${keysTableName} k \ + where ax.dx=k.dx and ax.co=k.co and ax.ao=k.ao and ax.pe=k.pe and ax.ou=k.ou and ax.year=k.year;""", + Map.of( + "tableName", quote(getAnalyticsTableType().getTableName()), + "keysTableName", quote(keysTableName))); + + invokeTimeAndLog(deleteSql, "Remove updated data values"); + + invokeTimeAndLog( + sqlBuilder.dropTableIfExistsCascade(keysTableName), + "Drop delete-keys staging table (post)"); + } + + /** + * Removes updated and deleted data on databases which do not require a unique key for analytics + * tables (Postgres, ClickHouse). Runs entirely against local tables, so the federated-join + * problem {@link #removeUpdatedDataViaStagingKeys} works around does not apply here. + */ + private void removeUpdatedDataViaSubquery(AnalyticsTablePartition partition) { String sql = - sqlBuilder.requiresUniqueKeyAnalyticsTables() - ? replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - using ${datavalue} dv \ - inner join analytics_rs_dataelementstructure des on dv.dataelementid=des.dataelementid \ - inner join analytics_rs_periodstructure ps on dv.periodid=ps.periodid \ - inner join analytics_rs_orgunitstructure ous on dv.sourceid=ous.organisationunitid \ - inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ - inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ - where ax.id = concat(des.dataelementuid,'-',ps.iso,'-',ous.organisationunituid,'-',dcs.categoryoptioncombouid,'-',acs.categoryoptioncombouid) \ - and dv.lastupdated >= '${startDate}' and dv.lastupdated < '${endDate}';""", - Map.of( - "tableName", quote(getAnalyticsTableType().getTableName()), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))) - : replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - where ax.id in ( \ - select concat(des.dataelementuid,'-',ps.iso,'-',ous.organisationunituid,'-',dcs.categoryoptioncombouid,'-',acs.categoryoptioncombouid) as id \ - from ${datavalue} dv \ - inner join analytics_rs_dataelementstructure des on dv.dataelementid=des.dataelementid \ - inner join analytics_rs_periodstructure ps on dv.periodid=ps.periodid \ - inner join analytics_rs_orgunitstructure ous on dv.sourceid=ous.organisationunitid \ - inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ - inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ - where dv.lastupdated >= '${startDate}'and dv.lastupdated < '${endDate}');""", - Map.of( - "tableName", sqlBuilder.qualifyTable(getAnalyticsTableType().getTableName()), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))); + replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + where ax.id in ( \ + select concat(des.dataelementuid,'-',ps.iso,'-',ous.organisationunituid,'-',dcs.categoryoptioncombouid,'-',acs.categoryoptioncombouid) as id \ + from ${datavalue} dv \ + inner join analytics_rs_dataelementstructure des on dv.dataelementid=des.dataelementid \ + inner join analytics_rs_periodstructure ps on dv.periodid=ps.periodid \ + inner join analytics_rs_orgunitstructure ous on dv.sourceid=ous.organisationunitid \ + inner join analytics_rs_categorystructure dcs on dv.categoryoptioncomboid=dcs.categoryoptioncomboid \ + inner join analytics_rs_categorystructure acs on dv.attributeoptioncomboid=acs.categoryoptioncomboid \ + where dv.lastupdated >= '${startDate}'and dv.lastupdated < '${endDate}');""", + Map.of( + "tableName", sqlBuilder.qualifyTable(getAnalyticsTableType().getTableName()), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))); invokeTimeAndLog(sql, "Remove updated data values"); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index 9c1042ff52f1..734c2adeb7f8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -154,7 +154,7 @@ void testGetRegularAnalyticsTableHasUniqueKeyOnId() { } @Test - void testRemoveUpdatedDataUsesUsingJoinNotSubquery() { + void testRemoveUpdatedDataMaterializesKeysNatively() { Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); Date lastLatestPartitionUpdate = new DateTime(2019, 3, 1, 9, 0).toDate(); Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); @@ -179,11 +179,54 @@ void testRemoveUpdatedDataUsesUsingJoinNotSubquery() { subject.removeUpdatedData(tables); org.mockito.ArgumentCaptor sql = org.mockito.ArgumentCaptor.forClass(String.class); - org.mockito.Mockito.verify(jdbcTemplate).execute(sql.capture()); + org.mockito.Mockito.verify(jdbcTemplate, org.mockito.Mockito.times(5)).execute(sql.capture()); + List statements = sql.getAllValues(); - assertTrue(sql.getValue().contains("using")); - assertTrue(sql.getValue().contains("ax.id =")); - assertTrue(!sql.getValue().contains("in (")); + String drop1 = statements.get(0); + assertTrue(drop1.toLowerCase().contains("drop table if exists")); + assertTrue(drop1.contains("analytics_delete_keys")); + + String create = statements.get(1); + assertTrue(create.toLowerCase().contains("create table")); + assertTrue(create.contains("analytics_delete_keys")); + assertTrue(create.contains("`dx`")); + assertTrue(create.contains("`co`")); + assertTrue(create.contains("`ao`")); + assertTrue(create.contains("`pe`")); + assertTrue(create.contains("`ou`")); + assertTrue(create.contains("`year`")); + assertFalse(create.contains("varchar(1020)")); + + String insert = statements.get(2); + assertTrue(insert.toLowerCase().contains("insert into")); + assertTrue(insert.contains("analytics_delete_keys")); + assertTrue(insert.contains("datavalue")); + assertTrue(insert.contains("analytics_rs_dataelementstructure")); + assertFalse(insert.contains("concat(")); + assertTrue(insert.contains("dv.lastupdated >=")); + assertTrue(insert.contains("dv.lastupdated <")); + assertTrue( + insert.contains( + "(dx,co,ao,pe,ou,year) select des.dataelementuid, dcs.categoryoptioncombouid, " + + "acs.categoryoptioncombouid, ps.iso, ous.organisationunituid, ps.year"), + () -> "Expected column-list-to-select-list pairing to match exactly, got: " + insert); + + String delete = statements.get(3); + assertTrue(delete.toLowerCase().contains("delete from")); + assertTrue(delete.toLowerCase().contains("using")); + assertTrue(delete.contains("analytics_delete_keys")); + assertTrue(delete.contains("ax.dx=k.dx")); + assertTrue(delete.contains("ax.co=k.co")); + assertTrue(delete.contains("ax.ao=k.ao")); + assertTrue(delete.contains("ax.pe=k.pe")); + assertTrue(delete.contains("ax.ou=k.ou")); + assertTrue(delete.contains("ax.year=k.year")); + assertFalse(delete.contains("datavalue")); + assertFalse(delete.contains("analytics_rs_")); + + String drop2 = statements.get(4); + assertTrue(drop2.toLowerCase().contains("drop table")); + assertTrue(drop2.contains("analytics_delete_keys")); } @Test From c35bace50f8407d80f2807987df13193cba69c51 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Fri, 31 Jul 2026 12:15:25 +0200 Subject: [PATCH 18/31] fix: scope Doris continuous-analytics window to last continuous update [DHIS2-21920] The "latest partition" window used by continuous analytics (lastYears=0) was always anchored to the last full rebuild's timestamp, never the last continuous run's, even though that timestamp is tracked and already used correctly for the "is there anything new" check. On Doris this meant every continuous run reprocessed everything changed since the last full rebuild rather than just since the last continuous run, with cost growing unboundedly between full rebuilds - confirmed live: a single-row mutation triggered a ~3.5 minute run, matching a ~96,000-row mutation's cost. Scoped the fix to sqlBuilder.requiresUniqueKeyAnalyticsTables() (true only for Doris) rather than the shared code path, since Postgres's "latest" partition is wholesale-replaced on every continuous run (not merged), so narrowing its window would silently discard previously-captured incremental data. Doris merges into the persistent main table with natural-key deduplication, so a narrower window is safe there. Verified zero behavior change for Postgres (existing tests, unchanged, still pass). Co-Authored-By: Claude Sonnet 5 --- .../table/AbstractJdbcTableManager.java | 13 ++++++- .../table/JdbcEventAnalyticsTableManager.java | 9 ++++- .../JdbcAnalyticsTableManagerDorisTest.java | 38 +++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index f1c3737296c4..2c393da57aa5 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -427,15 +427,24 @@ protected AnalyticsTable getLatestAnalyticsTable( Date endDate = params.getStartTime(); boolean hasUpdatedData = hasUpdatedLatestData(lastAnyTableUpdate, endDate); + // Engines without a unique key on analytics tables (Postgres) wholesale-replace the latest + // partition on every run, so its window must always span back to the last full rebuild, or + // data captured by an earlier continuous run would be discarded rather than carried forward. + // Engines with a unique key (Doris) instead merge into the persistent main table with + // natural-key deduplication, so it's safe and correct to only (re)process what changed since + // the last continuous run. + Date partitionStartDate = + sqlBuilder.requiresUniqueKeyAnalyticsTables() ? lastAnyTableUpdate : lastFullTableUpdate; + AnalyticsTable table = new AnalyticsTable(getAnalyticsTableType(), columns, List.of(), primaryKey, logged); if (hasUpdatedData) { table.addTablePartition( - List.of(), AnalyticsTablePartition.LATEST_PARTITION, lastFullTableUpdate, endDate); + List.of(), AnalyticsTablePartition.LATEST_PARTITION, partitionStartDate, endDate); log.info( "Added latest analytics partition with start: '{}' and end: '{}'", - toLongDate(lastFullTableUpdate), + toLongDate(partitionStartDate), toLongDate(endDate)); } else { log.info( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index d39c02d2017d..d47137dd44f8 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -239,7 +239,14 @@ private List getLatestAnalyticsTables(AnalyticsTableUpdateParams lastFullTableUpdate.getTime() > 0L, "A full analytics table update process must be run prior to a latest partition update"); - Date startDate = lastFullTableUpdate; + // Engines without a unique key on analytics tables (Postgres) wholesale-replace the latest + // partition on every run, so its window must always span back to the last full rebuild, or + // data captured by an earlier continuous run would be discarded rather than carried forward. + // Engines with a unique key (Doris) instead merge into the persistent main table with + // natural-key deduplication, so it's safe and correct to only (re)process what changed since + // the last continuous run. + Date startDate = + sqlBuilder.requiresUniqueKeyAnalyticsTables() ? lastAnyTableUpdate : lastFullTableUpdate; Date endDate = params.getStartTime(); List tables = new ArrayList<>(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index 734c2adeb7f8..246e4e7a6e98 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -47,6 +47,7 @@ import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsTable; import org.hisp.dhis.analytics.table.model.AnalyticsTableColumn; +import org.hisp.dhis.analytics.table.model.AnalyticsTablePartition; import org.hisp.dhis.analytics.table.setting.AnalyticsTableSettings; import org.hisp.dhis.category.CategoryService; import org.hisp.dhis.common.IdentifiableObjectManager; @@ -64,6 +65,7 @@ import org.hisp.dhis.setting.SystemSettingsProvider; import org.joda.time.DateTime; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; @@ -153,6 +155,42 @@ void testGetRegularAnalyticsTableHasUniqueKeyOnId() { + createTableSql); } + @Test + @DisplayName( + "Doris latest-partition window starts at the last continuous update, not the last full" + + " rebuild, since Doris merges into the persistent main table rather than replacing a" + + " partition wholesale") + void testGetLatestAnalyticsTableUsesLastAnyTableUpdate() { + Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); + Date lastLatestPartitionUpdate = new DateTime(2019, 3, 1, 9, 0).toDate(); + Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); + + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().startTime(startTime).build().withLatestPartition(); + + List> queryResp = new ArrayList<>(); + queryResp.add(Map.of("dataelementid", 1)); + + when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + .thenReturn(lastLatestPartitionUpdate); + when(analyticsTableSettings.getTableLogged()).thenReturn(Logged.UNLOGGED); + when(jdbcTemplate.queryForList(org.mockito.Mockito.anyString())).thenReturn(queryResp); + when(configurationService.getConfiguration()).thenReturn(configuration); + when(configuration.getDataOutputPeriodTypes()) + .thenReturn(PERIOD_TYPES.stream().collect(toUnmodifiableSet())); + + List tables = subject.getAnalyticsTables(params); + + assertEquals(1, tables.size()); + + AnalyticsTablePartition partition = tables.get(0).getLatestTablePartition(); + + assertTrue(partition.isLatestPartition()); + assertEquals(lastLatestPartitionUpdate, partition.getStartDate()); + assertEquals(startTime, partition.getEndDate()); + } + @Test void testRemoveUpdatedDataMaterializesKeysNatively() { Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); From 2651a6d58e52ceffbd9d874f9ced4f85263480ad Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Fri, 31 Jul 2026 15:08:22 +0200 Subject: [PATCH 19/31] fix: use the real latest-partition date range instead of a fake one on declarative-partitioning engines DefaultAnalyticsTableService.getTablePartitions() replaced the carefully constructed "latest partition" (with its real date range) with a "fake" partition (year=null, no dates) whenever the engine supports declarative partitioning - a condition true for Doris (and ClickHouse). With year=null, isLatestPartition() returns false, so getPartitionClause() silently drops the "dv.lastupdated >= ..." lower bound from the populate query, leaving only "< now" - matching nearly the entire historical table on every continuous run regardless of how narrow the intended window was. Confirmed live: a single-row mutation produced a populate step matching ~21M rows and taking the same ~2.5 minutes as a full-table sweep, while the DELETE side (which fetches its partition through a different, unaffected code path) correctly matched 1 row in under a second. Captured via SHOW FULL PROCESSLIST on Doris - the literal WHERE clause had a double space where "and dv.lastupdated >= '...'" should have been. Fix is scoped to continuous/latest-update mode only (isLatestUpdate), preserving the existing single-combined-partition optimization for regular multi-year rebuilds on declarative-partitioning engines, where removing it would cause each year's populate call to redundantly reprocess every other year (no per-partition year filter is applied on these engines, by design, since they route rows to physical partitions natively). Zero behavior change for Postgres, whose supportsDeclarativePartitioning() is false, making the new condition a no-op there regardless of isLatestUpdate. Co-Authored-By: Claude Sonnet 5 --- .../table/DefaultAnalyticsTableService.java | 11 ++- .../table/AnalyticsTableServiceTest.java | 70 ++++++++++++++++++- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index cbc975b37a03..bb0a930704ef 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -132,7 +132,7 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { createTables(tables, progress); clock.logTime("Created analytics tables"); - List partitions = getTablePartitions(tables); + List partitions = getTablePartitions(tables, params.isLatestUpdate()); int partitionSize = partitions.size(); progress.startingStage( @@ -341,11 +341,16 @@ private void swapTables( * @param tables the list of {@link AnalyticsTable}. * @return a list of {@link AnalyticsTablePartition}. */ - List getTablePartitions(List tables) { + List getTablePartitions( + List tables, boolean isLatestUpdate) { List partitions = new ArrayList<>(); for (AnalyticsTable table : tables) { - if (table.hasTablePartitions() && !sqlBuilder.supportsDeclarativePartitioning()) { + // The continuous/latest-update partition always needs its own real date range, regardless + // of whether the engine natively routes rows to physical partitions by column value; only + // regular multi-year updates on such engines can safely use one combined fake partition. + if (table.hasTablePartitions() + && (isLatestUpdate || !sqlBuilder.supportsDeclarativePartitioning())) { partitions.addAll(table.getTablePartitions()); } else { // Fake partition representing the master table diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java index b7b11f8483b5..40658e111133 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java @@ -32,11 +32,14 @@ import static org.hisp.dhis.db.model.DataType.DOUBLE; import static org.hisp.dhis.db.model.DataType.TEXT; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.util.Date; import java.util.List; import org.hisp.dhis.analytics.AnalyticsTableManager; import org.hisp.dhis.analytics.AnalyticsTableType; @@ -112,11 +115,76 @@ void testGetTablePartitions() { new DateTime(2011, 12, 31, 0, 0).toDate()); AnalyticsTable tB = new AnalyticsTable(AnalyticsTableType.ORG_UNIT_TARGET, columns, sortKey, Logged.UNLOGGED); - List partitions = tableService.getTablePartitions(List.of(tA, tB)); + List partitions = + tableService.getTablePartitions(List.of(tA, tB), false); assertEquals(3, partitions.size()); } + @Test + void testGetTablePartitionsUsesRealPartitionForLatestUpdateOnDeclarativePartitioningEngine() { + List columns = + List.of( + AnalyticsTableColumn.builder().name("dx").dataType(TEXT).selectExpression("dx").build(), + AnalyticsTableColumn.builder() + .name("value") + .dataType(DOUBLE) + .selectExpression("value") + .build()); + List sortKey = List.of("dx"); + + Date startDate = new DateTime(2026, 7, 31, 12, 48, 20).toDate(); + Date endDate = new DateTime(2026, 7, 31, 12, 55, 0).toDate(); + + AnalyticsTable table = + new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, sortKey, Logged.UNLOGGED); + table.addTablePartition( + List.of(), AnalyticsTablePartition.LATEST_PARTITION, startDate, endDate); + + List partitions = + tableService.getTablePartitions(List.of(table), true); + + assertEquals(1, partitions.size()); + AnalyticsTablePartition partition = partitions.get(0); + assertTrue(partition.isLatestPartition()); + assertEquals(startDate, partition.getStartDate()); + assertEquals(endDate, partition.getEndDate()); + } + + @Test + void testGetTablePartitionsUsesFakePartitionForRegularUpdateOnDeclarativePartitioningEngine() { + when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(true); + + List columns = + List.of( + AnalyticsTableColumn.builder().name("dx").dataType(TEXT).selectExpression("dx").build(), + AnalyticsTableColumn.builder() + .name("value") + .dataType(DOUBLE) + .selectExpression("value") + .build()); + List sortKey = List.of("dx"); + + AnalyticsTable table = + new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, sortKey, Logged.UNLOGGED); + table.addTablePartition( + List.of(), + 2010, + new DateTime(2010, 1, 1, 0, 0).toDate(), + new DateTime(2010, 12, 31, 0, 0).toDate()); + table.addTablePartition( + List.of(), + 2011, + new DateTime(2011, 1, 1, 0, 0).toDate(), + new DateTime(2011, 12, 31, 0, 0).toDate()); + + List partitions = + tableService.getTablePartitions(List.of(table), false); + + assertEquals(1, partitions.size()); + assertFalse(partitions.get(0).isLatestPartition()); + } + @Test void testGetParallelJobsA() { when(settingsProvider.getCurrentSettings()).thenReturn(settings); From 62398055e826d64f7c8b635990fb2c3506137ac7 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Fri, 31 Jul 2026 15:32:03 +0200 Subject: [PATCH 20/31] fix: target the actual physical table when populating the latest partition on declarative-partitioning engines The previous fix restored the real date range for the continuous/latest partition on Doris, but that partition's name also carried a year suffix ("analytics_0_temp") from the standard partition-naming logic. On Doris, CREATE only ever builds one physical table per AnalyticsTable (the master's plain name, e.g. "analytics_temp") - per-partition physical tables are deliberately not created for declarative-partitioning engines, since Doris routes rows to partitions natively within that one table. Confirmed live: the populate step failed with "Table [analytics_0_temp] does not exist in database [analytics]" - a table CREATE never built. Failed during staging (before any swap), so the live analytics table was unaffected. Added a constructor to AnalyticsTablePartition that carries a real year/date range but always names itself after the master table (no suffix), and use it specifically for the declarative-partitioning + continuous-update case in getTablePartitions(). Postgres and Doris/ ClickHouse full-rebuild behavior are both unchanged. Co-Authored-By: Claude Sonnet 5 --- .../table/DefaultAnalyticsTableService.java | 16 +++++++---- .../table/model/AnalyticsTablePartition.java | 27 +++++++++++++++++++ .../table/AnalyticsTableServiceTest.java | 7 +++++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index bb0a930704ef..8a04b5afe8f7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -346,12 +346,18 @@ List getTablePartitions( List partitions = new ArrayList<>(); for (AnalyticsTable table : tables) { - // The continuous/latest-update partition always needs its own real date range, regardless - // of whether the engine natively routes rows to physical partitions by column value; only - // regular multi-year updates on such engines can safely use one combined fake partition. - if (table.hasTablePartitions() - && (isLatestUpdate || !sqlBuilder.supportsDeclarativePartitioning())) { + if (table.hasTablePartitions() && !sqlBuilder.supportsDeclarativePartitioning()) { + // Each partition is its own physical table, so its own real date range and name apply. partitions.addAll(table.getTablePartitions()); + } else if (table.hasTablePartitions() && isLatestUpdate) { + // A single physical table serves every logical partition on this engine (CREATE only + // ever builds the master table's name), so the continuous/latest-update partition must + // target that name - while still carrying its own real date range, needed to correctly + // scope the populate window. + AnalyticsTablePartition latest = table.getTablePartitions().get(0); + partitions.add( + new AnalyticsTablePartition( + table, latest.getYear(), latest.getStartDate(), latest.getEndDate())); } else { // Fake partition representing the master table partitions.add(new AnalyticsTablePartition(table)); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTablePartition.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTablePartition.java index 0f210ad7d618..ce8b2e9efef6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTablePartition.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/model/AnalyticsTablePartition.java @@ -102,6 +102,33 @@ public AnalyticsTablePartition(AnalyticsTable table) { this.endDate = null; } + /** + * Constructor. Sets the name to represent the master table's staging table without a + * partition-specific suffix, for engines where a single physical table serves every logical + * partition (native/declarative partitioning) - while still carrying the given year and date + * range, so callers can determine {@link #isLatestPartition()} and build a correctly-scoped + * populate window against a table name that actually exists. + * + * @param masterTable the master {@link AnalyticsTable} of this partition. + * @param year the year which represents this partition. + * @param startDate the start date of data for this partition. + * @param endDate the end date of data for this partition. + */ + public AnalyticsTablePartition( + AnalyticsTable masterTable, Integer year, Date startDate, Date endDate) { + super( + toStaging(getTableName(masterTable.getMainName(), null)), + List.of(), + List.of(), + List.of(), + masterTable.getLogged(), + masterTable); + this.masterTable = masterTable; + this.year = year; + this.startDate = startDate; + this.endDate = endDate; + } + // ------------------------------------------------------------------------- // Static methods // ------------------------------------------------------------------------- diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java index 40658e111133..e19b5cda7717 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java @@ -123,6 +123,8 @@ void testGetTablePartitions() { @Test void testGetTablePartitionsUsesRealPartitionForLatestUpdateOnDeclarativePartitioningEngine() { + when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(true); + List columns = List.of( AnalyticsTableColumn.builder().name("dx").dataType(TEXT).selectExpression("dx").build(), @@ -149,6 +151,11 @@ void testGetTablePartitionsUsesRealPartitionForLatestUpdateOnDeclarativePartitio assertTrue(partition.isLatestPartition()); assertEquals(startDate, partition.getStartDate()); assertEquals(endDate, partition.getEndDate()); + assertEquals( + table.getName(), + partition.getName(), + "Populate/create target must be the one physical table this engine actually builds," + + " not a year-suffixed name nothing ever creates"); } @Test From dbcad2325fd9dfe5453d6278cc38d84fd1fdb477 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Sat, 1 Aug 2026 08:02:52 +0200 Subject: [PATCH 21/31] fix: avoid federated join inside Doris EVENT removeUpdatedData() DELETE Mirrors the DATA_VALUE fix (7c1177c50b): the single-table join shape used for populating (proven-fast) is different from the multi-way join a DELETE...USING statement builds against a live federated trackerevent/ singleevent table. Materialize matching events' natural key (event uid) into a small native Doris staging table via a plain INSERT...SELECT, then run a fully native DELETE against that staging table. Live-validated against a real Doris test instance: purge+merge for both a registration program and a single-event program completed in ~10s total, update batches correctly retained, soft-deleted batches correctly purged. --- .../table/JdbcEventAnalyticsTableManager.java | 241 +++++++++++++----- ...bcEventAnalyticsTableManagerDorisTest.java | 92 ++++++- 2 files changed, 251 insertions(+), 82 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 4fbb2bf6feae..1f64ddc70f02 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -81,8 +81,10 @@ import org.hisp.dhis.configuration.ConfigurationService; import org.hisp.dhis.dataapproval.DataApprovalLevelService; import org.hisp.dhis.dataelement.DataElement; +import org.hisp.dhis.db.model.Column; import org.hisp.dhis.db.model.DataType; import org.hisp.dhis.db.model.Logged; +import org.hisp.dhis.db.model.Table; import org.hisp.dhis.db.sql.SqlBuilder; import org.hisp.dhis.legend.Legend; import org.hisp.dhis.legend.LegendSet; @@ -381,53 +383,46 @@ public void removeUpdatedData(List tables) { for (AnalyticsTable table : tables) { AnalyticsTablePartition partition = table.getLatestTablePartition(); Program program = table.getProgram(); - boolean uniqueKey = sqlBuilder.requiresUniqueKeyAnalyticsTables(); - String sql = getSql(table, program, partition, uniqueKey); - if (isNotBlank(sql)) { - invokeTimeAndLog(sql, "Remove updated events for table: '{}'", table.getMainName()); + if (sqlBuilder.requiresUniqueKeyAnalyticsTables()) { + removeUpdatedDataViaStagingKeys(table, program, partition); + } else { + String sql = getSubquerySql(table, program, partition); + + if (isNotBlank(sql)) { + invokeTimeAndLog(sql, "Remove updated events for table: '{}'", table.getMainName()); + } } } } - private String getSql( - AnalyticsTable table, Program program, AnalyticsTablePartition partition, boolean uniqueKey) { + /** + * Removes updated and deleted events on databases which do not require a unique key for analytics + * tables (Postgres, ClickHouse). Runs entirely against local tables, so the federated-join + * problem {@link #removeUpdatedDataViaStagingKeys} works around does not apply here. + */ + private String getSubquerySql( + AnalyticsTable table, Program program, AnalyticsTablePartition partition) { String sql = null; if (table.getProgram().isRegistration()) { sql = - uniqueKey - ? replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - using ${trackerevent} ev \ - inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ - where ax.event = ev.uid \ - and en.programid = ${programId} \ - and ev.lastupdated >= '${startDate}' \ - and ev.lastupdated < '${endDate}';""", - Map.of( - "tableName", quote(table.getMainName()), - "programId", String.valueOf(program.getId()), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))) - : replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - where ax.event in ( \ - select ev.uid \ - from ${trackerevent} ev \ - inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ - where en.programid = ${programId} \ - and ev.lastupdated >= '${startDate}' \ - and ev.lastupdated < '${endDate}');""", - Map.of( - "tableName", sqlBuilder.qualifyTable(table.getMainName()), - "programId", String.valueOf(program.getId()), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))); + replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + where ax.event in ( \ + select ev.uid \ + from ${trackerevent} ev \ + inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ + where en.programid = ${programId} \ + and ev.lastupdated >= '${startDate}' \ + and ev.lastupdated < '${endDate}');""", + Map.of( + "tableName", sqlBuilder.qualifyTable(table.getMainName()), + "programId", String.valueOf(program.getId()), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))); } else { if (isEmpty(program.getProgramStages()) || program.getProgramStages().size() > 1) { log.warn("Single event program {}, must have one stage", program.getUid()); @@ -438,41 +433,151 @@ where ax.event in ( \ ? program.getProgramStages().stream().toList().get(0).getId() : EMPTY); sql = - uniqueKey - ? replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - using ${singleevent} ev \ - where ax.event = ev.uid \ - and ev.programstageid = ${programStageId} \ - and ev.lastupdated >= '${startDate}' \ - and ev.lastupdated < '${endDate}';""", - Map.of( - "tableName", quote(table.getMainName()), - "programStageId", String.valueOf(programStageId), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))) - : replaceQualify( - sqlBuilder, - """ - delete from ${tableName} ax \ - where ax.event in ( \ - select ev.uid \ - from ${singleevent} ev \ - where ev.programstageid = ${programStageId} \ - and ev.lastupdated >= '${startDate}' \ - and ev.lastupdated < '${endDate}');""", - Map.of( - "tableName", sqlBuilder.qualifyTable(table.getMainName()), - "programStageId", String.valueOf(programStageId), - "startDate", toLongDate(partition.getStartDate()), - "endDate", toLongDate(partition.getEndDate()))); + replaceQualify( + sqlBuilder, + """ + delete from ${tableName} ax \ + where ax.event in ( \ + select ev.uid \ + from ${singleevent} ev \ + where ev.programstageid = ${programStageId} \ + and ev.lastupdated >= '${startDate}' \ + and ev.lastupdated < '${endDate}');""", + Map.of( + "tableName", sqlBuilder.qualifyTable(table.getMainName()), + "programStageId", String.valueOf(programStageId), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))); } } return sql; } + /** + * Removes updated and deleted events on databases which require a unique key for analytics tables + * (Doris). The equivalent single {@code delete ... using } statement is avoided + * here because it joins a live federated table (the operational database's {@code + * trackerevent}/{@code singleevent}) against local tables inside a {@code delete}, which is + * dramatically slower than the same join shape used as a plain {@code select} (as {@link + * #populateTable} already does). Instead, the matching rows' {@code event} UID is first + * materialized into a small native staging table via that proven-fast federated {@code select}, + * and the actual delete runs entirely locally against that staging table. Unlike the aggregate + * ({@code DATA_VALUE}) case, the analytics event table's natural key is {@code event} alone (see + * {@link #PRIMARY_KEY}) - one physical table per program, one row per event UID - so the staging + * table only needs to carry that single column. + */ + private void removeUpdatedDataViaStagingKeys( + AnalyticsTable table, Program program, AnalyticsTablePartition partition) { + if (program.isRegistration()) { + removeUpdatedTrackerEventsViaStagingKeys(table, program, partition); + } else if (isEmpty(program.getProgramStages()) || program.getProgramStages().size() > 1) { + log.warn("Single event program {}, must have one stage", program.getUid()); + } else { + removeUpdatedSingleEventsViaStagingKeys(table, program, partition); + } + } + + private void removeUpdatedTrackerEventsViaStagingKeys( + AnalyticsTable table, Program program, AnalyticsTablePartition partition) { + String keysTableName = getDeleteKeysTableName(); + + invokeTimeAndLog( + sqlBuilder.dropTableIfExistsCascade(keysTableName), "Drop delete-keys staging table (pre)"); + + invokeTimeAndLog( + sqlBuilder.createTable(newDeleteKeysTable(keysTableName)), + "Create delete-keys staging table"); + + String insertSql = + replaceQualify( + sqlBuilder, + """ + insert into ${keysTableName} (event) \ + select ev.uid \ + from ${trackerevent} ev \ + inner join ${enrollment} en on ev.enrollmentid=en.enrollmentid \ + where en.programid = ${programId} \ + and ev.lastupdated >= '${startDate}' and ev.lastupdated < '${endDate}';""", + Map.of( + "keysTableName", quote(keysTableName), + "programId", String.valueOf(program.getId()), + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))); + + invokeTimeAndLog(insertSql, "Populate delete-keys staging table"); + + invokeTimeAndLog( + deleteFromMainTableUsingKeysSql(table, keysTableName), + "Remove updated events for table: '{}'", + table.getMainName()); + + invokeTimeAndLog( + sqlBuilder.dropTableIfExistsCascade(keysTableName), + "Drop delete-keys staging table (post)"); + } + + private void removeUpdatedSingleEventsViaStagingKeys( + AnalyticsTable table, Program program, AnalyticsTablePartition partition) { + String programStageId = + String.valueOf( + program.isSingleProgramStage() + ? program.getProgramStages().stream().toList().get(0).getId() + : EMPTY); + String keysTableName = getDeleteKeysTableName(); + + invokeTimeAndLog( + sqlBuilder.dropTableIfExistsCascade(keysTableName), "Drop delete-keys staging table (pre)"); + + invokeTimeAndLog( + sqlBuilder.createTable(newDeleteKeysTable(keysTableName)), + "Create delete-keys staging table"); + + String insertSql = + replaceQualify( + sqlBuilder, + """ + insert into ${keysTableName} (event) \ + select ev.uid \ + from ${singleevent} ev \ + where ev.programstageid = ${programStageId} \ + and ev.lastupdated >= '${startDate}' and ev.lastupdated < '${endDate}';""", + Map.of( + "keysTableName", quote(keysTableName), + "programStageId", programStageId, + "startDate", toLongDate(partition.getStartDate()), + "endDate", toLongDate(partition.getEndDate()))); + + invokeTimeAndLog(insertSql, "Populate delete-keys staging table"); + + invokeTimeAndLog( + deleteFromMainTableUsingKeysSql(table, keysTableName), + "Remove updated events for table: '{}'", + table.getMainName()); + + invokeTimeAndLog( + sqlBuilder.dropTableIfExistsCascade(keysTableName), + "Drop delete-keys staging table (post)"); + } + + private String deleteFromMainTableUsingKeysSql(AnalyticsTable table, String keysTableName) { + return replace( + """ + delete from ${tableName} ax \ + using ${keysTableName} k \ + where ax.event=k.event;""", + Map.of( + "tableName", quote(table.getMainName()), + "keysTableName", quote(keysTableName))); + } + + private String getDeleteKeysTableName() { + return getAnalyticsTableType().getTableName() + "_delete_keys"; + } + + private Table newDeleteKeysTable(String keysTableName) { + return new Table(keysTableName, List.of(new Column("event", CHARACTER_11)), List.of()); + } + @Override protected List getPartitionChecks(Integer year, Date endDate) { Objects.requireNonNull(year); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java index afc6506c2eb8..d45b9f0da2f4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java @@ -331,8 +331,10 @@ void testRegularAnalyticsTableHasUniqueKeyOnEvent() { } @Test - @DisplayName("removeUpdatedData emits DELETE...USING for a registration program on Doris") - void testRemoveUpdatedDataUsesUsingJoinForRegistration() { + @DisplayName( + "removeUpdatedData materializes event keys natively for a registration program on Doris," + + " instead of a federated join inside DELETE") + void testRemoveUpdatedDataMaterializesKeysNativelyForRegistration() { Program program = createProgram('A'); program.setProgramType(ProgramType.WITH_REGISTRATION); @@ -361,16 +363,49 @@ void testRemoveUpdatedDataUsesUsingJoinForRegistration() { subject.removeUpdatedData(tables); ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); - verify(jdbcTemplate).execute(sql.capture()); - - assertTrue(sql.getValue().contains("using")); - assertTrue(sql.getValue().contains("ax.event = ev.uid")); - assertTrue(!sql.getValue().contains("in (")); + verify(jdbcTemplate, Mockito.times(5)).execute(sql.capture()); + List statements = sql.getAllValues(); + + String drop1 = statements.get(0); + assertTrue(drop1.toLowerCase().contains("drop table if exists")); + assertTrue(drop1.contains("analytics_event_delete_keys")); + + String create = statements.get(1); + assertTrue(create.toLowerCase().contains("create table")); + assertTrue(create.contains("analytics_event_delete_keys")); + assertTrue(create.contains("`event`")); + assertTrue(!create.contains("`year`")); + + String insert = statements.get(2); + assertTrue(insert.toLowerCase().contains("insert into")); + assertTrue(insert.contains("analytics_event_delete_keys")); + assertTrue(insert.contains("trackerevent")); + assertTrue(insert.contains("enrollment")); + assertTrue(insert.contains("en.programid =")); + assertTrue(insert.contains("ev.lastupdated >=")); + assertTrue(insert.contains("ev.lastupdated <")); + assertTrue( + insert.contains("(event) select ev.uid"), + () -> "Expected column-list-to-select-list pairing to match exactly, got: " + insert); + + String delete = statements.get(3); + assertTrue(delete.toLowerCase().contains("delete from")); + assertTrue(delete.toLowerCase().contains("using")); + assertTrue(delete.contains("analytics_event_delete_keys")); + assertTrue(delete.contains("ax.event=k.event")); + assertTrue(!delete.contains("trackerevent")); + assertTrue(!delete.contains("enrollment")); + + String drop2 = statements.get(4); + assertTrue(drop2.toLowerCase().contains("drop table")); + assertTrue(drop2.contains("analytics_event_delete_keys")); } @Test - @DisplayName("removeUpdatedData emits DELETE...USING for a single event program on Doris") - void testRemoveUpdatedDataUsesUsingJoinForSingleEvent() { + @DisplayName( + "removeUpdatedData materializes event keys natively for a single event program on Doris," + + " instead of a federated join inside DELETE") + void testRemoveUpdatedDataMaterializesKeysNativelyForSingleEvent() { ProgramStage psA = new ProgramStage(); psA.setId(123456); @@ -403,10 +438,39 @@ void testRemoveUpdatedDataUsesUsingJoinForSingleEvent() { subject.removeUpdatedData(tables); ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); - verify(jdbcTemplate).execute(sql.capture()); - - assertTrue(sql.getValue().contains("using")); - assertTrue(sql.getValue().contains("ax.event =")); - assertTrue(!sql.getValue().contains("in (")); + verify(jdbcTemplate, Mockito.times(5)).execute(sql.capture()); + List statements = sql.getAllValues(); + + String drop1 = statements.get(0); + assertTrue(drop1.toLowerCase().contains("drop table if exists")); + assertTrue(drop1.contains("analytics_event_delete_keys")); + + String create = statements.get(1); + assertTrue(create.toLowerCase().contains("create table")); + assertTrue(create.contains("analytics_event_delete_keys")); + assertTrue(create.contains("`event`")); + assertTrue(!create.contains("`year`")); + + String insert = statements.get(2); + assertTrue(insert.toLowerCase().contains("insert into")); + assertTrue(insert.contains("analytics_event_delete_keys")); + assertTrue(insert.contains("singleevent")); + assertTrue(insert.contains("ev.programstageid =")); + assertTrue(insert.contains("ev.lastupdated >=")); + assertTrue(insert.contains("ev.lastupdated <")); + assertTrue( + insert.contains("(event) select ev.uid"), + () -> "Expected column-list-to-select-list pairing to match exactly, got: " + insert); + + String delete = statements.get(3); + assertTrue(delete.toLowerCase().contains("delete from")); + assertTrue(delete.toLowerCase().contains("using")); + assertTrue(delete.contains("analytics_event_delete_keys")); + assertTrue(delete.contains("ax.event=k.event")); + assertTrue(!delete.contains("singleevent")); + + String drop2 = statements.get(4); + assertTrue(drop2.toLowerCase().contains("drop table")); + assertTrue(drop2.contains("analytics_event_delete_keys")); } } From a14a6d695eaaac6aa0e58b1e40cae3ea419151c8 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 28 Sep 2026 12:27:34 +0200 Subject: [PATCH 22/31] fix: reject continuous updates against non-unique-key or bounded-years Doris tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses two review comments from Luciano on #24440: - A main table created before unique-key analytics tables were introduced stays on the duplicate-key model until a full rebuild recreates it, but removeUpdatedData()'s DELETE requires unique-key (Doris only supports DELETE on unique-key/MOW tables). Add AnalyticsTableManager. isReadyForContinuousUpdate(), checked before a continuous run is attempted: it queries SHOW CREATE TABLE and refuses to proceed if the existing main table isn't unique-key, instead of letting the DELETE fail and letting the update continue regardless (removeUpdatedData()'s stage uses FailurePolicy.SKIP_STAGE, so a failure there doesn't abort the job on its own). Also make the abort explicit if removeUpdatedData() fails for any other reason, rather than proceeding to swap. - A bounded lastYears=1..N update against an existing main table on a continuous-analytics engine (Doris) previously discarded the rebuilt staging data silently, since removeUpdatedData() only runs for the latest-partition (lastYears=0) case and merging without it would leave stale/deleted rows behind. Fail loudly instead: only a full rebuild or lastYears=0 are supported once the main table exists on such an engine; bounded updates are tracked separately as follow-up work. Postgres is unaffected (no declarative partitioning); ClickHouse is unaffected (no continuous analytics support). 🤖 AI Assisted --- .../dhis/analytics/AnalyticsTableManager.java | 15 +++- .../table/AbstractJdbcTableManager.java | 65 ++++++++++++++- .../table/DefaultAnalyticsTableService.java | 31 ++++++- .../table/AnalyticsTableServiceTest.java | 61 +++++++++++++- .../JdbcAnalyticsTableManagerDorisTest.java | 81 ++++++++++++++++--- .../hisp/dhis/db/sql/AbstractSqlBuilder.java | 7 +- .../org/hisp/dhis/db/sql/DorisSqlBuilder.java | 7 +- .../java/org/hisp/dhis/db/sql/SqlBuilder.java | 12 ++- .../hisp/dhis/db/sql/DorisSqlBuilderTest.java | 7 +- 9 files changed, 264 insertions(+), 22 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java index de245125919c..f18d8b510400 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -94,6 +94,19 @@ default boolean hasUpdatedLatestData(Date startDate, Date endDate) { */ void preCreateTables(AnalyticsTableUpdateParams params); + /** + * Indicates whether the given tables are in a state that supports a continuous ("latest" + * partition) update. For engines requiring unique-key analytics tables (e.g. Doris), a main table + * created before that support was introduced remains on the previous key model until a full + * rebuild recreates it, and continuous updates must not be attempted against it. + * + * @param tables the list of {@link AnalyticsTable}. + * @return true if a continuous update can proceed. + */ + default boolean isReadyForContinuousUpdate(List tables) { + return true; + } + /** * Removes updated and deleted data from the given tables for "latest" partition update. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index ce1c2fb084fd..52ae40934f94 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -253,6 +253,20 @@ public void swapTable(AnalyticsTableUpdateParams params, AnalyticsTable table) { // removeUpdatedData(). String fromTable = sqlBuilder.quote(table.getName()); jdbcTemplate.execute(sqlBuilder.insertIntoSelectFrom(table.fromStaging(), fromTable)); + } else if (sqlBuilder.supportsContinuousAnalytics()) { + // A bounded lastYears update (not the "latest partition" case above) targeting a main + // table that already exists on a continuous-analytics-capable engine. removeUpdatedData() + // only runs for the latest-partition case, so merging staging data here would leave stale + // or deleted rows behind. Rather than silently discarding the rebuilt staging data (which + // would leave the main table showing outdated values with no indication anything is + // wrong), fail loudly: only lastYears=0 (continuous) or a full rebuild are supported once + // the main table exists. + throw new IllegalStateException( + format( + "Bounded lastYears update of table '{}' is not supported once the main table " + + "exists on an engine requiring unique-key analytics tables: run a full " + + "analytics table rebuild or a continuous (lastYears=0) update instead", + table.getMainName())); } dropTable(table); } @@ -374,6 +388,55 @@ private boolean tableExists(String name) { return !jdbcTemplate.queryForList(sqlBuilder.tableExists(name)).isEmpty(); } + @Override + public boolean isReadyForContinuousUpdate(List tables) { + if (!sqlBuilder.requiresUniqueKeyAnalyticsTables()) { + return true; + } + + return tables.stream().allMatch(this::isReadyForContinuousUpdate); + } + + /** + * A main table created before unique-key analytics tables were introduced remains on its original + * key model until a full rebuild recreates it. A continuous update's delete step requires the + * unique key model, so it must not be attempted against such a table. + */ + private boolean isReadyForContinuousUpdate(AnalyticsTable table) { + String tableName = table.getMainName(); + + if (!tableExists(tableName)) { + // No main table yet; the initial create uses the required key model. + return true; + } + + if (hasUniqueKeyModel(tableName)) { + return true; + } + + log.error( + "Table '{}' does not use the unique key model required for continuous analytics " + + "updates on this database; run a full analytics table rebuild to recreate it with " + + "the required key model before running a continuous (lastYears=0) update", + tableName); + + return false; + } + + private boolean hasUniqueKeyModel(String tableName) { + List> rows = + jdbcTemplate.queryForList(sqlBuilder.showCreateTable(tableName)); + + if (rows.isEmpty()) { + return false; + } + + String createTableSql = + rows.get(0).values().stream().map(String::valueOf).collect(Collectors.joining(" ")); + + return createTableSql.toLowerCase().contains("unique key"); + } + // ------------------------------------------------------------------------- // Abstract methods // ------------------------------------------------------------------------- diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index 740b8fb839f4..4bf6af46f241 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -170,10 +170,37 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { } if (params.isLatestUpdate() && sqlBuilder.supportsContinuousAnalytics()) { + progress.startingStage(format("Validating continuous update readiness: '{}'", tableType)); + boolean readyForContinuousUpdate = tableManager.isReadyForContinuousUpdate(tables); + progress.completedStage( + "Validated continuous update readiness with outcome: '{}'", readyForContinuousUpdate); + + if (!readyForContinuousUpdate) { + // A main table predating unique-key analytics tables (or otherwise not in a state that + // supports a continuous update) must not have the delete step attempted against it, see + // isReadyForContinuousUpdate() for detail; the specific reason is already logged there. + clock.logTime("Continuous analytics update aborted, not ready: '{}'", tableType); + return; + } + progress.startingStage( format("Removing updated and deleted data: '{}'", tableType), SKIP_STAGE); - progress.runStage(() -> tableManager.removeUpdatedData(tables)); + boolean removedUpdatedData = progress.runStage(() -> tableManager.removeUpdatedData(tables)); clock.logTime("Removed updated and deleted data"); + + if (!removedUpdatedData) { + // Swapping in the staged data without having removed the stale/deleted rows it is meant + // to replace would corrupt the main table (duplicate or orphaned rows), so the continuous + // update for this table type is aborted here rather than proceeding to swap. A likely + // cause is a main table created before unique-key analytics tables were introduced; a + // full analytics table rebuild recreates it with the required key type. + log.error( + "Aborting continuous analytics update for '{}': failed to remove updated and " + + "deleted data, see preceding error. A full analytics table rebuild may be " + + "required before continuous updates can run for this table.", + tableType); + return; + } } swapTables(params, tables, progress); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java index caeb67b5d453..76139c0041cc 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2023, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -258,12 +258,71 @@ void testRemoveUpdatedDataRunWhenContinuousAnalyticsSupported() { when(sqlBuilder.requiresIndexesForAnalytics()).thenReturn(false); when(sqlBuilder.supportsAnalyze()).thenReturn(false); when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); + when(tableManager.isReadyForContinuousUpdate(List.of(table))).thenReturn(true); tableService.create(params, JobProgress.noop()); verify(tableManager).removeUpdatedData(List.of(table)); } + @Test + void testRemoveUpdatedDataNotRunWhenNotReadyForContinuousUpdate() { + AnalyticsTable table = latestPartitionTableFixture(); + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build() + .withLatestPartition(); + + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(tableManager.getAnalyticsTableType()).thenReturn(AnalyticsTableType.DATA_VALUE); + when(tableManager.validState()).thenReturn(true); + when(tableManager.getAnalyticsTables(params)).thenReturn(List.of(table)); + when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(false); + when(sqlBuilder.requiresIndexesForAnalytics()).thenReturn(false); + when(sqlBuilder.supportsAnalyze()).thenReturn(false); + when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); + when(tableManager.isReadyForContinuousUpdate(List.of(table))).thenReturn(false); + + // e.g. a main table predating unique-key analytics tables on Doris: the delete step must + // never even be attempted against it, not just have its failure tolerated. + tableService.create(params, JobProgress.noop()); + + verify(tableManager, never()).removeUpdatedData(anyList()); + verify(tableManager, never()).swapTable(eq(params), any(AnalyticsTable.class)); + } + + @Test + void testSwapTableNotRunWhenRemoveUpdatedDataFails() { + AnalyticsTable table = latestPartitionTableFixture(); + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build() + .withLatestPartition(); + + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(tableManager.getAnalyticsTableType()).thenReturn(AnalyticsTableType.DATA_VALUE); + when(tableManager.validState()).thenReturn(true); + when(tableManager.getAnalyticsTables(params)).thenReturn(List.of(table)); + when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(false); + when(sqlBuilder.requiresIndexesForAnalytics()).thenReturn(false); + when(sqlBuilder.supportsAnalyze()).thenReturn(false); + when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); + when(tableManager.isReadyForContinuousUpdate(List.of(table))).thenReturn(true); + doThrow(new IllegalStateException("Delete failed")) + .when(tableManager) + .removeUpdatedData(anyList()); + + // removeUpdatedData()'s stage uses FailurePolicy.SKIP_STAGE, so its failure alone does not + // cancel the job (other table types must still be able to proceed). The table update must + // instead explicitly stop itself before swapping staged data into a main table it was unable + // to purge stale/deleted rows from. + tableService.create(params, JobProgress.noop()); + + verify(tableManager, never()).swapTable(eq(params), any(AnalyticsTable.class)); + } + private AnalyticsTable latestPartitionTableFixture() { List columns = List.of( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index 246e4e7a6e98..299216dc7952 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -313,12 +313,14 @@ void testSwapTableInsertsStagingDataIntoMainTableWhenSkippingMasterTable() { } @Test - void testSwapTableDoesNotInsertStagingDataForBoundedYearsUpdate() { + void testSwapTableRejectsBoundedYearsUpdateOnceMainTableExists() { // lastYears(1) (a normal bounded-years rebuild) makes isPartialUpdate()==true but // isLatestUpdate()==false -- distinct from the "latest partition" (lastYears=0) case the // insert-into-main branch is scoped to. removeUpdatedData() never runs for this case (it's // gated on isLatestUpdate()), so merging staging data into main here would leave deleted rows - // behind forever. The insert branch must not fire. + // behind forever. Silently dropping the rebuilt staging data instead (the prior behavior) + // left the main table showing stale values with no indication anything went wrong, so this + // combination must fail loudly rather than no-op. AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder() .lastYears(1) @@ -340,17 +342,70 @@ void testSwapTableDoesNotInsertStagingDataForBoundedYearsUpdate() { when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) .thenReturn(List.of(Map.of("table_name", "analytics"))); - subject.swapTable(params, table); + org.junit.jupiter.api.Assertions.assertThrows( + IllegalStateException.class, () -> subject.swapTable(params, table)); - org.mockito.ArgumentCaptor sql = org.mockito.ArgumentCaptor.forClass(String.class); - org.mockito.Mockito.verify(jdbcTemplate, org.mockito.Mockito.times(1)).execute(sql.capture()); + org.mockito.Mockito.verify(jdbcTemplate, org.mockito.Mockito.never()) + .execute(org.mockito.ArgumentMatchers.anyString()); + } - String statement = sql.getValue(); - assertFalse( - statement.startsWith("insert into"), - () -> "Unexpected insert statement for a bounded-years update: " + statement); - assertTrue( - statement.contains("drop table"), - () -> "Expected a drop-table statement only, got: " + statement); + @Test + void testIsReadyForContinuousUpdateWhenMainTableDoesNotExistYet() { + AnalyticsTable table = dataValueTableFixture(); + + when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) + .thenReturn(List.of()); + + assertTrue(subject.isReadyForContinuousUpdate(List.of(table))); + + org.mockito.Mockito.verify(jdbcTemplate, org.mockito.Mockito.never()) + .queryForList(sqlBuilder.showCreateTable(table.getMainName())); + } + + @Test + void testIsReadyForContinuousUpdateWhenMainTableHasUniqueKey() { + AnalyticsTable table = dataValueTableFixture(); + + when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) + .thenReturn(List.of(Map.of("table_name", "analytics"))); + when(jdbcTemplate.queryForList(sqlBuilder.showCreateTable(table.getMainName()))) + .thenReturn( + List.of( + Map.of( + "Table", "analytics", + "Create Table", + "CREATE TABLE `analytics` (...) UNIQUE KEY (`id`,`year`) ..."))); + + assertTrue(subject.isReadyForContinuousUpdate(List.of(table))); + } + + @Test + void testIsReadyForContinuousUpdateWhenMainTablePredatesUniqueKeySupport() { + // A main table built before unique-key analytics tables were introduced remains on the + // duplicate key model until a full rebuild recreates it. The delete step a continuous update + // depends on only works on unique-key tables, so it must not be attempted. + AnalyticsTable table = dataValueTableFixture(); + + when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) + .thenReturn(List.of(Map.of("table_name", "analytics"))); + when(jdbcTemplate.queryForList(sqlBuilder.showCreateTable(table.getMainName()))) + .thenReturn( + List.of( + Map.of( + "Table", "analytics", + "Create Table", "CREATE TABLE `analytics` (...) DUPLICATE KEY (`id`) ..."))); + + assertFalse(subject.isReadyForContinuousUpdate(List.of(table))); + } + + private AnalyticsTable dataValueTableFixture() { + List columns = + List.of( + AnalyticsTableColumn.builder() + .name("dx") + .dataType(TEXT) + .selectExpression("dx") + .build()); + return new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, List.of(), Logged.UNLOGGED); } } diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java index daf157da406e..f20fd4aded4b 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/AbstractSqlBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2024, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -147,6 +147,11 @@ public String tableExists(Table table) { return tableExists(table.getName()); } + @Override + public String showCreateTable(String name) { + return notSupported(); + } + @Override public String countRows(Table table) { return String.format("select count(*) as row_count from %s;", quote(table.getName())); diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java index 3abaf2e27c06..8a14123b05d8 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/DorisSqlBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2024, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -553,6 +553,11 @@ public String tableExists(String name) { singleQuote(name)); } + @Override + public String showCreateTable(String name) { + return String.format("show create table %s;", quote(name)); + } + /** * Doris supports indexes but relies on concurrency and compression for query performance instead * of indexes on arbitrary columns. Read more at {@link https://t.ly/AHhJ1}. diff --git a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java index e286e62855ee..17758196c9fa 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/main/java/org/hisp/dhis/db/sql/SqlBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2024, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -553,6 +553,16 @@ String ifThenElse( */ String tableExists(String name); + /** + * Only meaningful for engines where {@link #requiresUniqueKeyAnalyticsTables()} is true, which + * may have main tables predating the introduction of unique-key analytics tables. + * + * @param name the table name. + * @return a statement which returns the table's create statement, which can be inspected to + * determine the table's key model. + */ + String showCreateTable(String name); + /** * @param table the {@link Table}. * @return a count rows statement. diff --git a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java index fa7572703f78..e1f8549a6a7a 100644 --- a/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java +++ b/dhis-2/dhis-support/dhis-support-sql/src/test/java/org/hisp/dhis/db/sql/DorisSqlBuilderTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2025, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -376,6 +376,11 @@ void testTableExists() { assertEquals(expected, sqlBuilder.tableExists("immunization")); } + @Test + void testShowCreateTable() { + assertEquals("show create table `immunization`;", sqlBuilder.showCreateTable("immunization")); + } + @Test void testCountRows() { String expected = From 37201c94c64295e399ca12fcc060b4848a3bfd23 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Mon, 28 Sep 2026 12:34:38 +0200 Subject: [PATCH 23/31] refactor: extract swapTable() publish decision into a resolver method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses Luciano's readability comment on #24440 (gist: f93f938afb471511281593203159edd2): swapTable() packed together several independent decisions (full vs. partial vs. continuous update, whether the main table exists, how the database manages partitions, and how to publish: replace/reparent/merge) into nested conditionals. Extract a TableUpdateAction enum (REPLACE_TABLE, REPLACE_PARTITIONS, MERGE_ROWS) and a resolveUpdateAction() method that makes the decision once, up front, then dispatches to three small named methods. This also generalizes the two fail-loudly behaviors from the previous commit from Doris-only to any declarative-partitioning engine without continuous analytics support (currently ClickHouse): a continuous (lastYears=0) or bounded (lastYears=1..N) update against an existing main table on such an engine now throws instead of silently discarding the rebuilt staging data, since there is no purge step available to make either publish safe. 🤖 AI Assisted --- .../table/AbstractJdbcTableManager.java | 144 +++++++++++++----- ...bcAnalyticsTableManagerClickHouseTest.java | 70 +++++---- 2 files changed, 144 insertions(+), 70 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 52ae40934f94..549e0c537cf2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -222,54 +222,120 @@ public void createIndex(Index index) { jdbcTemplate.execute(sql); } + /** How staged data for a table update is published to the main table. */ + private enum TableUpdateAction { + REPLACE_TABLE, + REPLACE_PARTITIONS, + MERGE_ROWS + } + @Override public void swapTable(AnalyticsTableUpdateParams params, AnalyticsTable table) { - boolean tableExists = tableExists(table.getMainName()); - boolean skipMasterTable = - params.isPartialUpdate() && tableExists && table.getTableType().isLatestPartition(); + TableUpdateAction action = resolveUpdateAction(params, table); - log.info("Swapping table: '{}'", table.getMainName()); - log.info("Master table exists: '{}', skip master table: '{}'", tableExists, skipMasterTable); + log.info("Swapping table: '{}', action: '{}'", table.getMainName(), action); - List swappedPartitions = new UniqueArrayList<>(); + switch (action) { + case REPLACE_TABLE -> replaceMainTable(table); + case REPLACE_PARTITIONS -> replaceAndAttachPartitions(table); + case MERGE_ROWS -> mergeIntoMainTable(table); + } + } + /** + * Determines how staged data for the given table should be published to the main table, based on + * the update type, whether the table type supports a "latest partition", whether a main table + * already exists, and how the target database manages partitions. + */ + private TableUpdateAction resolveUpdateAction( + AnalyticsTableUpdateParams params, AnalyticsTable table) { + boolean supportsLatestPartition = table.getTableType().isLatestPartition(); + + if (params.isLatestUpdate() + && supportsLatestPartition + && !sqlBuilder.supportsContinuousAnalytics()) { + // Stale/deleted rows are only removed for the latest-partition case, and only on engines + // that support continuous analytics (see removeUpdatedData()); merging or replacing without + // that purge would corrupt or silently discard data, so refuse rather than guess. + throw new IllegalStateException( + format( + "Continuous (lastYears=0) update of table '{}' is not supported by this database", + table.getMainName())); + } + + // Full updates, and table types with no "latest partition" concept, always replace the table. + if (!params.isPartialUpdate() || !supportsLatestPartition) { + return TableUpdateAction.REPLACE_TABLE; + } + + // No existing table to preserve; treat as the initial build. + if (!tableExists(table.getMainName())) { + return TableUpdateAction.REPLACE_TABLE; + } + + // Postgres publishes partial updates by replacing physical (inheritance-based) partitions. if (!sqlBuilder.supportsDeclarativePartitioning()) { - table.getTablePartitions().forEach(part -> swapTable(part, part.getMainName())); - table.getTablePartitions().forEach(part -> swappedPartitions.add(part.fromStaging())); + return TableUpdateAction.REPLACE_PARTITIONS; } - if (!skipMasterTable) { - // Full replace update and main table exist, swap main table - swapTable(table, table.getMainName()); - } else { - // Incremental append update, update parent of partitions to existing main table - if (!sqlBuilder.supportsDeclarativePartitioning()) { - swappedPartitions.forEach( - partition -> swapParentTable(partition, table.getName(), table.getMainName())); - } else if (params.isLatestUpdate() && sqlBuilder.supportsContinuousAnalytics()) { - // For databases with declarative partitioning that support continuous analytics (e.g. - // Doris): staging data must be inserted directly into the main table since there is no - // inheritance-based partition attachment. Stale rows are already removed by - // removeUpdatedData(). - String fromTable = sqlBuilder.quote(table.getName()); - jdbcTemplate.execute(sqlBuilder.insertIntoSelectFrom(table.fromStaging(), fromTable)); - } else if (sqlBuilder.supportsContinuousAnalytics()) { - // A bounded lastYears update (not the "latest partition" case above) targeting a main - // table that already exists on a continuous-analytics-capable engine. removeUpdatedData() - // only runs for the latest-partition case, so merging staging data here would leave stale - // or deleted rows behind. Rather than silently discarding the rebuilt staging data (which - // would leave the main table showing outdated values with no indication anything is - // wrong), fail loudly: only lastYears=0 (continuous) or a full rebuild are supported once - // the main table exists. - throw new IllegalStateException( - format( - "Bounded lastYears update of table '{}' is not supported once the main table " - + "exists on an engine requiring unique-key analytics tables: run a full " - + "analytics table rebuild or a continuous (lastYears=0) update instead", - table.getMainName())); - } - dropTable(table); + // Declarative-partitioning engines with no inheritance-based partition attachment (e.g. + // Doris) publish a continuous update by merging staged rows into the persistent main table. + if (params.isLatestUpdate()) { + return TableUpdateAction.MERGE_ROWS; + } + + // A bounded lastYears update against a main table that already exists, with no publication + // method available (removeUpdatedData() only runs for the latest-partition case, so merging + // here would leave stale/deleted rows behind). Fail loudly rather than silently discard the + // rebuilt staging data. + throw new IllegalStateException( + format( + "Bounded lastYears update of table '{}' is not supported once the main table exists " + + "on this database: run a full analytics table rebuild or a continuous " + + "(lastYears=0) update instead", + table.getMainName())); + } + + private void replaceMainTable(AnalyticsTable table) { + replacePhysicalPartitions(table); + swapTable(table, table.getMainName()); + } + + private void replaceAndAttachPartitions(AnalyticsTable table) { + List
partitions = replacePhysicalPartitions(table); + + partitions.forEach( + partition -> swapParentTable(partition, table.getName(), table.getMainName())); + + dropTable(table); + } + + private void mergeIntoMainTable(AnalyticsTable table) { + String fromTable = sqlBuilder.quote(table.getName()); + jdbcTemplate.execute(sqlBuilder.insertIntoSelectFrom(table.fromStaging(), fromTable)); + dropTable(table); + } + + /** + * Swaps each of the table's physical (inheritance-based) partitions from staging to main. A no-op + * for declarative-partitioning engines, which have no separate physical partition tables. + * + * @return the swapped partitions, for a caller that needs to reparent them onto an existing main + * table. + */ + private List
replacePhysicalPartitions(AnalyticsTable table) { + if (sqlBuilder.supportsDeclarativePartitioning()) { + return List.of(); + } + + List
partitions = new UniqueArrayList<>(); + + for (AnalyticsTablePartition partition : table.getTablePartitions()) { + swapTable(partition, partition.getMainName()); + partitions.add(partition.fromStaging()); } + + return partitions; } @Override diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java index 28e35b861b13..4bcf6d052769 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,8 +30,6 @@ package org.hisp.dhis.analytics.table; import static org.hisp.dhis.db.model.DataType.TEXT; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; import java.util.Date; @@ -61,7 +59,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; @@ -72,10 +69,10 @@ import org.springframework.jdbc.core.JdbcTemplate; /** - * Covers the {@code swapTable()} branch for databases with declarative partitioning that do NOT - * support continuous analytics (e.g. ClickHouse): staging data must not be inserted into the main - * table, the staging table is simply dropped, same as before the Doris continuous-analytics - * pipeline was wired up. + * Covers {@code swapTable()} for databases with declarative partitioning that do NOT support + * continuous analytics (e.g. ClickHouse): with no purge step available, neither a continuous nor a + * bounded-years update can safely publish staged data into an existing main table, so both must be + * rejected rather than silently discarding the staging data or corrupting the main table. */ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) @@ -119,13 +116,45 @@ void setUp() { } @Test - void testSwapTableDoesNotInsertStagingDataWhenContinuousAnalyticsNotSupported() { + void testSwapTableRejectsContinuousUpdateWhenNotSupported() { AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder() .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) .build() .withLatestPartition(); + AnalyticsTable table = dataValueTableFixture(); + + // ClickHouse supports declarative partitioning but not continuous analytics: without the + // purge removeUpdatedData() would otherwise perform, neither replacing nor merging staged + // data into an existing main table is safe, so the update must fail rather than silently + // dropping the rebuilt staging data (the prior behavior) or corrupting the main table. + org.junit.jupiter.api.Assertions.assertThrows( + IllegalStateException.class, () -> subject.swapTable(params, table)); + + Mockito.verify(jdbcTemplate, Mockito.never()).execute(Mockito.anyString()); + } + + @Test + void testSwapTableRejectsBoundedYearsUpdateOnceMainTableExists() { + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .lastYears(1) + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build(); + + AnalyticsTable table = dataValueTableFixture(); + + when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) + .thenReturn(List.of(Map.of("table_name", "analytics"))); + + org.junit.jupiter.api.Assertions.assertThrows( + IllegalStateException.class, () -> subject.swapTable(params, table)); + + Mockito.verify(jdbcTemplate, Mockito.never()).execute(Mockito.anyString()); + } + + private AnalyticsTable dataValueTableFixture() { List columns = List.of( AnalyticsTableColumn.builder() @@ -133,27 +162,6 @@ void testSwapTableDoesNotInsertStagingDataWhenContinuousAnalyticsNotSupported() .dataType(TEXT) .selectExpression("dx") .build()); - AnalyticsTable table = - new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, List.of(), Logged.UNLOGGED); - - // Main table already exists, and params.isPartialUpdate() (via withLatestPartition()) plus - // AnalyticsTableType.DATA_VALUE.isLatestPartition()==true together push swapTable() into the - // skipMasterTable branch. ClickHouse supports declarative partitioning but not continuous - // analytics, so neither the reparenting branch nor the new insert-into-main branch should - // fire here. - when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) - .thenReturn(List.of(Map.of("table_name", "analytics"))); - - subject.swapTable(params, table); - - ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); - Mockito.verify(jdbcTemplate, Mockito.times(1)).execute(sql.capture()); - - String statement = sql.getValue(); - assertFalse( - statement.startsWith("insert into"), () -> "Unexpected insert statement: " + statement); - assertTrue( - statement.contains("drop table"), - () -> "Expected a drop-table statement, got: " + statement); + return new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, List.of(), Logged.UNLOGGED); } } From 009045837494e672f98aae62a2629ea1b8403b77 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Tue, 29 Sep 2026 07:16:56 +0200 Subject: [PATCH 24/31] fix: keep bounded lastYears updates working on Doris MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A bounded lastYears=N update against an existing Doris main table threw an IllegalStateException. Before this PR the same update gave correct results by replacing the whole table, so throwing broke something that worked, including scheduled analytics jobs that use "last years". On declarative-partitioning engines the populate step does not filter by year, so the staging table holds every year and replacing the main table is correct. Return REPLACE_TABLE for this case on engines that support continuous analytics (Doris) and log a warning that the whole table was rebuilt. ClickHouse, where this path has not been verified, still throws. 🤖 AI Assisted --- .../table/AbstractJdbcTableManager.java | 19 ++++++++++--- .../JdbcAnalyticsTableManagerDorisTest.java | 27 ++++++++++++------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 549e0c537cf2..cb515efc18f3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -284,10 +284,21 @@ private TableUpdateAction resolveUpdateAction( return TableUpdateAction.MERGE_ROWS; } - // A bounded lastYears update against a main table that already exists, with no publication - // method available (removeUpdatedData() only runs for the latest-partition case, so merging - // here would leave stale/deleted rows behind). Fail loudly rather than silently discard the - // rebuilt staging data. + // A bounded lastYears update against a main table that already exists. Merging is not an option + // here, since removeUpdatedData() only runs for the latest-partition case and merging would + // leave stale/deleted rows behind. On declarative-partitioning engines the populate step does + // not filter by year, so the staging table holds every year and replacing the whole main table + // gives correct results, as it did before continuous analytics support was added for Doris. + // Limited to engines supporting continuous analytics (Doris); this path has not been verified + // on ClickHouse, which still fails loudly. + if (sqlBuilder.supportsContinuousAnalytics()) { + log.warn( + "Bounded lastYears update of table '{}' rebuilds the whole table with all years on this " + + "database, use a continuous (lastYears=0) update for incremental updates", + table.getMainName()); + return TableUpdateAction.REPLACE_TABLE; + } + throw new IllegalStateException( format( "Bounded lastYears update of table '{}' is not supported once the main table exists " diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index 299216dc7952..b268d837bd69 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -313,14 +313,14 @@ void testSwapTableInsertsStagingDataIntoMainTableWhenSkippingMasterTable() { } @Test - void testSwapTableRejectsBoundedYearsUpdateOnceMainTableExists() { + void testSwapTableReplacesMainTableForBoundedYearsUpdateOnceMainTableExists() { // lastYears(1) (a normal bounded-years rebuild) makes isPartialUpdate()==true but // isLatestUpdate()==false -- distinct from the "latest partition" (lastYears=0) case the // insert-into-main branch is scoped to. removeUpdatedData() never runs for this case (it's // gated on isLatestUpdate()), so merging staging data into main here would leave deleted rows - // behind forever. Silently dropping the rebuilt staging data instead (the prior behavior) - // left the main table showing stale values with no indication anything went wrong, so this - // combination must fail loudly rather than no-op. + // behind forever, and dropping the staging data would leave stale values. The staging table + // holds every year on Doris (no year filter on declarative partitioning), so the whole main + // table is replaced instead. AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder() .lastYears(1) @@ -337,16 +337,25 @@ void testSwapTableRejectsBoundedYearsUpdateOnceMainTableExists() { AnalyticsTable table = new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, List.of(), Logged.UNLOGGED); - // Same skipMasterTable=true preconditions as the positive test: main table already exists, + // Same preconditions as the merge test: main table already exists, // params.isPartialUpdate()==true (via lastYears(1)), and DATA_VALUE.isLatestPartition()==true. when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) .thenReturn(List.of(Map.of("table_name", "analytics"))); - org.junit.jupiter.api.Assertions.assertThrows( - IllegalStateException.class, () -> subject.swapTable(params, table)); + subject.swapTable(params, table); - org.mockito.Mockito.verify(jdbcTemplate, org.mockito.Mockito.never()) - .execute(org.mockito.ArgumentMatchers.anyString()); + org.mockito.ArgumentCaptor sql = org.mockito.ArgumentCaptor.forClass(String.class); + org.mockito.Mockito.verify(jdbcTemplate, org.mockito.Mockito.atLeastOnce()) + .execute(sql.capture()); + + List statements = sql.getAllValues(); + assertTrue( + statements.stream() + .anyMatch(s -> s.contains("alter table `analytics_temp` rename `analytics`;")), + () -> "Expected the staging table to replace the main table, got: " + statements); + assertTrue( + statements.stream().noneMatch(s -> s.startsWith("insert into")), + () -> "Bounded update must not merge into the main table, got: " + statements); } @Test From 0539f4cb611da82081021655a4ef826b2661f260 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Tue, 29 Sep 2026 07:35:00 +0200 Subject: [PATCH 25/31] fix: show continuous update readiness failure in the analytics job log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a Doris analytics table was created before continuous updates were supported, the continuous update for that table type was skipped with an ERROR in the server log only. The job log showed the readiness check at INFO level and the job ended as updated, so administrators had no sign in the job output of why continuous updates were not being applied. Report the readiness check as a failed stage with SKIP_STAGE, so the reason and the required full rebuild appear as an ERROR in the job log while the other table types are still updated. 🤖 AI Assisted --- .../table/DefaultAnalyticsTableService.java | 16 ++++++++++++---- .../table/AnalyticsTableServiceTest.java | 8 +++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index 4bf6af46f241..894187255249 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -170,19 +170,27 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { } if (params.isLatestUpdate() && sqlBuilder.supportsContinuousAnalytics()) { - progress.startingStage(format("Validating continuous update readiness: '{}'", tableType)); + progress.startingStage( + format("Validating continuous update readiness: '{}'", tableType), SKIP_STAGE); boolean readyForContinuousUpdate = tableManager.isReadyForContinuousUpdate(tables); - progress.completedStage( - "Validated continuous update readiness with outcome: '{}'", readyForContinuousUpdate); if (!readyForContinuousUpdate) { // A main table predating unique-key analytics tables (or otherwise not in a state that // supports a continuous update) must not have the delete step attempted against it, see - // isReadyForContinuousUpdate() for detail; the specific reason is already logged there. + // isReadyForContinuousUpdate() for detail. The stage is reported as failed so the reason + // shows in the job log, while the remaining table types still get updated. + progress.failedStage( + format( + "Continuous update skipped for '{}': the analytics table was created before " + + "continuous updates were supported on this database. Run a full analytics " + + "table rebuild before running a continuous (lastYears=0) update", + tableType)); clock.logTime("Continuous analytics update aborted, not ready: '{}'", tableType); return; } + progress.completedStage("Validated continuous update readiness: '{}'", tableType); + progress.startingStage( format("Removing updated and deleted data: '{}'", tableType), SKIP_STAGE); boolean removedUpdatedData = progress.runStage(() -> tableManager.removeUpdatedData(tables)); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java index 76139c0041cc..23406aa1383b 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java @@ -36,10 +36,12 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.contains; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -284,12 +286,16 @@ void testRemoveUpdatedDataNotRunWhenNotReadyForContinuousUpdate() { when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); when(tableManager.isReadyForContinuousUpdate(List.of(table))).thenReturn(false); + JobProgress progress = spy(JobProgress.noop()); + // e.g. a main table predating unique-key analytics tables on Doris: the delete step must // never even be attempted against it, not just have its failure tolerated. - tableService.create(params, JobProgress.noop()); + tableService.create(params, progress); verify(tableManager, never()).removeUpdatedData(anyList()); verify(tableManager, never()).swapTable(eq(params), any(AnalyticsTable.class)); + // The reason must reach the job log, not only the server log. + verify(progress).failedStage(contains("Run a full analytics table rebuild")); } @Test From 50d1bceea1891becb71315411f548a0e4b6ea895 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Tue, 29 Sep 2026 08:18:35 +0200 Subject: [PATCH 26/31] fix: keep continuous and bounded analytics updates working on ClickHouse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On ClickHouse a continuous (lastYears=0) update threw an IllegalStateException at the swap step, failing the whole analytics job. Before this PR the same update gave correct results by repopulating and replacing the whole table. The latest-update partition change in getTablePartitions() also applied to ClickHouse, narrowing the staging table to changed rows only, so simply replacing the main table would have lost data. Only use the narrowed latest-update partition on engines that support continuous analytics (Doris), so the staging table holds all data on ClickHouse, and replace the whole main table instead of throwing for a continuous update on engines without continuous support and for a bounded lastYears update on any declarative-partitioning engine. Both log a warning that the whole table was rebuilt. 🤖 AI Assisted --- .../table/AbstractJdbcTableManager.java | 47 +++++++------------ .../table/DefaultAnalyticsTableService.java | 8 +++- .../table/AnalyticsTableServiceTest.java | 34 ++++++++++++++ ...bcAnalyticsTableManagerClickHouseTest.java | 42 +++++++++++------ 4 files changed, 85 insertions(+), 46 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index cb515efc18f3..3cda53ef7f45 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -251,23 +251,23 @@ private TableUpdateAction resolveUpdateAction( AnalyticsTableUpdateParams params, AnalyticsTable table) { boolean supportsLatestPartition = table.getTableType().isLatestPartition(); - if (params.isLatestUpdate() - && supportsLatestPartition - && !sqlBuilder.supportsContinuousAnalytics()) { - // Stale/deleted rows are only removed for the latest-partition case, and only on engines - // that support continuous analytics (see removeUpdatedData()); merging or replacing without - // that purge would corrupt or silently discard data, so refuse rather than guess. - throw new IllegalStateException( - format( - "Continuous (lastYears=0) update of table '{}' is not supported by this database", - table.getMainName())); - } - // Full updates, and table types with no "latest partition" concept, always replace the table. if (!params.isPartialUpdate() || !supportsLatestPartition) { return TableUpdateAction.REPLACE_TABLE; } + // Without continuous analytics support (e.g. ClickHouse) there is no step removing updated and + // deleted rows, so a continuous update cannot be merged. The staging table holds all data on + // such engines (see DefaultAnalyticsTableService.getTablePartitions()), so replacing the whole + // main table gives correct results, as a full rebuild would. + if (params.isLatestUpdate() && !sqlBuilder.supportsContinuousAnalytics()) { + log.warn( + "Continuous (lastYears=0) update of table '{}' rebuilds the whole table on this " + + "database, which does not support incremental updates", + table.getMainName()); + return TableUpdateAction.REPLACE_TABLE; + } + // No existing table to preserve; treat as the initial build. if (!tableExists(table.getMainName())) { return TableUpdateAction.REPLACE_TABLE; @@ -288,23 +288,12 @@ private TableUpdateAction resolveUpdateAction( // here, since removeUpdatedData() only runs for the latest-partition case and merging would // leave stale/deleted rows behind. On declarative-partitioning engines the populate step does // not filter by year, so the staging table holds every year and replacing the whole main table - // gives correct results, as it did before continuous analytics support was added for Doris. - // Limited to engines supporting continuous analytics (Doris); this path has not been verified - // on ClickHouse, which still fails loudly. - if (sqlBuilder.supportsContinuousAnalytics()) { - log.warn( - "Bounded lastYears update of table '{}' rebuilds the whole table with all years on this " - + "database, use a continuous (lastYears=0) update for incremental updates", - table.getMainName()); - return TableUpdateAction.REPLACE_TABLE; - } - - throw new IllegalStateException( - format( - "Bounded lastYears update of table '{}' is not supported once the main table exists " - + "on this database: run a full analytics table rebuild or a continuous " - + "(lastYears=0) update instead", - table.getMainName())); + // gives correct results. + log.warn( + "Bounded lastYears update of table '{}' rebuilds the whole table with all years on this " + + "database, use a continuous (lastYears=0) update for incremental updates", + table.getMainName()); + return TableUpdateAction.REPLACE_TABLE; } private void replaceMainTable(AnalyticsTable table) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index 894187255249..5092b8408677 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -390,11 +390,15 @@ List getTablePartitions( if (table.hasTablePartitions() && !sqlBuilder.supportsDeclarativePartitioning()) { // Each partition is its own physical table, so its own real date range and name apply. partitions.addAll(table.getTablePartitions()); - } else if (table.hasTablePartitions() && isLatestUpdate) { + } else if (table.hasTablePartitions() + && isLatestUpdate + && sqlBuilder.supportsContinuousAnalytics()) { // A single physical table serves every logical partition on this engine (CREATE only // ever builds the master table's name), so the continuous/latest-update partition must // target that name - while still carrying its own real date range, needed to correctly - // scope the populate window. + // scope the populate window. Engines without continuous analytics support (ClickHouse) + // fall through to the fake partition below, so the staging table holds all data and can + // replace the whole main table. AnalyticsTablePartition latest = table.getTablePartitions().get(0); partitions.add( new AnalyticsTablePartition( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java index 23406aa1383b..a33916ec4c50 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java @@ -33,6 +33,7 @@ import static org.hisp.dhis.db.model.DataType.TEXT; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; @@ -135,6 +136,7 @@ void testGetTablePartitions() { @Test void testGetTablePartitionsUsesRealPartitionForLatestUpdateOnDeclarativePartitioningEngine() { when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(true); + when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); List columns = List.of( @@ -169,6 +171,38 @@ void testGetTablePartitionsUsesRealPartitionForLatestUpdateOnDeclarativePartitio + " not a year-suffixed name nothing ever creates"); } + @Test + void testGetTablePartitionsUsesFakePartitionForLatestUpdateWithoutContinuousAnalytics() { + // e.g. ClickHouse: with no step removing updated and deleted rows, a continuous update + // replaces the whole main table, so the staging table must hold all data rather than only + // the rows changed within the latest partition's date range. + when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(true); + when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(false); + + List columns = + List.of( + AnalyticsTableColumn.builder() + .name("dx") + .dataType(TEXT) + .selectExpression("dx") + .build()); + + AnalyticsTable table = + new AnalyticsTable(AnalyticsTableType.DATA_VALUE, columns, List.of("dx"), Logged.UNLOGGED); + table.addTablePartition( + List.of(), + AnalyticsTablePartition.LATEST_PARTITION, + new DateTime(2026, 7, 31, 12, 48, 20).toDate(), + new DateTime(2026, 7, 31, 12, 55, 0).toDate()); + + List partitions = + tableService.getTablePartitions(List.of(table), true); + + assertEquals(1, partitions.size()); + assertFalse(partitions.get(0).isLatestPartition()); + assertNull(partitions.get(0).getStartDate()); + } + @Test void testGetTablePartitionsUsesFakePartitionForRegularUpdateOnDeclarativePartitioningEngine() { when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(true); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java index 4bcf6d052769..fc09f2cf5ba3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java @@ -30,6 +30,7 @@ package org.hisp.dhis.analytics.table; import static org.hisp.dhis.db.model.DataType.TEXT; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; import java.util.Date; @@ -59,6 +60,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; @@ -70,9 +72,9 @@ /** * Covers {@code swapTable()} for databases with declarative partitioning that do NOT support - * continuous analytics (e.g. ClickHouse): with no purge step available, neither a continuous nor a - * bounded-years update can safely publish staged data into an existing main table, so both must be - * rejected rather than silently discarding the staging data or corrupting the main table. + * continuous analytics (e.g. ClickHouse): with no purge step available, staged data can not be + * merged into an existing main table. The staging table holds all data on such engines, so both a + * continuous and a bounded-years update replace the whole main table, as a full rebuild would. */ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) @@ -116,7 +118,7 @@ void setUp() { } @Test - void testSwapTableRejectsContinuousUpdateWhenNotSupported() { + void testSwapTableReplacesMainTableForContinuousUpdateWhenNotSupported() { AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder() .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) @@ -125,18 +127,16 @@ void testSwapTableRejectsContinuousUpdateWhenNotSupported() { AnalyticsTable table = dataValueTableFixture(); - // ClickHouse supports declarative partitioning but not continuous analytics: without the - // purge removeUpdatedData() would otherwise perform, neither replacing nor merging staged - // data into an existing main table is safe, so the update must fail rather than silently - // dropping the rebuilt staging data (the prior behavior) or corrupting the main table. - org.junit.jupiter.api.Assertions.assertThrows( - IllegalStateException.class, () -> subject.swapTable(params, table)); + when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) + .thenReturn(List.of(Map.of("table_name", "analytics"))); + + subject.swapTable(params, table); - Mockito.verify(jdbcTemplate, Mockito.never()).execute(Mockito.anyString()); + assertReplacedMainTable(); } @Test - void testSwapTableRejectsBoundedYearsUpdateOnceMainTableExists() { + void testSwapTableReplacesMainTableForBoundedYearsUpdateOnceMainTableExists() { AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder() .lastYears(1) @@ -148,10 +148,22 @@ void testSwapTableRejectsBoundedYearsUpdateOnceMainTableExists() { when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) .thenReturn(List.of(Map.of("table_name", "analytics"))); - org.junit.jupiter.api.Assertions.assertThrows( - IllegalStateException.class, () -> subject.swapTable(params, table)); + subject.swapTable(params, table); + + assertReplacedMainTable(); + } - Mockito.verify(jdbcTemplate, Mockito.never()).execute(Mockito.anyString()); + private void assertReplacedMainTable() { + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + Mockito.verify(jdbcTemplate, Mockito.atLeastOnce()).execute(sql.capture()); + + List statements = sql.getAllValues(); + assertTrue( + statements.contains("rename table \"analytics_temp\" to \"analytics\";"), + () -> "Expected the staging table to replace the main table, got: " + statements); + assertTrue( + statements.stream().noneMatch(s -> s.startsWith("insert into")), + () -> "Must not merge into the main table without a purge step, got: " + statements); } private AnalyticsTable dataValueTableFixture() { From 7c2c609a1496b648e0c4c17d6a87b50e4accc852 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Thu, 1 Oct 2026 11:10:19 +0200 Subject: [PATCH 27/31] docs: clarify event staging-keys delete matches on event across years MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PRIMARY_KEY is (event, year), not event alone; year is only part of it because Doris requires the partition column in a unique key. Spell out that the staging table carries just event and the delete therefore matches across all years, so an event moved to another year is still removed. 🤖 AI Assisted --- .../analytics/table/JdbcEventAnalyticsTableManager.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 1f64ddc70f02..3cbab155b485 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -462,9 +462,11 @@ where ax.event in ( \ * #populateTable} already does). Instead, the matching rows' {@code event} UID is first * materialized into a small native staging table via that proven-fast federated {@code select}, * and the actual delete runs entirely locally against that staging table. Unlike the aggregate - * ({@code DATA_VALUE}) case, the analytics event table's natural key is {@code event} alone (see - * {@link #PRIMARY_KEY}) - one physical table per program, one row per event UID - so the staging - * table only needs to carry that single column. + * ({@code DATA_VALUE}) case, an analytics event row is identified by its {@code event} UID alone + * (one physical table per program, one row per event). {@link #PRIMARY_KEY} also includes {@code + * year} only because Doris requires the partition column in a unique key. The staging table + * therefore carries just {@code event}, and the delete matches on it across all years, so an + * event whose date moved to a different year is still removed. */ private void removeUpdatedDataViaStagingKeys( AnalyticsTable table, Program program, AnalyticsTablePartition partition) { From 385dec080a1f046ad4acd9ddb74f090dd37cccec Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Thu, 1 Oct 2026 11:29:01 +0200 Subject: [PATCH 28/31] fix: keep Doris continuous updates from skipping changes after an aborted run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a continuous analytics update aborted for a table type (the continuous update readiness check or removeUpdatedData() failed), the type was still stamped as successfully updated, and on Doris the next run's window started after the changes the aborted run never published. AnalyticsTableService.create() now returns whether the update completed; only completed types get their per-type last successful update settings. On engines with a unique key on analytics tables (Doris), the continuous update window now starts at the per-type settings rather than the shared ones, which also advance when a type was skipped or aborted. This is the read path switch deferred from DHIS2-21992 (#24899), for Doris; Postgres still uses the shared settings. create() returns true when there is no source data or nothing to update, and false when the job was cancelled. Co-authored-by: Luciano Fiandesio 🤖 AI Assisted --- .../dhis/analytics/AnalyticsTableService.java | 5 +- .../table/AbstractJdbcTableManager.java | 25 +++++++++- .../table/DefaultAnalyticsTableGenerator.java | 26 +++++----- .../table/DefaultAnalyticsTableService.java | 17 ++++--- .../table/JdbcEventAnalyticsTableManager.java | 4 +- .../table/AnalyticsTableServiceTest.java | 50 +++++++++++++++++-- .../DefaultAnalyticsTableGeneratorTest.java | 25 ++++++++++ .../JdbcAnalyticsTableManagerDorisTest.java | 45 ++++++++++++++++- ...JdbcCompletenessTableManagerDorisTest.java | 9 +++- ...bcEventAnalyticsTableManagerDorisTest.java | 8 ++- 10 files changed, 180 insertions(+), 34 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java index 3383f6f73ef8..9b5eeb5eccd0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableService.java @@ -49,8 +49,11 @@ public interface AnalyticsTableService { * * @param params the {@link AnalyticsTableUpdateParams}. * @param progress job progress tracking and control flow + * @return true if the analytics tables reflect all source data changed before the start time of + * the update, false if the update was cancelled or aborted, in which case it must not be + * recorded as a successful update of this table type. */ - void create(AnalyticsTableUpdateParams params, JobProgress progress); + boolean create(AnalyticsTableUpdateParams params, JobProgress progress); /** Drops main and staging analytics tables. */ void dropTables(); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index 3cda53ef7f45..c3311e8a3de0 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -584,8 +584,7 @@ protected AnalyticsTable getLatestAnalyticsTable( List primaryKey) { SystemSettings settings = settingsProvider.getCurrentSettings(); Date lastFullTableUpdate = settings.getLastSuccessfulAnalyticsTablesUpdate(); - Date lastLatestPartitionUpdate = settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(); - Date lastAnyTableUpdate = DateUtils.getLatest(lastLatestPartitionUpdate, lastFullTableUpdate); + Date lastAnyTableUpdate = getLastAnyTableUpdate(settings); Assert.isTrue( lastFullTableUpdate.getTime() > 0L, @@ -624,6 +623,28 @@ protected AnalyticsTable getLatestAnalyticsTable( return table; } + /** + * Returns the time of the last successful analytics table update of any kind, full or continuous. + * Engines with a unique key on analytics tables (Doris) start the continuous update window at + * this time, so it is read from the settings of this table type: the shared settings are also + * written when this table type was skipped or its update was aborted, and a window starting there + * would never pick up the changes that update did not publish. + * + * @param settings the current {@link SystemSettings}. + */ + protected Date getLastAnyTableUpdate(SystemSettings settings) { + if (sqlBuilder.requiresUniqueKeyAnalyticsTables()) { + AnalyticsTableType tableType = getAnalyticsTableType(); + return DateUtils.getLatest( + settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(tableType), + settings.getLastSuccessfulAnalyticsTablesUpdate(tableType)); + } + + return DateUtils.getLatest( + settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(), + settings.getLastSuccessfulAnalyticsTablesUpdate()); + } + /** * Executes the given SQL statement. Logs and times the operation. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java index 7bc13c1582f6..8c329bb775e2 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGenerator.java @@ -126,18 +126,17 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr } } - Set processedTypes = EnumSet.noneOf(AnalyticsTableType.class); + Set publishedTypes = EnumSet.noneOf(AnalyticsTableType.class); for (AnalyticsTableService service : analyticsTableServices) { AnalyticsTableType tableType = service.getAnalyticsTableType(); - if (!skipTypes.contains(tableType)) { - service.create(params, progress); - processedTypes.add(tableType); + if (!skipTypes.contains(tableType) && service.create(params, progress)) { + publishedTypes.add(tableType); } } progress.startingStage("Updating system settings"); - progress.runStage(() -> updateLastSuccessfulSystemSettings(params, clock, processedTypes)); + progress.runStage(() -> updateLastSuccessfulSystemSettings(params, clock, publishedTypes)); progress.startingStage("Invalidate analytics caches", SKIP_STAGE); progress.runStage(analyticsCache::invalidateAll); @@ -151,31 +150,32 @@ public void generateAnalyticsTables(AnalyticsTableUpdateParams params0, JobProgr * * @param params the {@link AnalyticsTableUpdateParams}. * @param clock the {@link Clock}. - * @param processedTypes the {@link AnalyticsTableType} values actually processed in this run, - * i.e. not in {@link AnalyticsTableUpdateParams#getSkipTableTypes()}. + * @param publishedTypes the {@link AnalyticsTableType} values published in this run, i.e. not in + * {@link AnalyticsTableUpdateParams#getSkipTableTypes()} and not cancelled or aborted, see + * {@link AnalyticsTableService#create}. */ private void updateLastSuccessfulSystemSettings( - AnalyticsTableUpdateParams params, Clock clock, Set processedTypes) { + AnalyticsTableUpdateParams params, Clock clock, Set publishedTypes) { if (params.isLatestUpdate()) { settingsService.put("keyLastSuccessfulLatestAnalyticsPartitionUpdate", params.getStartTime()); settingsService.put("keyLastSuccessfulLatestAnalyticsPartitionRuntime", clock.time()); updatePerTypeSuccessfulSystemSettings( SystemSettings::keyLastSuccessfulLatestAnalyticsPartitionUpdate, params.getStartTime(), - processedTypes); + publishedTypes); } else { settingsService.put("keyLastSuccessfulAnalyticsTablesUpdate", params.getStartTime()); settingsService.put("keyLastSuccessfulAnalyticsTablesRuntime", clock.time()); updatePerTypeSuccessfulSystemSettings( SystemSettings::keyLastSuccessfulAnalyticsTablesUpdate, params.getStartTime(), - processedTypes); + publishedTypes); } } /** * Stamps the per-{@link AnalyticsTableType} counterpart of the shared "last successful update" - * clock, for every processed type that has one registered (see {@link + * clock, for every published type that has one registered (see {@link * AnalyticsTableType#isLatestPartition()}). A type without a registered key is skipped rather * than attempted, since {@link SystemSettingsService#put} silently drops writes for keys not in * {@link SystemSettings#keysWithDefaults()}. @@ -183,8 +183,8 @@ private void updateLastSuccessfulSystemSettings( private void updatePerTypeSuccessfulSystemSettings( Function keyFunction, Date startTime, - Set processedTypes) { - processedTypes.stream() + Set publishedTypes) { + publishedTypes.stream() .filter(AnalyticsTableType::isLatestPartition) .forEach(type -> settingsService.put(keyFunction.apply(type), startTime)); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index 5092b8408677..6c9a91cb43de 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -83,7 +83,7 @@ public AnalyticsTableType getAnalyticsTableType() { } @Override - public void create(AnalyticsTableUpdateParams params, JobProgress progress) { + public boolean create(AnalyticsTableUpdateParams params, JobProgress progress) { final int parallelJobs = getParallelJobs(); int tableUpdates = 0; @@ -102,8 +102,12 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { boolean validState = tableManager.validState(); progress.completedStage("Validated analytics tables with outcome: {}", validState); - if (!validState || progress.isCancelled()) { - return; + if (progress.isCancelled()) { + return false; + } + + if (!validState) { + return true; } List tables = tableManager.getAnalyticsTables(params); @@ -112,7 +116,7 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { clock.logTime("Table update aborted, nothing to update: '{}'", tableType.getTableName()); progress.startingStage("Table update of type: '{}'", tableType); progress.completedStage("Table updated aborted, no table or partitions to be updated"); - return; + return true; } clock.logTime( @@ -186,7 +190,7 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { + "table rebuild before running a continuous (lastYears=0) update", tableType)); clock.logTime("Continuous analytics update aborted, not ready: '{}'", tableType); - return; + return false; } progress.completedStage("Validated continuous update readiness: '{}'", tableType); @@ -207,7 +211,7 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { + "deleted data, see preceding error. A full analytics table rebuild may be " + "required before continuous updates can run for this table.", tableType); - return; + return false; } } @@ -220,6 +224,7 @@ public void create(AnalyticsTableUpdateParams params, JobProgress progress) { } clock.logTime("Table update done: '{}'", tableType.getTableName()); + return true; } @Override diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 3cbab155b485..4b8742ff1341 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -97,7 +97,6 @@ import org.hisp.dhis.setting.SystemSettings; import org.hisp.dhis.setting.SystemSettingsProvider; import org.hisp.dhis.trackedentity.TrackedEntityAttribute; -import org.hisp.dhis.util.DateUtils; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -234,8 +233,7 @@ private List getRegularAnalyticsTables( private List getLatestAnalyticsTables(AnalyticsTableUpdateParams params) { SystemSettings settings = settingsProvider.getCurrentSettings(); Date lastFullTableUpdate = settings.getLastSuccessfulAnalyticsTablesUpdate(); - Date lastLatestPartitionUpdate = settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(); - Date lastAnyTableUpdate = DateUtils.getLatest(lastLatestPartitionUpdate, lastFullTableUpdate); + Date lastAnyTableUpdate = getLastAnyTableUpdate(settings); Assert.isTrue( lastFullTableUpdate.getTime() > 0L, diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java index a33916ec4c50..08ca47cf3cc7 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java @@ -296,7 +296,7 @@ void testRemoveUpdatedDataRunWhenContinuousAnalyticsSupported() { when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); when(tableManager.isReadyForContinuousUpdate(List.of(table))).thenReturn(true); - tableService.create(params, JobProgress.noop()); + assertTrue(tableService.create(params, JobProgress.noop())); verify(tableManager).removeUpdatedData(List.of(table)); } @@ -324,7 +324,7 @@ void testRemoveUpdatedDataNotRunWhenNotReadyForContinuousUpdate() { // e.g. a main table predating unique-key analytics tables on Doris: the delete step must // never even be attempted against it, not just have its failure tolerated. - tableService.create(params, progress); + assertFalse(tableService.create(params, progress)); verify(tableManager, never()).removeUpdatedData(anyList()); verify(tableManager, never()).swapTable(eq(params), any(AnalyticsTable.class)); @@ -358,11 +358,55 @@ void testSwapTableNotRunWhenRemoveUpdatedDataFails() { // cancel the job (other table types must still be able to proceed). The table update must // instead explicitly stop itself before swapping staged data into a main table it was unable // to purge stale/deleted rows from. - tableService.create(params, JobProgress.noop()); + assertFalse(tableService.create(params, JobProgress.noop())); verify(tableManager, never()).swapTable(eq(params), any(AnalyticsTable.class)); } + @Test + void testCreateReturnsTrueWhenThereIsNoSourceData() { + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().build().withLatestPartition(); + + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(tableManager.getAnalyticsTableType()).thenReturn(AnalyticsTableType.DATA_VALUE); + when(tableManager.validState()).thenReturn(false); + + // Nothing to publish, so the update is complete and must be recorded as such. + assertTrue(tableService.create(params, JobProgress.noop())); + + verify(tableManager, never()).getAnalyticsTables(any()); + } + + @Test + void testCreateReturnsTrueWhenThereIsNothingToUpdate() { + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().build().withLatestPartition(); + + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(tableManager.getAnalyticsTableType()).thenReturn(AnalyticsTableType.DATA_VALUE); + when(tableManager.validState()).thenReturn(true); + when(tableManager.getAnalyticsTables(params)).thenReturn(List.of()); + + assertTrue(tableService.create(params, JobProgress.noop())); + } + + @Test + void testCreateReturnsFalseWhenCancelled() { + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().build().withLatestPartition(); + + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(tableManager.getAnalyticsTableType()).thenReturn(AnalyticsTableType.DATA_VALUE); + when(tableManager.validState()).thenReturn(true); + JobProgress progress = spy(JobProgress.noop()); + when(progress.isCancelled()).thenReturn(true); + + assertFalse(tableService.create(params, progress)); + + verify(tableManager, never()).getAnalyticsTables(any()); + } + private AnalyticsTable latestPartitionTableFixture() { List columns = List.of( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGeneratorTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGeneratorTest.java index 4904c3620be9..dc76e1fe6b28 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGeneratorTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableGeneratorTest.java @@ -34,6 +34,7 @@ import static org.hisp.dhis.analytics.AnalyticsTableType.EVENT; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -81,6 +82,8 @@ class DefaultAnalyticsTableGeneratorTest { private void setUp() { when(dataValueTableService.getAnalyticsTableType()).thenReturn(DATA_VALUE); when(eventTableService.getAnalyticsTableType()).thenReturn(EVENT); + lenient().when(dataValueTableService.create(any(), any())).thenReturn(true); + lenient().when(eventTableService.create(any(), any())).thenReturn(true); when(settingsService.getCurrentSettings()).thenReturn(SystemSettings.of(Map.of())); generator = new DefaultAnalyticsTableGenerator( @@ -135,9 +138,31 @@ void skippedTypeGetsNoPerTypeFullUpdateKey_processedTypeDoes() { .put(SystemSettings.keyLastSuccessfulAnalyticsTablesUpdate(DATA_VALUE), startTime); } + @Test + void abortedTypeGetsNoPerTypeLatestPartitionKey_publishedTypeDoes() { + setUp(); + // e.g. the continuous update readiness check or removeUpdatedData() failed for EVENT + when(eventTableService.create(any(), any())).thenReturn(false); + Date startTime = new Date(); + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .skipResourceTables(true) + .startTime(startTime) + .build() + .withLatestPartition(); + + generator.generateAnalyticsTables(params, JobProgress.noop()); + + verify(settingsService, never()) + .put(eq(SystemSettings.keyLastSuccessfulLatestAnalyticsPartitionUpdate(EVENT)), any()); + verify(settingsService) + .put(SystemSettings.keyLastSuccessfulLatestAnalyticsPartitionUpdate(DATA_VALUE), startTime); + } + @Test void typeWithoutLatestPartitionSupportGetsNoPerTypeKeyEvenWhenProcessed() { when(dataValueTableService.getAnalyticsTableType()).thenReturn(ENROLLMENT); + when(dataValueTableService.create(any(), any())).thenReturn(true); when(settingsService.getCurrentSettings()).thenReturn(SystemSettings.of(Map.of())); generator = new DefaultAnalyticsTableGenerator( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index b268d837bd69..4cb126141c89 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -35,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; import java.util.ArrayList; @@ -172,7 +173,9 @@ void testGetLatestAnalyticsTableUsesLastAnyTableUpdate() { queryResp.add(Map.of("dataelementid", 1)); when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); - when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + when(settings.getLastSuccessfulAnalyticsTablesUpdate(AnalyticsTableType.DATA_VALUE)) + .thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(AnalyticsTableType.DATA_VALUE)) .thenReturn(lastLatestPartitionUpdate); when(analyticsTableSettings.getTableLogged()).thenReturn(Logged.UNLOGGED); when(jdbcTemplate.queryForList(org.mockito.Mockito.anyString())).thenReturn(queryResp); @@ -191,6 +194,42 @@ void testGetLatestAnalyticsTableUsesLastAnyTableUpdate() { assertEquals(startTime, partition.getEndDate()); } + @Test + @DisplayName( + "Doris latest-partition window starts at this table type's own last update, not the shared" + + " one, which also advances when this type was skipped or its update was aborted") + void testGetLatestAnalyticsTableUsesPerTypeLastUpdateNotShared() { + Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); + Date lastDataValueUpdate = new DateTime(2019, 3, 1, 5, 0).toDate(); + Date lastSharedUpdate = new DateTime(2019, 3, 1, 9, 0).toDate(); + Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); + + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().startTime(startTime).build().withLatestPartition(); + + List> queryResp = new ArrayList<>(); + queryResp.add(Map.of("dataelementid", 1)); + + when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); + lenient() + .when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + .thenReturn(lastSharedUpdate); + when(settings.getLastSuccessfulAnalyticsTablesUpdate(AnalyticsTableType.DATA_VALUE)) + .thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(AnalyticsTableType.DATA_VALUE)) + .thenReturn(lastDataValueUpdate); + when(analyticsTableSettings.getTableLogged()).thenReturn(Logged.UNLOGGED); + when(jdbcTemplate.queryForList(org.mockito.Mockito.anyString())).thenReturn(queryResp); + when(configurationService.getConfiguration()).thenReturn(configuration); + when(configuration.getDataOutputPeriodTypes()) + .thenReturn(PERIOD_TYPES.stream().collect(toUnmodifiableSet())); + + List tables = subject.getAnalyticsTables(params); + + assertEquals(1, tables.size()); + assertEquals(lastDataValueUpdate, tables.get(0).getLatestTablePartition().getStartDate()); + } + @Test void testRemoveUpdatedDataMaterializesKeysNatively() { Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); @@ -204,7 +243,9 @@ void testRemoveUpdatedDataMaterializesKeysNatively() { queryResp.add(Map.of("dataelementid", 1)); when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); - when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + when(settings.getLastSuccessfulAnalyticsTablesUpdate(AnalyticsTableType.DATA_VALUE)) + .thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(AnalyticsTableType.DATA_VALUE)) .thenReturn(lastLatestPartitionUpdate); when(jdbcTemplate.queryForList(org.mockito.Mockito.anyString())).thenReturn(queryResp); when(configurationService.getConfiguration()).thenReturn(configuration); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java index e7d9ecf14eea..1e99b021f96f 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Map; import org.hisp.dhis.analytics.AnalyticsTableHookService; +import org.hisp.dhis.analytics.AnalyticsTableType; import org.hisp.dhis.analytics.AnalyticsTableUpdateParams; import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.model.AnalyticsTable; @@ -129,7 +130,9 @@ void testGetLatestAnalyticsTableHasUniqueKeyOnId() { queryResp.add(Map.of("datasetid", 1)); when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); - when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + when(settings.getLastSuccessfulAnalyticsTablesUpdate(AnalyticsTableType.COMPLETENESS)) + .thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(AnalyticsTableType.COMPLETENESS)) .thenReturn(lastLatestPartitionUpdate); when(jdbcTemplate.queryForList(anyString())).thenReturn(queryResp); @@ -168,7 +171,9 @@ void testRemoveUpdatedDataUsesUsingJoinNotSubquery() { queryResp.add(Map.of("datasetid", 1)); when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); - when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + when(settings.getLastSuccessfulAnalyticsTablesUpdate(AnalyticsTableType.COMPLETENESS)) + .thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(AnalyticsTableType.COMPLETENESS)) .thenReturn(lastLatestPartitionUpdate); when(jdbcTemplate.queryForList(anyString())).thenReturn(queryResp); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java index d45b9f0da2f4..8bc95dc8e022 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java @@ -349,7 +349,9 @@ void testRemoveUpdatedDataMaterializesKeysNativelyForRegistration() { queryResp.add(Map.of("eventid", 1)); when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); - when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + when(settings.getLastSuccessfulAnalyticsTablesUpdate(AnalyticsTableType.EVENT)) + .thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(AnalyticsTableType.EVENT)) .thenReturn(lastLatestPartitionUpdate); when(jdbcTemplate.queryForList(Mockito.anyString())).thenReturn(queryResp); when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program)); @@ -424,7 +426,9 @@ void testRemoveUpdatedDataMaterializesKeysNativelyForSingleEvent() { queryResp.add(Map.of("eventid", 1)); when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); - when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + when(settings.getLastSuccessfulAnalyticsTablesUpdate(AnalyticsTableType.EVENT)) + .thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(AnalyticsTableType.EVENT)) .thenReturn(lastLatestPartitionUpdate); when(jdbcTemplate.queryForList(Mockito.anyString())).thenReturn(queryResp); when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(program)); From 976c9b9cae43bcfc8a1a49e01fe76ed9ed172a18 Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Thu, 1 Oct 2026 12:31:19 +0200 Subject: [PATCH 29/31] fix: don't abort continuous updates for programs without an event table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A program created after the last full rebuild has no event analytics table yet. removeUpdatedData() ran its delete against that missing table, which failed, and the guard against swapping after a failed delete then skipped the swap for every program of the table type. The deletes already committed for the other programs were not followed by their swap, so their updated events disappeared from analytics, and every following continuous update failed the same way until a full rebuild. Affects Postgres and Doris. - Only remove updated data from tables whose main table exists; a new table has no stale rows and becomes the main table when swapped. - Give a program without a main table all of its data rather than the continuous update window, so its new main table is complete. Reported by Luciano Fiandesio. 🤖 AI Assisted --- .../dhis/analytics/AnalyticsTableManager.java | 10 +++ .../table/AbstractJdbcTableManager.java | 7 +- .../table/DefaultAnalyticsTableService.java | 10 ++- .../table/JdbcEventAnalyticsTableManager.java | 16 +++- .../table/AnalyticsTableServiceTest.java | 76 +++++++++++++++++++ ...bcEventAnalyticsTableManagerDorisTest.java | 40 ++++++++++ 6 files changed, 153 insertions(+), 6 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java index f18d8b510400..25501c12632d 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/AnalyticsTableManager.java @@ -107,6 +107,16 @@ default boolean isReadyForContinuousUpdate(List tables) { return true; } + /** + * Checks whether the main table of the given analytics table exists. + * + * @param table the {@link AnalyticsTable}. + * @return true if the main table exists. + */ + default boolean mainTableExists(AnalyticsTable table) { + return true; + } + /** * Removes updated and deleted data from the given tables for "latest" partition update. * diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java index c3311e8a3de0..1eb479b6d4c4 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/AbstractJdbcTableManager.java @@ -176,6 +176,11 @@ public void preCreateTables(AnalyticsTableUpdateParams params) {} @Override public void removeUpdatedData(List tables) {} + @Override + public boolean mainTableExists(AnalyticsTable table) { + return tableExists(table.getMainName()); + } + @Override public void createTable(AnalyticsTable table) { createAnalyticsTable(table); @@ -450,7 +455,7 @@ private void swapParentTable(Table partition, String stagingMasterName, String m * @param name the table name. * @return true if a table with the given name exists. */ - private boolean tableExists(String name) { + protected boolean tableExists(String name) { return !jdbcTemplate.queryForList(sqlBuilder.tableExists(name)).isEmpty(); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index 6c9a91cb43de..268288683720 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -195,9 +195,17 @@ public boolean create(AnalyticsTableUpdateParams params, JobProgress progress) { progress.completedStage("Validated continuous update readiness: '{}'", tableType); + // A table without a main table yet (e.g. the event table of a program created after the + // last full rebuild) has no stale rows to remove, and the delete would fail on the missing + // table. Its staging table becomes the main table when swapped. + List existingTables = + tables.stream().filter(tableManager::mainTableExists).toList(); + progress.startingStage( format("Removing updated and deleted data: '{}'", tableType), SKIP_STAGE); - boolean removedUpdatedData = progress.runStage(() -> tableManager.removeUpdatedData(tables)); + boolean removedUpdatedData = + existingTables.isEmpty() + || progress.runStage(() -> tableManager.removeUpdatedData(existingTables)); clock.logTime("Removed updated and deleted data"); if (!removedUpdatedData) { diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 4b8742ff1341..4e2712f848ed 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -261,7 +261,15 @@ private List getLatestAnalyticsTables(AnalyticsTableUpdateParams : idObjectManager.getAllNoAcl(Program.class); for (Program program : programs) { - boolean hasUpdatedData = hasUpdatedLatestData(lastAnyTableUpdate, endDate, program); + // A program without a main table yet (e.g. created after the last full rebuild) gets all of + // its data, since its staging table becomes the main table when swapped. A window starting + // at the last update would leave out its events last updated before then. + boolean hasMainTable = + tableExists(AnalyticsTable.getTableName(getAnalyticsTableType(), program)); + Date programStartDate = hasMainTable ? startDate : new Date(0L); + Date updatedSince = hasMainTable ? lastAnyTableUpdate : new Date(0L); + + boolean hasUpdatedData = hasUpdatedLatestData(updatedSince, endDate, program); if (hasUpdatedData) { AnalyticsTable table = @@ -272,19 +280,19 @@ private List getLatestAnalyticsTables(AnalyticsTableUpdateParams logged, program); table.addTablePartition( - List.of(), AnalyticsTablePartition.LATEST_PARTITION, startDate, endDate); + List.of(), AnalyticsTablePartition.LATEST_PARTITION, programStartDate, endDate); tables.add(table); log.info( "Added latest event analytics partition for program: '{}', start: '{}' and end: '{}'", program.getUid(), - toLongDate(startDate), + toLongDate(programStartDate), toLongDate(endDate)); } else { log.info( "No updated latest event data found for program: '{}', start: '{}' and end: '{}", program.getUid(), - toLongDate(lastAnyTableUpdate), + toLongDate(updatedSince), toLongDate(endDate)); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java index 08ca47cf3cc7..3bc9ebc43bf6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java @@ -31,6 +31,7 @@ import static org.hisp.dhis.db.model.DataType.DOUBLE; import static org.hisp.dhis.db.model.DataType.TEXT; +import static org.hisp.dhis.test.TestBase.createProgram; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; @@ -295,6 +296,7 @@ void testRemoveUpdatedDataRunWhenContinuousAnalyticsSupported() { when(sqlBuilder.supportsAnalyze()).thenReturn(false); when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); when(tableManager.isReadyForContinuousUpdate(List.of(table))).thenReturn(true); + when(tableManager.mainTableExists(table)).thenReturn(true); assertTrue(tableService.create(params, JobProgress.noop())); @@ -350,6 +352,7 @@ void testSwapTableNotRunWhenRemoveUpdatedDataFails() { when(sqlBuilder.supportsAnalyze()).thenReturn(false); when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); when(tableManager.isReadyForContinuousUpdate(List.of(table))).thenReturn(true); + when(tableManager.mainTableExists(table)).thenReturn(true); doThrow(new IllegalStateException("Delete failed")) .when(tableManager) .removeUpdatedData(anyList()); @@ -407,6 +410,79 @@ void testCreateReturnsFalseWhenCancelled() { verify(tableManager, never()).getAnalyticsTables(any()); } + @Test + void testRemoveUpdatedDataSkipsTableWithoutMainTable() { + // e.g. the event table of a program created after the last full rebuild + AnalyticsTable existing = latestPartitionEventTableFixture('A'); + AnalyticsTable created = latestPartitionEventTableFixture('C'); + List tables = List.of(existing, created); + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build() + .withLatestPartition(); + + stubContinuousUpdate(params, tables); + when(tableManager.mainTableExists(existing)).thenReturn(true); + when(tableManager.mainTableExists(created)).thenReturn(false); + + assertTrue(tableService.create(params, JobProgress.noop())); + + verify(tableManager).removeUpdatedData(List.of(existing)); + verify(tableManager).swapTable(params, existing); + verify(tableManager).swapTable(params, created); + } + + @Test + void testRemoveUpdatedDataNotRunWhenNoMainTableExists() { + AnalyticsTable created = latestPartitionEventTableFixture('C'); + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build() + .withLatestPartition(); + + stubContinuousUpdate(params, List.of(created)); + when(tableManager.mainTableExists(created)).thenReturn(false); + + assertTrue(tableService.create(params, JobProgress.noop())); + + verify(tableManager, never()).removeUpdatedData(anyList()); + verify(tableManager).swapTable(params, created); + } + + private void stubContinuousUpdate( + AnalyticsTableUpdateParams params, List tables) { + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(tableManager.getAnalyticsTableType()).thenReturn(AnalyticsTableType.EVENT); + when(tableManager.validState()).thenReturn(true); + when(tableManager.getAnalyticsTables(params)).thenReturn(tables); + when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(false); + when(sqlBuilder.requiresIndexesForAnalytics()).thenReturn(false); + when(sqlBuilder.supportsAnalyze()).thenReturn(false); + when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); + when(tableManager.isReadyForContinuousUpdate(tables)).thenReturn(true); + } + + private AnalyticsTable latestPartitionEventTableFixture(char programCharacter) { + List columns = + List.of( + AnalyticsTableColumn.builder() + .name("event") + .dataType(TEXT) + .selectExpression("event") + .build()); + AnalyticsTable table = + new AnalyticsTable( + AnalyticsTableType.EVENT, columns, Logged.UNLOGGED, createProgram(programCharacter)); + table.addTablePartition( + List.of(), + AnalyticsTablePartition.LATEST_PARTITION, + new DateTime(2020, 1, 1, 0, 0).toDate(), + new DateTime(2020, 3, 1, 10, 0).toDate()); + return table; + } + private AnalyticsTable latestPartitionTableFixture() { List columns = List.of( diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java index 8bc95dc8e022..a42cf5ffa60e 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java @@ -330,6 +330,46 @@ void testRegularAnalyticsTableHasUniqueKeyOnEvent() { assertTrue(sqlBuilder.createTable(table).contains("unique key (`event`,`year`)")); } + @Test + @DisplayName( + "A program without a main table yet gets all its data in a continuous update, since its" + + " staging table becomes the main table; a program with one gets the continuous window") + void testLatestTableOfProgramWithoutMainTableStartsAtEpoch() { + Program existing = createProgram('A'); + Program created = createProgram('C'); + + Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); + Date lastLatestPartitionUpdate = new DateTime(2019, 3, 1, 9, 0).toDate(); + Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); + + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().startTime(startTime).build().withLatestPartition(); + + List> queryResp = new ArrayList<>(); + queryResp.add(Map.of("eventid", 1)); + + when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulAnalyticsTablesUpdate(AnalyticsTableType.EVENT)) + .thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate(AnalyticsTableType.EVENT)) + .thenReturn(lastLatestPartitionUpdate); + when(jdbcTemplate.queryForList(Mockito.anyString())).thenReturn(queryResp); + when(jdbcTemplate.queryForList( + sqlBuilder.tableExists(AnalyticsTable.getTableName(AnalyticsTableType.EVENT, created)))) + .thenReturn(List.of()); + when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(existing, created)); + when(configurationService.getConfiguration()).thenReturn(configuration); + when(configuration.getDataOutputPeriodTypes()) + .thenReturn(PERIOD_TYPES.stream().collect(toUnmodifiableSet())); + + List tables = subject.getAnalyticsTables(params); + assertThat(tables, hasSize(2)); + + assertEquals(lastLatestPartitionUpdate, tables.get(0).getLatestTablePartition().getStartDate()); + assertEquals(new Date(0L), tables.get(1).getLatestTablePartition().getStartDate()); + assertEquals(startTime, tables.get(1).getLatestTablePartition().getEndDate()); + } + @Test @DisplayName( "removeUpdatedData materializes event keys natively for a registration program on Doris," From ef74df42a2bbda63f75165d40ebbbae4bd815d1c Mon Sep 17 00:00:00 2001 From: Luciano Fiandesio Date: Wed, 7 Oct 2026 11:25:51 +0200 Subject: [PATCH 30/31] refactor: extract continuous update preparation from analytics table create --- .../table/DefaultAnalyticsTableService.java | 119 +++++++++++------- 1 file changed, 71 insertions(+), 48 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index 268288683720..72889af0bad3 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -173,54 +173,10 @@ public boolean create(AnalyticsTableUpdateParams params, JobProgress progress) { clock.logTime("Analyzed tables"); } - if (params.isLatestUpdate() && sqlBuilder.supportsContinuousAnalytics()) { - progress.startingStage( - format("Validating continuous update readiness: '{}'", tableType), SKIP_STAGE); - boolean readyForContinuousUpdate = tableManager.isReadyForContinuousUpdate(tables); - - if (!readyForContinuousUpdate) { - // A main table predating unique-key analytics tables (or otherwise not in a state that - // supports a continuous update) must not have the delete step attempted against it, see - // isReadyForContinuousUpdate() for detail. The stage is reported as failed so the reason - // shows in the job log, while the remaining table types still get updated. - progress.failedStage( - format( - "Continuous update skipped for '{}': the analytics table was created before " - + "continuous updates were supported on this database. Run a full analytics " - + "table rebuild before running a continuous (lastYears=0) update", - tableType)); - clock.logTime("Continuous analytics update aborted, not ready: '{}'", tableType); - return false; - } - - progress.completedStage("Validated continuous update readiness: '{}'", tableType); - - // A table without a main table yet (e.g. the event table of a program created after the - // last full rebuild) has no stale rows to remove, and the delete would fail on the missing - // table. Its staging table becomes the main table when swapped. - List existingTables = - tables.stream().filter(tableManager::mainTableExists).toList(); - - progress.startingStage( - format("Removing updated and deleted data: '{}'", tableType), SKIP_STAGE); - boolean removedUpdatedData = - existingTables.isEmpty() - || progress.runStage(() -> tableManager.removeUpdatedData(existingTables)); - clock.logTime("Removed updated and deleted data"); - - if (!removedUpdatedData) { - // Swapping in the staged data without having removed the stale/deleted rows it is meant - // to replace would corrupt the main table (duplicate or orphaned rows), so the continuous - // update for this table type is aborted here rather than proceeding to swap. A likely - // cause is a main table created before unique-key analytics tables were introduced; a - // full analytics table rebuild recreates it with the required key type. - log.error( - "Aborting continuous analytics update for '{}': failed to remove updated and " - + "deleted data, see preceding error. A full analytics table rebuild may be " - + "required before continuous updates can run for this table.", - tableType); - return false; - } + if (params.isLatestUpdate() + && sqlBuilder.supportsContinuousAnalytics() + && !prepareContinuousUpdate(tables, tableType, progress, clock)) { + return false; } swapTables(params, tables, progress); @@ -369,6 +325,73 @@ private void analyzeTables(List tables, JobProgress progress) { getParallelJobs(), tables, Table::getName, tableManager::analyzeTable); } + /** + * Prepares the main analytics tables for a continuous update by removing the rows which the + * staged data replaces. + * + * @param tables the list of {@link AnalyticsTable}. + * @param tableType the {@link AnalyticsTableType}. + * @param progress the {@link JobProgress}. + * @param clock the {@link Clock}. + * @return true if the staging tables can be swapped in, false if the continuous update must be + * aborted. + */ + private boolean prepareContinuousUpdate( + List tables, + AnalyticsTableType tableType, + JobProgress progress, + Clock clock) { + progress.startingStage( + format("Validating continuous update readiness: '{}'", tableType), SKIP_STAGE); + boolean readyForContinuousUpdate = tableManager.isReadyForContinuousUpdate(tables); + + if (!readyForContinuousUpdate) { + // A main table predating unique-key analytics tables (or otherwise not in a state that + // supports a continuous update) must not have the delete step attempted against it, see + // isReadyForContinuousUpdate() for detail. The stage is reported as failed so the reason + // shows in the job log, while the remaining table types still get updated. + progress.failedStage( + format( + "Continuous update skipped for '{}': the analytics table was created before " + + "continuous updates were supported on this database. Run a full analytics " + + "table rebuild before running a continuous (lastYears=0) update", + tableType)); + clock.logTime("Continuous analytics update aborted, not ready: '{}'", tableType); + return false; + } + + progress.completedStage("Validated continuous update readiness: '{}'", tableType); + + // A table without a main table yet (e.g. the event table of a program created after the + // last full rebuild) has no stale rows to remove, and the delete would fail on the missing + // table. Its staging table becomes the main table when swapped. + List existingTables = + tables.stream().filter(tableManager::mainTableExists).toList(); + + progress.startingStage( + format("Removing updated and deleted data: '{}'", tableType), SKIP_STAGE); + boolean removedUpdatedData = + existingTables.isEmpty() + || progress.runStage(() -> tableManager.removeUpdatedData(existingTables)); + clock.logTime("Removed updated and deleted data"); + + if (!removedUpdatedData) { + // Swapping in the staged data without having removed the stale/deleted rows it is meant + // to replace would corrupt the main table (duplicate or orphaned rows), so the continuous + // update for this table type is aborted here rather than proceeding to swap. A likely + // cause is a main table created before unique-key analytics tables were introduced; a + // full analytics table rebuild recreates it with the required key type. + log.error( + "Aborting continuous analytics update for '{}': failed to remove updated and " + + "deleted data, see preceding error. A full analytics table rebuild may be " + + "required before continuous updates can run for this table.", + tableType); + return false; + } + + return true; + } + /** * Swaps the given analytics tables. * From f287797f58d8710fda7ac39e63b7f79a828f1f3c Mon Sep 17 00:00:00 2001 From: Jason Pickering Date: Wed, 7 Oct 2026 12:05:04 +0200 Subject: [PATCH 31/31] fix: Check Doris continuous update readiness before staging, publish per table [DHIS2-22183] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback: - Check continuous update readiness right after the tables are known, before any staging work. A table type that is not ready no longer populates its staging tables in full on every scheduled run only to throw them away. - On unique-key databases (Doris), remove updated data per table. A table whose delete fails keeps its staged data unpublished, the other tables are published, and the update is reported as incomplete so the window is not advanced and the next run covers it again (merging a window again is idempotent). Before, one failed table aborted the swap for all tables after rows had already been deleted from the others. - Other databases (Postgres) swap the tables in after the delete whatever its outcome, unchanged from master. - Limit the full-history window for programs without a main table to unique-key databases. On Postgres the latest partition is replaced with the same window on every run, so the epoch window made a new program's older events appear and then vanish; it also no longer queries tableExists per program there. - Tests: drop class-wide lenient strictness for lenient() on the shared stubs that need it, and use assertFalse instead of assertTrue(!...). 🤖 AI Assisted --- .../table/DefaultAnalyticsTableService.java | 124 +++++++++++++----- .../table/JdbcEventAnalyticsTableManager.java | 19 +-- .../table/AnalyticsTableServiceTest.java | 82 ++++++++++-- ...bcAnalyticsTableManagerClickHouseTest.java | 13 +- .../JdbcAnalyticsTableManagerDorisTest.java | 8 +- ...JdbcCompletenessTableManagerDorisTest.java | 7 +- ...bcEventAnalyticsTableManagerDorisTest.java | 13 +- .../JdbcEventAnalyticsTableManagerTest.java | 37 +++++- 8 files changed, 225 insertions(+), 78 deletions(-) diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java index 72889af0bad3..e64b93fe56f6 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java @@ -119,6 +119,14 @@ public boolean create(AnalyticsTableUpdateParams params, JobProgress progress) { return true; } + boolean continuousUpdate = params.isLatestUpdate() && sqlBuilder.supportsContinuousAnalytics(); + + // Checked before any staging work: a table type that is not ready would otherwise populate + // its staging tables in full on every scheduled run, only to throw them away. + if (continuousUpdate && !checkContinuousUpdateReadiness(tables, tableType, progress, clock)) { + return false; + } + clock.logTime( "Table update start: {}, earliest: {}, parameters: {}", tableType.getTableName(), @@ -173,13 +181,15 @@ public boolean create(AnalyticsTableUpdateParams params, JobProgress progress) { clock.logTime("Analyzed tables"); } - if (params.isLatestUpdate() - && sqlBuilder.supportsContinuousAnalytics() - && !prepareContinuousUpdate(tables, tableType, progress, clock)) { - return false; + List publishedTables = tables; + + if (continuousUpdate) { + publishedTables = removeUpdatedData(tables, tableType, progress, clock); } - swapTables(params, tables, progress); + if (!publishedTables.isEmpty()) { + swapTables(params, publishedTables, progress); + } if (params.isPartialUpdate() && !params.isLatestUpdate()) { progress.startingStage("Removing latest partition overlap: '{}'", tableType); @@ -188,7 +198,9 @@ public boolean create(AnalyticsTableUpdateParams params, JobProgress progress) { } clock.logTime("Table update done: '{}'", tableType.getTableName()); - return true; + // Not complete if a table's staged data could not be published, so the update window is not + // advanced and the next continuous run covers its changes again + return publishedTables.size() == tables.size(); } @Override @@ -326,26 +338,24 @@ private void analyzeTables(List tables, JobProgress progress) { } /** - * Prepares the main analytics tables for a continuous update by removing the rows which the - * staged data replaces. + * Checks that the main analytics tables can take a continuous update. The check is cheap, so it + * runs before any staging tables are created and populated. * * @param tables the list of {@link AnalyticsTable}. * @param tableType the {@link AnalyticsTableType}. * @param progress the {@link JobProgress}. * @param clock the {@link Clock}. - * @return true if the staging tables can be swapped in, false if the continuous update must be - * aborted. + * @return true if the continuous update can proceed, false if it must be aborted. */ - private boolean prepareContinuousUpdate( + private boolean checkContinuousUpdateReadiness( List tables, AnalyticsTableType tableType, JobProgress progress, Clock clock) { progress.startingStage( format("Validating continuous update readiness: '{}'", tableType), SKIP_STAGE); - boolean readyForContinuousUpdate = tableManager.isReadyForContinuousUpdate(tables); - if (!readyForContinuousUpdate) { + if (!tableManager.isReadyForContinuousUpdate(tables)) { // A main table predating unique-key analytics tables (or otherwise not in a state that // supports a continuous update) must not have the delete step attempted against it, see // isReadyForContinuousUpdate() for detail. The stage is reported as failed so the reason @@ -361,35 +371,79 @@ private boolean prepareContinuousUpdate( } progress.completedStage("Validated continuous update readiness: '{}'", tableType); + return true; + } - // A table without a main table yet (e.g. the event table of a program created after the - // last full rebuild) has no stale rows to remove, and the delete would fail on the missing - // table. Its staging table becomes the main table when swapped. - List existingTables = - tables.stream().filter(tableManager::mainTableExists).toList(); - + /** + * Removes the rows which the staged data replaces from the main analytics tables. + * + *

On databases which require unique-key analytics tables (Doris), each table is handled on its + * own: a table whose delete fails keeps its staged data unpublished, while the other tables are + * published. Merging a window again is idempotent there, so the next run, which covers the same + * window again, publishes the failed table's changes once its delete succeeds. + * + *

On other databases the tables are swapped in after the delete whatever its outcome, as + * before continuous updates were supported on unique-key databases. + * + * @param tables the list of {@link AnalyticsTable}. + * @param tableType the {@link AnalyticsTableType}. + * @param progress the {@link JobProgress}. + * @param clock the {@link Clock}. + * @return the tables whose staged data can be published. + */ + private List removeUpdatedData( + List tables, + AnalyticsTableType tableType, + JobProgress progress, + Clock clock) { progress.startingStage( format("Removing updated and deleted data: '{}'", tableType), SKIP_STAGE); - boolean removedUpdatedData = - existingTables.isEmpty() - || progress.runStage(() -> tableManager.removeUpdatedData(existingTables)); + + if (!sqlBuilder.requiresUniqueKeyAnalyticsTables()) { + progress.runStage(() -> tableManager.removeUpdatedData(tables)); + clock.logTime("Removed updated and deleted data"); + return tables; + } + + List publishedTables = new ArrayList<>(); + List failedTables = new ArrayList<>(); + + for (AnalyticsTable table : tables) { + // A table without a main table yet (e.g. the event table of a program created after the + // last full rebuild) has no stale rows to remove. Its staging table becomes the main table. + if (!tableManager.mainTableExists(table)) { + publishedTables.add(table); + continue; + } + + try { + tableManager.removeUpdatedData(List.of(table)); + publishedTables.add(table); + } catch (RuntimeException ex) { + // Swapping in the staged data without having removed the rows it replaces would leave + // duplicate or orphaned rows, so this table's staged data is not published + log.error( + "Failed to remove updated and deleted data from '{}', its staged data is not published", + table.getMainName(), + ex); + failedTables.add(table.getMainName()); + } + } + clock.logTime("Removed updated and deleted data"); - if (!removedUpdatedData) { - // Swapping in the staged data without having removed the stale/deleted rows it is meant - // to replace would corrupt the main table (duplicate or orphaned rows), so the continuous - // update for this table type is aborted here rather than proceeding to swap. A likely - // cause is a main table created before unique-key analytics tables were introduced; a - // full analytics table rebuild recreates it with the required key type. - log.error( - "Aborting continuous analytics update for '{}': failed to remove updated and " - + "deleted data, see preceding error. A full analytics table rebuild may be " - + "required before continuous updates can run for this table.", - tableType); - return false; + if (failedTables.isEmpty()) { + progress.completedStage("Removed updated and deleted data: '{}'", tableType); + } else { + progress.failedStage( + format( + "Failed to remove updated and deleted data from {}. Their staged data was not " + + "published and the next continuous update covers it again. If this keeps " + + "failing, a full analytics table rebuild may be required", + failedTables)); } - return true; + return publishedTables; } /** diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java index 4e2712f848ed..b2a885a57634 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManager.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -261,13 +261,16 @@ private List getLatestAnalyticsTables(AnalyticsTableUpdateParams : idObjectManager.getAllNoAcl(Program.class); for (Program program : programs) { - // A program without a main table yet (e.g. created after the last full rebuild) gets all of - // its data, since its staging table becomes the main table when swapped. A window starting - // at the last update would leave out its events last updated before then. - boolean hasMainTable = - tableExists(AnalyticsTable.getTableName(getAnalyticsTableType(), program)); - Date programStartDate = hasMainTable ? startDate : new Date(0L); - Date updatedSince = hasMainTable ? lastAnyTableUpdate : new Date(0L); + // On databases which merge the latest partition into the main table (Doris), a program + // without a main table yet (e.g. created after the last full rebuild) gets all of its data, + // since its staging table becomes the main table and later runs only merge into it. A window + // starting at the last update would leave out its events last updated before then. Other + // databases replace the latest partition with the same window on every run. + boolean fullHistory = + sqlBuilder.requiresUniqueKeyAnalyticsTables() + && !tableExists(AnalyticsTable.getTableName(getAnalyticsTableType(), program)); + Date programStartDate = fullHistory ? new Date(0L) : startDate; + Date updatedSince = fullHistory ? new Date(0L) : lastAnyTableUpdate; boolean hasUpdatedData = hasUpdatedLatestData(updatedSince, endDate, program); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java index 3bc9ebc43bf6..34b30ef22969 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/AnalyticsTableServiceTest.java @@ -74,14 +74,11 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; /** * @author Lars Helge Overland */ @ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) class AnalyticsTableServiceTest { @Mock private AnalyticsTableManager tableManager; @@ -296,7 +293,6 @@ void testRemoveUpdatedDataRunWhenContinuousAnalyticsSupported() { when(sqlBuilder.supportsAnalyze()).thenReturn(false); when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); when(tableManager.isReadyForContinuousUpdate(List.of(table))).thenReturn(true); - when(tableManager.mainTableExists(table)).thenReturn(true); assertTrue(tableService.create(params, JobProgress.noop())); @@ -316,9 +312,6 @@ void testRemoveUpdatedDataNotRunWhenNotReadyForContinuousUpdate() { when(tableManager.getAnalyticsTableType()).thenReturn(AnalyticsTableType.DATA_VALUE); when(tableManager.validState()).thenReturn(true); when(tableManager.getAnalyticsTables(params)).thenReturn(List.of(table)); - when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(false); - when(sqlBuilder.requiresIndexesForAnalytics()).thenReturn(false); - when(sqlBuilder.supportsAnalyze()).thenReturn(false); when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); when(tableManager.isReadyForContinuousUpdate(List.of(table))).thenReturn(false); @@ -330,12 +323,16 @@ void testRemoveUpdatedDataNotRunWhenNotReadyForContinuousUpdate() { verify(tableManager, never()).removeUpdatedData(anyList()); verify(tableManager, never()).swapTable(eq(params), any(AnalyticsTable.class)); + // Checked before any staging work, so a table type that is not ready does not populate its + // staging tables on every scheduled run only to throw them away. + verify(tableManager, never()).preCreateTables(any()); + verify(tableManager, never()).createTable(any()); // The reason must reach the job log, not only the server log. verify(progress).failedStage(contains("Run a full analytics table rebuild")); } @Test - void testSwapTableNotRunWhenRemoveUpdatedDataFails() { + void testSwapTableNotRunWhenRemoveUpdatedDataFailsOnUniqueKeyDatabase() { AnalyticsTable table = latestPartitionTableFixture(); AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder() @@ -353,19 +350,79 @@ void testSwapTableNotRunWhenRemoveUpdatedDataFails() { when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); when(tableManager.isReadyForContinuousUpdate(List.of(table))).thenReturn(true); when(tableManager.mainTableExists(table)).thenReturn(true); + when(sqlBuilder.requiresUniqueKeyAnalyticsTables()).thenReturn(true); doThrow(new IllegalStateException("Delete failed")) .when(tableManager) .removeUpdatedData(anyList()); - // removeUpdatedData()'s stage uses FailurePolicy.SKIP_STAGE, so its failure alone does not - // cancel the job (other table types must still be able to proceed). The table update must - // instead explicitly stop itself before swapping staged data into a main table it was unable - // to purge stale/deleted rows from. + // The table's staged data must not be swapped into a main table whose stale/deleted rows + // could not be removed. The update is reported as incomplete, so the window is not advanced. assertFalse(tableService.create(params, JobProgress.noop())); verify(tableManager, never()).swapTable(eq(params), any(AnalyticsTable.class)); } + @Test + void testOtherTablesPublishedWhenOneTableDeleteFailsOnUniqueKeyDatabase() { + AnalyticsTable failing = latestPartitionEventTableFixture('A'); + AnalyticsTable working = latestPartitionEventTableFixture('B'); + List tables = List.of(failing, working); + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build() + .withLatestPartition(); + + stubContinuousUpdate(params, tables); + when(tableManager.mainTableExists(failing)).thenReturn(true); + when(tableManager.mainTableExists(working)).thenReturn(true); + doThrow(new IllegalStateException("Delete failed")) + .when(tableManager) + .removeUpdatedData(List.of(failing)); + + JobProgress progress = spy(JobProgress.noop()); + + // The table whose delete failed is not published, the others are. The update is reported as + // incomplete, so the window is not advanced and the next run covers the failed table again. + assertFalse(tableService.create(params, progress)); + + verify(tableManager).removeUpdatedData(List.of(working)); + verify(tableManager).swapTable(params, working); + verify(tableManager, never()).swapTable(params, failing); + verify(progress).failedStage(contains(failing.getMainName())); + } + + @Test + void testTablesSwappedWhenRemoveUpdatedDataFailsWithoutUniqueKeys() { + AnalyticsTable table = latestPartitionTableFixture(); + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder() + .startTime(new DateTime(2020, 3, 1, 10, 0).toDate()) + .build() + .withLatestPartition(); + + when(settingsProvider.getCurrentSettings()).thenReturn(settings); + when(tableManager.getAnalyticsTableType()).thenReturn(AnalyticsTableType.DATA_VALUE); + when(tableManager.validState()).thenReturn(true); + when(tableManager.getAnalyticsTables(params)).thenReturn(List.of(table)); + when(sqlBuilder.supportsDeclarativePartitioning()).thenReturn(false); + when(sqlBuilder.requiresIndexesForAnalytics()).thenReturn(false); + when(sqlBuilder.supportsAnalyze()).thenReturn(false); + when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); + when(sqlBuilder.requiresUniqueKeyAnalyticsTables()).thenReturn(false); + when(tableManager.isReadyForContinuousUpdate(List.of(table))).thenReturn(true); + doThrow(new IllegalStateException("Delete failed")) + .when(tableManager) + .removeUpdatedData(anyList()); + + // Postgres: unchanged from before continuous updates on unique-key databases, the tables are + // swapped in after the delete whatever its outcome. + assertTrue(tableService.create(params, JobProgress.noop())); + + verify(tableManager).removeUpdatedData(List.of(table)); + verify(tableManager).swapTable(params, table); + } + @Test void testCreateReturnsTrueWhenThereIsNoSourceData() { AnalyticsTableUpdateParams params = @@ -461,6 +518,7 @@ private void stubContinuousUpdate( when(sqlBuilder.requiresIndexesForAnalytics()).thenReturn(false); when(sqlBuilder.supportsAnalyze()).thenReturn(false); when(sqlBuilder.supportsContinuousAnalytics()).thenReturn(true); + when(sqlBuilder.requiresUniqueKeyAnalyticsTables()).thenReturn(true); when(tableManager.isReadyForContinuousUpdate(tables)).thenReturn(true); } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java index fc09f2cf5ba3..acefc85a1a12 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerClickHouseTest.java @@ -31,6 +31,7 @@ import static org.hisp.dhis.db.model.DataType.TEXT; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; import java.util.Date; @@ -66,8 +67,6 @@ import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; import org.springframework.jdbc.core.JdbcTemplate; /** @@ -77,7 +76,6 @@ * continuous and a bounded-years update replace the whole main table, as a full rebuild would. */ @ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) class JdbcAnalyticsTableManagerClickHouseTest { @Mock private IdentifiableObjectManager idObjectManager; @@ -113,8 +111,9 @@ class JdbcAnalyticsTableManagerClickHouseTest { @BeforeEach void setUp() { - when(settingsProvider.getCurrentSettings()).thenReturn(settings); - when(settings.getLastSuccessfulResourceTablesUpdate()).thenReturn(new Date(0L)); + // shared by the tests that build tables, not needed by every test + lenient().when(settingsProvider.getCurrentSettings()).thenReturn(settings); + lenient().when(settings.getLastSuccessfulResourceTablesUpdate()).thenReturn(new Date(0L)); } @Test @@ -127,7 +126,9 @@ void testSwapTableReplacesMainTableForContinuousUpdateWhenNotSupported() { AnalyticsTable table = dataValueTableFixture(); - when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) + // the main table exists; the ClickHouse path replaces it without having to check + lenient() + .when(jdbcTemplate.queryForList(sqlBuilder.tableExists(table.getMainName()))) .thenReturn(List.of(Map.of("table_name", "analytics"))); subject.swapTable(params, table); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java index 4cb126141c89..1f212e030076 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcAnalyticsTableManagerDorisTest.java @@ -73,12 +73,9 @@ import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; import org.springframework.jdbc.core.JdbcTemplate; @ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) class JdbcAnalyticsTableManagerDorisTest { @Mock private IdentifiableObjectManager idObjectManager; @@ -116,8 +113,9 @@ class JdbcAnalyticsTableManagerDorisTest { @BeforeEach void setUp() { - when(settingsProvider.getCurrentSettings()).thenReturn(settings); - when(settings.getLastSuccessfulResourceTablesUpdate()).thenReturn(new Date(0L)); + // shared by the tests that build tables, not needed by every test + lenient().when(settingsProvider.getCurrentSettings()).thenReturn(settings); + lenient().when(settings.getLastSuccessfulResourceTablesUpdate()).thenReturn(new Date(0L)); } @Test diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java index 1e99b021f96f..d3586ec1ef38 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcCompletenessTableManagerDorisTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -69,12 +69,9 @@ import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; import org.springframework.jdbc.core.JdbcTemplate; @ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) class JdbcCompletenessTableManagerDorisTest { @Mock private IdentifiableObjectManager idObjectManager; @@ -187,6 +184,6 @@ void testRemoveUpdatedDataUsesUsingJoinNotSubquery() { assertTrue(sql.getValue().contains("using")); assertTrue(sql.getValue().contains("ax.id =")); - assertTrue(!sql.getValue().contains("in (")); + assertFalse(sql.getValue().contains("in (")); } } diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java index a42cf5ffa60e..362c771f7128 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerDorisTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -50,6 +50,7 @@ import static org.hisp.dhis.test.TestBase.createProgramTrackedEntityAttribute; import static org.hisp.dhis.test.TestBase.createTrackedEntityAttribute; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -416,7 +417,7 @@ void testRemoveUpdatedDataMaterializesKeysNativelyForRegistration() { assertTrue(create.toLowerCase().contains("create table")); assertTrue(create.contains("analytics_event_delete_keys")); assertTrue(create.contains("`event`")); - assertTrue(!create.contains("`year`")); + assertFalse(create.contains("`year`")); String insert = statements.get(2); assertTrue(insert.toLowerCase().contains("insert into")); @@ -435,8 +436,8 @@ void testRemoveUpdatedDataMaterializesKeysNativelyForRegistration() { assertTrue(delete.toLowerCase().contains("using")); assertTrue(delete.contains("analytics_event_delete_keys")); assertTrue(delete.contains("ax.event=k.event")); - assertTrue(!delete.contains("trackerevent")); - assertTrue(!delete.contains("enrollment")); + assertFalse(delete.contains("trackerevent")); + assertFalse(delete.contains("enrollment")); String drop2 = statements.get(4); assertTrue(drop2.toLowerCase().contains("drop table")); @@ -493,7 +494,7 @@ void testRemoveUpdatedDataMaterializesKeysNativelyForSingleEvent() { assertTrue(create.toLowerCase().contains("create table")); assertTrue(create.contains("analytics_event_delete_keys")); assertTrue(create.contains("`event`")); - assertTrue(!create.contains("`year`")); + assertFalse(create.contains("`year`")); String insert = statements.get(2); assertTrue(insert.toLowerCase().contains("insert into")); @@ -511,7 +512,7 @@ void testRemoveUpdatedDataMaterializesKeysNativelyForSingleEvent() { assertTrue(delete.toLowerCase().contains("using")); assertTrue(delete.contains("analytics_event_delete_keys")); assertTrue(delete.contains("ax.event=k.event")); - assertTrue(!delete.contains("singleevent")); + assertFalse(delete.contains("singleevent")); String drop2 = statements.get(4); assertTrue(drop2.toLowerCase().contains("drop table")); diff --git a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java index 993107de1bf3..0665c18c5b88 100644 --- a/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java +++ b/dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/JdbcEventAnalyticsTableManagerTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2026, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -63,7 +63,9 @@ import static org.hisp.dhis.test.TestBase.createTrackedEntityAttribute; import static org.hisp.dhis.test.TestBase.getDate; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -1130,6 +1132,39 @@ void removeUpdatedDataTargetsLiveTable() { assertThat(sql.getValue(), not(containsString(quote(mainTableName + STAGING_TABLE_SUFFIX)))); } + @Test + @DisplayName( + "On Postgres a program without a main table gets the same continuous window as any other," + + " without checking whether its main table exists") + void latestTableOfProgramWithoutMainTableKeepsWindowOnPostgres() { + Program created = createProgram('C'); + + Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate(); + Date lastLatestPartitionUpdate = new DateTime(2019, 3, 1, 9, 0).toDate(); + Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate(); + + AnalyticsTableUpdateParams params = + AnalyticsTableUpdateParams.newBuilder().startTime(startTime).build().withLatestPartition(); + + List> queryResp = new ArrayList<>(); + queryResp.add(Map.of("eventid", 1)); + String tableExistsSql = + sqlBuilder.tableExists(AnalyticsTable.getTableName(AnalyticsTableType.EVENT, created)); + + when(settings.getLastSuccessfulAnalyticsTablesUpdate()).thenReturn(lastFullTableUpdate); + when(settings.getLastSuccessfulLatestAnalyticsPartitionUpdate()) + .thenReturn(lastLatestPartitionUpdate); + when(jdbcTemplate.queryForList(Mockito.anyString())).thenReturn(queryResp); + when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(List.of(created)); + whenConfigurationPeriodSettings(); + + List tables = subject.getAnalyticsTables(params); + assertThat(tables, hasSize(1)); + + assertNotEquals(new Date(0L), tables.get(0).getLatestTablePartition().getStartDate()); + verify(jdbcTemplate, never()).queryForList(tableExistsSql); + } + @Test @DisplayName("getRegularAnalyticsTables excludes programs listed in skipPrograms") void getRegularAnalyticsTablesExcludesSkippedPrograms() {