Skip to content

Commit d6cec7e

Browse files
harshachharshach
authored
Fixes open-metadata#28229: cascade Table certification PATCH to child search docs (open-metadata#28236)
* Fixes open-metadata#28229: cascade Table certification PATCH to child search docs When a Table's certification is added, changed, or removed via PATCH the existing cascadeCertificationToChildren path never fired because SearchRepository.requiresPropagation returned false on a cert-only ChangeDescription. TableRepository did not list certification in its propagation descriptors, so the gate stayed closed and the DQ dashboard's Certification filter kept returning the stale cert on test_case / test_case_result / test_case_resolution_status / test_suite / column docs until a full reindex. Add an EXTERNAL_HANDLER PropagationType for fields whose cascade is driven by a dedicated SearchRepository handler instead of the generic descriptor-driven script (cert needs full-object replace on add/update and explicit removal on delete, which RAW_REPLACE can't express because it restores the old value on delete). Register certification with this type on TableRepository so the gate opens and the existing cascadeCertificationToChildren method runs. Add no-op cases in the three appendAdd/Update/DeleteScript switches so the new enum value doesn't accidentally trigger generic auto-propagation.
1 parent e276571 commit d6cec7e

5 files changed

Lines changed: 278 additions & 1 deletion

File tree

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TableCertificationPropagationIT.java‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
/*
2+
* Copyright 2026 Collate
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
* Unless required by applicable law or agreed to in writing, software
8+
* distributed under the License is distributed on an "AS IS" BASIS,
9+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
* See the License for the specific language governing permissions and
11+
* limitations under the License.
12+
*/
13+
package org.openmetadata.it.tests;
14+
15+
import static org.awaitility.Awaitility.await;
16+
import static org.junit.jupiter.api.Assertions.assertEquals;
17+
import static org.junit.jupiter.api.Assertions.assertNotNull;
18+
import static org.junit.jupiter.api.Assertions.assertTrue;
19+
20+
import com.fasterxml.jackson.databind.JsonNode;
21+
import com.fasterxml.jackson.databind.ObjectMapper;
22+
import java.time.Duration;
23+
import java.util.List;
24+
import java.util.Map;
25+
import org.junit.jupiter.api.Test;
26+
import org.junit.jupiter.api.TestInstance;
27+
import org.junit.jupiter.api.parallel.Execution;
28+
import org.junit.jupiter.api.parallel.ExecutionMode;
29+
import org.openmetadata.it.bootstrap.SharedEntities;
30+
import org.openmetadata.it.util.SdkClients;
31+
import org.openmetadata.schema.api.data.CreateDatabase;
32+
import org.openmetadata.schema.api.data.CreateDatabaseSchema;
33+
import org.openmetadata.schema.api.data.CreateTable;
34+
import org.openmetadata.schema.api.tests.CreateTestCase;
35+
import org.openmetadata.schema.entity.data.Database;
36+
import org.openmetadata.schema.entity.data.DatabaseSchema;
37+
import org.openmetadata.schema.entity.data.Table;
38+
import org.openmetadata.schema.tests.TestCase;
39+
import org.openmetadata.schema.tests.TestCaseParameterValue;
40+
import org.openmetadata.schema.type.AssetCertification;
41+
import org.openmetadata.schema.type.Column;
42+
import org.openmetadata.schema.type.ColumnDataType;
43+
import org.openmetadata.schema.type.TagLabel;
44+
import org.openmetadata.sdk.client.OpenMetadataClient;
45+
46+
/**
47+
* Regression for the Table certification cascade bug (issue #28229). When a Table's certification
48+
* is added, changed, or removed via PATCH, the existing {@code cascadeCertificationToChildren} path
49+
* in {@code SearchRepository} must propagate the new cert onto every denormalized child search doc
50+
* (test_case, test_case_result, test_case_resolution_status, test_suite).
51+
*
52+
* <p>Without the fix on {@code TableRepository.getSearchPropagationDescriptors}, the
53+
* {@code requiresPropagation} gate in {@code SearchRepository.updateEntityIndex} returns
54+
* {@code false} on a cert-only ChangeDescription, the cascade never fires, and the Data Quality
55+
* dashboard's Certification filter keeps returning the stale cert until a full reindex.
56+
*/
57+
@Execution(ExecutionMode.CONCURRENT)
58+
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
59+
public class TableCertificationPropagationIT {
60+
61+
private static final ObjectMapper MAPPER = new ObjectMapper();
62+
private static final String CERTIFICATION_GOLD = "Certification.Gold";
63+
private static final String CERTIFICATION_SILVER = "Certification.Silver";
64+
private static final Duration AWAIT_TIMEOUT = Duration.ofMinutes(1);
65+
private static final Duration POLL_INTERVAL = Duration.ofSeconds(2);
66+
67+
@Test
68+
void certChangeOnTable_cascadesToTestCaseSearchDoc() throws Exception {
69+
OpenMetadataClient client = SdkClients.adminClient();
70+
long ts = System.currentTimeMillis();
71+
Database database = null;
72+
try {
73+
database =
74+
client
75+
.databases()
76+
.create(
77+
new CreateDatabase()
78+
.withName("cert_prop_db_" + ts)
79+
.withService(SharedEntities.get().MYSQL_SERVICE.getFullyQualifiedName()));
80+
DatabaseSchema schema =
81+
client
82+
.databaseSchemas()
83+
.create(
84+
new CreateDatabaseSchema()
85+
.withName("cert_prop_schema_" + ts)
86+
.withDatabase(database.getFullyQualifiedName()));
87+
Table table =
88+
client
89+
.tables()
90+
.create(
91+
new CreateTable()
92+
.withName("cert_prop_table_" + ts)
93+
.withDatabaseSchema(schema.getFullyQualifiedName())
94+
.withColumns(
95+
List.of(
96+
new Column().withName("id").withDataType(ColumnDataType.BIGINT))));
97+
98+
long now = System.currentTimeMillis();
99+
long expiry = now + Duration.ofDays(30).toMillis();
100+
table.setCertification(buildCertification(CERTIFICATION_GOLD, now, expiry));
101+
client.tables().update(table.getId().toString(), table);
102+
103+
TestCase testCase =
104+
client
105+
.testCases()
106+
.create(
107+
new CreateTestCase()
108+
.withName("cert_prop_tc_" + ts)
109+
.withEntityLink("<#E::table::" + table.getFullyQualifiedName() + ">")
110+
.withTestDefinition("tableRowCountToEqual")
111+
.withParameterValues(
112+
List.of(
113+
new TestCaseParameterValue().withName("value").withValue("100"))));
114+
115+
awaitTestCaseCertification(client, testCase.getFullyQualifiedName(), CERTIFICATION_GOLD);
116+
117+
table = client.tables().get(table.getId().toString(), "certification");
118+
table.setCertification(buildCertification(CERTIFICATION_SILVER, now, expiry));
119+
client.tables().update(table.getId().toString(), table);
120+
121+
awaitTestCaseCertification(client, testCase.getFullyQualifiedName(), CERTIFICATION_SILVER);
122+
123+
table = client.tables().get(table.getId().toString(), "certification");
124+
table.setCertification(null);
125+
client.tables().update(table.getId().toString(), table);
126+
127+
awaitTestCaseCertificationAbsent(client, testCase.getFullyQualifiedName());
128+
} finally {
129+
if (database != null) {
130+
try {
131+
client
132+
.databases()
133+
.delete(
134+
database.getId().toString(), Map.of("hardDelete", "true", "recursive", "true"));
135+
} catch (Exception ignored) {
136+
// best-effort cleanup; assertion failures take precedence
137+
}
138+
}
139+
}
140+
}
141+
142+
private static void awaitTestCaseCertification(
143+
OpenMetadataClient client, String testCaseFqn, String expectedFqn) {
144+
await("test_case_search_index reflects cert " + expectedFqn + " for " + testCaseFqn)
145+
.atMost(AWAIT_TIMEOUT)
146+
.pollInterval(POLL_INTERVAL)
147+
.ignoreExceptions()
148+
.untilAsserted(
149+
() -> {
150+
JsonNode src = fetchTestCaseSource(client, testCaseFqn);
151+
JsonNode certFqn = src.path("certification").path("tagLabel").path("tagFQN");
152+
assertEquals(
153+
expectedFqn,
154+
certFqn.asText(),
155+
() ->
156+
"test_case search doc certification mismatch; cert was: "
157+
+ src.path("certification"));
158+
});
159+
}
160+
161+
private static void awaitTestCaseCertificationAbsent(
162+
OpenMetadataClient client, String testCaseFqn) {
163+
await("test_case_search_index has no certification for " + testCaseFqn)
164+
.atMost(AWAIT_TIMEOUT)
165+
.pollInterval(POLL_INTERVAL)
166+
.ignoreExceptions()
167+
.untilAsserted(
168+
() -> {
169+
JsonNode src = fetchTestCaseSource(client, testCaseFqn);
170+
JsonNode cert = src.path("certification");
171+
assertTrue(
172+
cert.isMissingNode() || cert.isNull(),
173+
() -> "test_case search doc still carries certification: " + cert);
174+
});
175+
}
176+
177+
private static JsonNode fetchTestCaseSource(OpenMetadataClient client, String testCaseFqn)
178+
throws Exception {
179+
String rawJson =
180+
client
181+
.search()
182+
.query("fullyQualifiedName.keyword:\"" + testCaseFqn + "\"")
183+
.index("test_case_search_index")
184+
.size(1)
185+
.execute();
186+
JsonNode root = MAPPER.readTree(rawJson);
187+
JsonNode hits = root.path("hits").path("hits");
188+
assertNotNull(hits, "search response missing hits");
189+
assertTrue(
190+
hits.isArray() && hits.size() > 0,
191+
() -> "test case " + testCaseFqn + " not yet indexed; raw=" + rawJson);
192+
return hits.get(0).path("_source");
193+
}
194+
195+
private static AssetCertification buildCertification(String fqn, long appliedDate, long expiry) {
196+
return new AssetCertification()
197+
.withTagLabel(
198+
new TagLabel()
199+
.withTagFQN(fqn)
200+
.withSource(TagLabel.TagSource.CLASSIFICATION)
201+
.withLabelType(TagLabel.LabelType.MANUAL))
202+
.withAppliedDate(appliedDate)
203+
.withExpiryDate(expiry);
204+
}
205+
}

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import static org.openmetadata.schema.type.Include.ALL;
2525
import static org.openmetadata.schema.type.Include.NON_DELETED;
2626
import static org.openmetadata.service.Entity.DATABASE_SCHEMA;
27+
import static org.openmetadata.service.Entity.FIELD_CERTIFICATION;
2728
import static org.openmetadata.service.Entity.FIELD_DATA_PRODUCTS;
2829
import static org.openmetadata.service.Entity.FIELD_OWNERS;
2930
import static org.openmetadata.service.Entity.FIELD_TAGS;
@@ -1724,6 +1725,13 @@ public List<PropagationDescriptor> getSearchPropagationDescriptors() {
17241725
FIELD_DATA_PRODUCTS,
17251726
PropagationDescriptor.PropagationType.ENTITY_REFERENCE_LIST,
17261727
null));
1728+
// Required so SearchRepository.requiresPropagation opens the gate on a cert-only PATCH;
1729+
// the actual cascade onto child docs (test_case, test_case_result, test_case_resolution_status,
1730+
// test_suite, column) is handled by SearchRepository.cascadeCertificationToChildren, not by
1731+
// the generic descriptor-driven script.
1732+
descriptors.add(
1733+
new PropagationDescriptor(
1734+
FIELD_CERTIFICATION, PropagationDescriptor.PropagationType.EXTERNAL_HANDLER, null));
17271735
return descriptors;
17281736
}
17291737

‎openmetadata-service/src/main/java/org/openmetadata/service/search/PropagationDescriptor.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,12 @@ public enum PropagationType {
1111
TAG_LABEL_LIST,
1212
NESTED_FIELD,
1313
SIMPLE_VALUE,
14-
RAW_REPLACE
14+
RAW_REPLACE,
15+
// Field is gated for propagation but the actual cascade is driven by a dedicated handler
16+
// in SearchRepository (e.g. propagateCertificationTags / cascadeCertificationToChildren),
17+
// because the generic descriptor-driven scripts can't express its semantics — cert, for
18+
// example, needs full-object replace on add/update and explicit removal on delete, which
19+
// RAW_REPLACE can't do (RAW_REPLACE restores the old value on delete).
20+
EXTERNAL_HANDLER
1521
}
1622
}

‎openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2062,6 +2062,9 @@ private void appendAddScript(
20622062
script.append(
20632063
String.format("ctx._source.%s = params.%s", field.getName(), field.getName()));
20642064
}
2065+
case EXTERNAL_HANDLER -> {
2066+
// No-op: a dedicated handler (e.g. propagateCertificationTags) drives the cascade.
2067+
}
20652068
}
20662069
script.append(" ");
20672070
}
@@ -2113,6 +2116,9 @@ private void appendDeleteScript(
21132116
script.append(
21142117
String.format("ctx._source.%s = params.%s", field.getName(), field.getName()));
21152118
}
2119+
case EXTERNAL_HANDLER -> {
2120+
// No-op: a dedicated handler (e.g. propagateCertificationTags) drives the cascade.
2121+
}
21162122
}
21172123
script.append(" ");
21182124
}
@@ -2176,6 +2182,9 @@ private void appendUpdateScript(
21762182
script.append(
21772183
String.format("ctx._source.%s = params.%s", field.getName(), field.getName()));
21782184
}
2185+
case EXTERNAL_HANDLER -> {
2186+
// No-op: a dedicated handler (e.g. propagateCertificationTags) drives the cascade.
2187+
}
21792188
}
21802189
script.append(" ");
21812190
}

‎openmetadata-service/src/test/java/org/openmetadata/service/search/SearchRepositoryBehaviorTest.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,9 @@ private List<PropagationDescriptor> buildDescriptorsFor(String entityType) {
297297
Entity.FIELD_DATA_PRODUCTS,
298298
PropagationDescriptor.PropagationType.ENTITY_REFERENCE_LIST,
299299
null));
300+
descriptors.add(
301+
new PropagationDescriptor(
302+
"certification", PropagationDescriptor.PropagationType.EXTERNAL_HANDLER, null));
300303
} else if (Entity.GLOSSARY_TERM.equals(entityType)) {
301304
descriptors.add(
302305
new PropagationDescriptor(
@@ -1808,6 +1811,52 @@ void requiresPropagationReturnsTrueForTagCertificationUpdateEvenWhenCertificatio
18081811
tag));
18091812
}
18101813

1814+
@Test
1815+
void requiresPropagationReturnsTrueForTableCertificationUpdate() throws Exception {
1816+
// Regression for issue #28229: a cert-only PATCH on a Table must open the propagation gate
1817+
// so cascadeCertificationToChildren can push the new cert onto every denormalized child doc
1818+
// (test_case, test_case_result, test_case_resolution_status, test_suite, column).
1819+
EntityInterface table = mockEntity(Entity.TABLE, UUID.randomUUID(), "orders");
1820+
assertTrue(
1821+
invokeRequiresPropagation(
1822+
changeDescription(
1823+
List.of(),
1824+
List.of(
1825+
new FieldChange()
1826+
.withName("certification")
1827+
.withOldValue("{}")
1828+
.withNewValue("{}")),
1829+
List.of()),
1830+
Entity.TABLE,
1831+
table));
1832+
}
1833+
1834+
@Test
1835+
void requiresPropagationReturnsTrueForTableCertificationAdded() throws Exception {
1836+
EntityInterface table = mockEntity(Entity.TABLE, UUID.randomUUID(), "orders");
1837+
assertTrue(
1838+
invokeRequiresPropagation(
1839+
changeDescription(
1840+
List.of(new FieldChange().withName("certification").withNewValue("{}")),
1841+
List.of(),
1842+
List.of()),
1843+
Entity.TABLE,
1844+
table));
1845+
}
1846+
1847+
@Test
1848+
void requiresPropagationReturnsTrueForTableCertificationRemoved() throws Exception {
1849+
EntityInterface table = mockEntity(Entity.TABLE, UUID.randomUUID(), "orders");
1850+
assertTrue(
1851+
invokeRequiresPropagation(
1852+
changeDescription(
1853+
List.of(),
1854+
List.of(),
1855+
List.of(new FieldChange().withName("certification").withOldValue("{}"))),
1856+
Entity.TABLE,
1857+
table));
1858+
}
1859+
18111860
@Test
18121861
void requiresPropagationReturnsFalseForUpstreamEntityRelationshipNotInDescriptors()
18131862
throws Exception {

0 commit comments

Comments
 (0)