diff --git a/src/bound_plan/lower/embedded.rs b/src/bound_plan/lower/embedded.rs index 3ffd87fe9..272eb1e89 100644 --- a/src/bound_plan/lower/embedded.rs +++ b/src/bound_plan/lower/embedded.rs @@ -294,7 +294,21 @@ impl<'s> Lowerer<'s> { } } } - false => mapper.any_non_null(&value), + false => { + // Array/Map/Tuple/Dynamic cannot be inside Nullable on + // ClickHouse 25.8: `anyIf`, not `anyIfOrNull`. (#1490) + let is_complex = relation + .sources + .iter() + .flat_map(|s| s.columns.iter()) + .filter(|(prop, _)| prop.as_str() == p.as_str()) + .any(|(_, column)| { + relation.column_types.get(column).is_some_and( + crate::graph_catalog::schema_types::KeyPartType::is_complex, + ) + }); + mapper.any_non_null(&value, is_complex) + } }; items.push(select(RenderExpr::Raw(value), p)); } diff --git a/src/bound_plan/lower/expr.rs b/src/bound_plan/lower/expr.rs index 987868d68..75fc1da97 100644 --- a/src/bound_plan/lower/expr.rs +++ b/src/bound_plan/lower/expr.rs @@ -2209,7 +2209,7 @@ impl Lowerer<'_> { // the group's key, determines it), all of it one aggregate. Ok(match self.grouped_paths.contains(&p) { true => RenderExpr::Raw( - current_function_mapper().any_non_null(&render_expr_to_sql_plain(&length)), + current_function_mapper().any_non_null(&render_expr_to_sql_plain(&length), false), ), false => length, }) diff --git a/src/bound_plan/lower/tests.rs b/src/bound_plan/lower/tests.rs index 28b3e8505..b4be0ce75 100644 --- a/src/bound_plan/lower/tests.rs +++ b/src/bound_plan/lower/tests.rs @@ -289,6 +289,65 @@ fn an_end_paired_with_its_id_columns_in_another_order_is_refused() { } } +/// #1490: an embedded node's property read from an `Array` / `Map` / `Tuple` / +/// `Dynamic` column is `anyIf` (ClickHouse 25.8 cannot put one inside +/// `Nullable`, which `anyIfOrNull` returns); a scalar column stays +/// `anyIfOrNull` (NULL for a group of NULLs only). The column types come from +/// the node's `column_types` (T1), of every table that holds the property. +#[test] +fn an_embedded_nodes_complex_property_is_read_with_any_if_1490() { + let schema = GraphSchemaConfig::from_yaml_str( + r#" +name: t +graph_schema: + nodes: + - label: Airport + database: db + table: fl + node_id: code + property_mappings: {} + from_node_properties: { code: org, city: org_city, tags: org_tags } + to_node_properties: { code: dst, city: dst_city, tags: dst_tags } + column_types: { org: String, dst: String, org_city: String, dst_city: String, org_tags: "Array(String)", dst_tags: "Array(String)" } + edges: + - type: FLIGHT + database: db + table: fl + from_id: org + to_id: dst + from_node: Airport + to_node: Airport + edge_id: fid + property_mappings: {} +"#, + ) + .unwrap() + .to_graph_schema() + .unwrap(); + let got = squash( + &translate_bound_plan( + "MATCH (a:Airport) RETURN a.code, a.city, a.tags", + &schema, + &ReadOptions::default(), + ) + .unwrap() + .sql, + ); + assert!( + got.contains("anyIf(CAST(coalesce(e.tags__cg0, e.tags__cg1), "), + "{got}" + ); + assert!( + !got.contains("anyIfOrNull(CAST(coalesce(e.tags__cg0"), + "{got}" + ); + // A scalar column is as before. + assert!( + got.contains("anyIfOrNull(CAST(coalesce(e.city__cg0, e.city__cg1), "), + "{got}" + ); +} + /// S8b: labels embedded in edge tables (one, and one over two tables), an /// own-table label at an embedding edge's end, and a "foreign" denormalized /// label (its own table; edges carry its id). diff --git a/src/graph_catalog/graph_schema.rs b/src/graph_catalog/graph_schema.rs index a0138ba15..3de99a164 100644 --- a/src/graph_catalog/graph_schema.rs +++ b/src/graph_catalog/graph_schema.rs @@ -1064,6 +1064,16 @@ impl GraphSchema { .collect(); let name = format!("__cg_nodes_{label}"); let first = defs[0]; + // The columns' types of every definition's table (a column named alike + // in two of them: the first's), for `NodeRelation::column_types`. + let mut column_types = std::collections::HashMap::new(); + for def in defs.iter() { + for (column, ty) in &def.column_types { + column_types + .entry(column.clone()) + .or_insert_with(|| ty.clone()); + } + } let node = NodeSchema { column_types: Default::default(), database: String::new(), @@ -1104,6 +1114,7 @@ impl GraphSchema { id, properties, sources, + column_types, }, node, )) @@ -1363,6 +1374,11 @@ pub struct NodeRelation { /// Every property its sources hold (the id's among them), sorted. pub properties: Vec, pub sources: Vec, + /// Column types of the underlying table(s), keyed by column name + /// (VALUE_TYPES.md T1). Used to tell complex (Array/Map/Tuple/Dynamic) + /// properties apart from scalar ones for embedded nodes. (#1490) + pub column_types: + std::collections::HashMap, } /// `a` and `b` (either may be absent) as one schema filter. diff --git a/src/graph_catalog/node_property_check.rs b/src/graph_catalog/node_property_check.rs index 76f888b39..6ad4c07ef 100644 --- a/src/graph_catalog/node_property_check.rs +++ b/src/graph_catalog/node_property_check.rs @@ -154,6 +154,7 @@ mod tests { id: vec!["code".to_string()], properties: vec!["city".to_string(), "code".to_string(), "tz".to_string()], sources, + column_types: std::collections::HashMap::new(), } } diff --git a/src/graph_catalog/schema_types.rs b/src/graph_catalog/schema_types.rs index 46e4b3487..3e5224cfd 100644 --- a/src/graph_catalog/schema_types.rs +++ b/src/graph_catalog/schema_types.rs @@ -492,6 +492,15 @@ impl KeyPartType { Some(KeyPartType::parse(inner)) } + /// Whether this is an `Array`, `Map`, `Tuple`, `Dynamic` or `Variant` + /// column: a type ClickHouse 25.8 cannot put inside `Nullable` (#1490). + pub fn is_complex(&self) -> bool { + let raw = self.raw.to_ascii_lowercase(); + ["array(", "map(", "tuple(", "dynamic", "variant("] + .iter() + .any(|p| raw.starts_with(p)) + } + /// Whether this is an integer of more than 64 bits (`Int128`, `UInt256`). pub fn is_wider_than_64_bits(&self) -> bool { let raw = self.raw.to_ascii_lowercase(); @@ -864,6 +873,31 @@ mod tests { assert_eq!(KeyPartType::parse("String").array_element(), None); } + /// #1490: the types ClickHouse 25.8 cannot put inside `Nullable`. + #[test] + fn a_complex_column_is_an_array_map_tuple_dynamic_or_variant() { + for ty in [ + "Array(String)", + "Map(String, UInt8)", + "Tuple(UInt8, String)", + "Dynamic", + "Dynamic(max_types=8)", + "Variant(Array(Int64), String)", + ] { + assert!(KeyPartType::parse(ty).is_complex(), "{ty}"); + } + for ty in [ + "String", + "Nullable(Int64)", + "LowCardinality(String)", + "DateTime64(3)", + "Bool", + "UUID", + ] { + assert!(!KeyPartType::parse(ty).is_complex(), "{ty}"); + } + } + /// A space before a parenthesis is ClickHouse's same type. #[test] fn a_spaced_spelling_parses_as_its_type() { diff --git a/src/sql_generator/function_mapper/clickhouse.rs b/src/sql_generator/function_mapper/clickhouse.rs index ccd6c18a4..3a712e51a 100644 --- a/src/sql_generator/function_mapper/clickhouse.rs +++ b/src/sql_generator/function_mapper/clickhouse.rs @@ -64,13 +64,16 @@ impl FunctionMapper for ClickhouseFunctionMapper { "any" } - fn any_non_null(&self, value: &str) -> String { - // `any` skips the NULLs of a `Nullable` column but not a `Variant`'s - // NULL: `NULL UNION ALL` an `Array` column is a `Variant`, and `any` - // could pick the arm without the property (#1426). `-OrNull`: a - // group of NULLs only is NULL, not the type's default (`anyIf` of a - // `Nullable(Tuple)` would be `(0, '')`). - format!("anyIfOrNull({value}, NOT isNull({value}))") + fn any_non_null(&self, value: &str, is_complex: bool) -> String { + if is_complex { + // Array/Map/Tuple/Dynamic cannot be inside Nullable on 25.8; + // `anyIf` returns the type's default for an all-NULL group. + format!("anyIf({value}, NOT isNull({value}))") + } else { + // `any` skips the NULLs of a `Nullable` column but not a + // `Variant`'s NULL (#1426). `-OrNull`: NULL for all-NULL. + format!("anyIfOrNull({value}, NOT isNull({value}))") + } } fn one_value(&self, value: &str, message: &str) -> Option { @@ -83,7 +86,7 @@ impl FunctionMapper for ClickhouseFunctionMapper { let message = message.replace('\\', "\\\\").replace('\'', "\\'"); Some(format!( "arrayElement([{}], 1 + throwIf(uniqUpToIf(1)(cityHash64({key}), NOT isNull({value})) > 1, '{message}'))", - self.any_non_null(value) + self.any_non_null(value, false) )) } @@ -585,6 +588,16 @@ mod tests { ); } + /// #1490: a complex value (`Array` / `Map` / `Tuple` / `Dynamic`) cannot be + /// inside `Nullable` on ClickHouse 25.8, so its aggregate is `anyIf`; + /// every other is `anyIfOrNull`, which is NULL for a group of NULLs only. + #[test] + fn any_non_null_is_any_if_for_a_complex_value_1490() { + let m = ClickhouseFunctionMapper; + assert_eq!(m.any_non_null("v", true), "anyIf(v, NOT isNull(v))"); + assert_eq!(m.any_non_null("v", false), "anyIfOrNull(v, NOT isNull(v))"); + } + #[test] fn cast_bool_uses_nullable_bool_for_null_safety() { // #1057: Nullable(Bool), not bare Bool — CAST(NULL>2 AS Bool) throws diff --git a/src/sql_generator/function_mapper/databricks.rs b/src/sql_generator/function_mapper/databricks.rs index 731ac6f95..c3d11ca54 100644 --- a/src/sql_generator/function_mapper/databricks.rs +++ b/src/sql_generator/function_mapper/databricks.rs @@ -457,7 +457,7 @@ impl FunctionMapper for DatabricksFunctionMapper { format!("array_size({list})") } - fn any_non_null(&self, value: &str) -> String { + fn any_non_null(&self, value: &str, _is_complex: bool) -> String { format!("any_value({value}, true)") } @@ -472,7 +472,7 @@ impl FunctionMapper for DatabricksFunctionMapper { let message = message.replace('\\', "\\\\").replace('\'', "\\'"); Some(format!( "CASE WHEN min({key}) <> max({key}) THEN raise_error('{message}') ELSE {} END", - self.any_non_null(value) + self.any_non_null(value, false) )) } diff --git a/src/sql_generator/function_mapper/mod.rs b/src/sql_generator/function_mapper/mod.rs index 0f2968a0a..2229c7191 100644 --- a/src/sql_generator/function_mapper/mod.rs +++ b/src/sql_generator/function_mapper/mod.rs @@ -52,8 +52,11 @@ pub(crate) trait FunctionMapper: Send + Sync { /// An aggregate of any of the values `{0}` that are not NULL (NULL when /// all are): CH `any` skips a `Nullable`'s NULLs but not a `Variant`'s; - /// Spark's `any_value` needs its `ignoreNulls` argument. - fn any_non_null(&self, value: &str) -> String { + /// Spark's `any_value` needs its `ignoreNulls` argument. `is_complex`: the + /// value is an `Array` / `Map` / `Tuple` / `Dynamic`, which ClickHouse 25.8 + /// cannot put inside `Nullable` (`anyIfOrNull` returns one, #1490); only + /// ClickHouse acts on it. + fn any_non_null(&self, value: &str, _is_complex: bool) -> String { format!("{}({value})", self.any()) }