Skip to content
Merged
16 changes: 15 additions & 1 deletion src/bound_plan/lower/embedded.rs
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,21 @@ impl<'s> Lowerer<'s> {
}
}
}
false => mapper.any_non_null(&value),
false => {
// Array/Map/Tuple/Dynamic cannot be inside Nullable on
// ClickHouse 25.8: `anyIf`, not `anyIfOrNull`. (#1490)
let is_complex = relation
.sources
.iter()
.flat_map(|s| s.columns.iter())
.filter(|(prop, _)| prop.as_str() == p.as_str())
.any(|(_, column)| {
relation.column_types.get(column).is_some_and(
crate::graph_catalog::schema_types::KeyPartType::is_complex,
)
});
mapper.any_non_null(&value, is_complex)
}
};
items.push(select(RenderExpr::Raw(value), p));
}
Expand Down
2 changes: 1 addition & 1 deletion src/bound_plan/lower/expr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2209,7 +2209,7 @@ impl Lowerer<'_> {
// the group's key, determines it), all of it one aggregate.
Ok(match self.grouped_paths.contains(&p) {
true => RenderExpr::Raw(
current_function_mapper().any_non_null(&render_expr_to_sql_plain(&length)),
current_function_mapper().any_non_null(&render_expr_to_sql_plain(&length), false),
),
false => length,
})
Expand Down
59 changes: 59 additions & 0 deletions src/bound_plan/lower/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,65 @@ fn an_end_paired_with_its_id_columns_in_another_order_is_refused() {
}
}

/// #1490: an embedded node's property read from an `Array` / `Map` / `Tuple` /
/// `Dynamic` column is `anyIf` (ClickHouse 25.8 cannot put one inside
/// `Nullable`, which `anyIfOrNull` returns); a scalar column stays
/// `anyIfOrNull` (NULL for a group of NULLs only). The column types come from
/// the node's `column_types` (T1), of every table that holds the property.
#[test]
fn an_embedded_nodes_complex_property_is_read_with_any_if_1490() {
let schema = GraphSchemaConfig::from_yaml_str(
r#"
name: t
graph_schema:
nodes:
- label: Airport
database: db
table: fl
node_id: code
property_mappings: {}
from_node_properties: { code: org, city: org_city, tags: org_tags }
to_node_properties: { code: dst, city: dst_city, tags: dst_tags }
column_types: { org: String, dst: String, org_city: String, dst_city: String, org_tags: "Array(String)", dst_tags: "Array(String)" }
edges:
- type: FLIGHT
database: db
table: fl
from_id: org
to_id: dst
from_node: Airport
to_node: Airport
edge_id: fid
property_mappings: {}
"#,
)
.unwrap()
.to_graph_schema()
.unwrap();
let got = squash(
&translate_bound_plan(
"MATCH (a:Airport) RETURN a.code, a.city, a.tags",
&schema,
&ReadOptions::default(),
)
.unwrap()
.sql,
);
assert!(
got.contains("anyIf(CAST(coalesce(e.tags__cg0, e.tags__cg1), "),
"{got}"
);
assert!(
!got.contains("anyIfOrNull(CAST(coalesce(e.tags__cg0"),
"{got}"
);
// A scalar column is as before.
assert!(
got.contains("anyIfOrNull(CAST(coalesce(e.city__cg0, e.city__cg1), "),
"{got}"
);
}

/// S8b: labels embedded in edge tables (one, and one over two tables), an
/// own-table label at an embedding edge's end, and a "foreign" denormalized
/// label (its own table; edges carry its id).
Expand Down
16 changes: 16 additions & 0 deletions src/graph_catalog/graph_schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1064,6 +1064,16 @@ impl GraphSchema {
.collect();
let name = format!("__cg_nodes_{label}");
let first = defs[0];
// The columns' types of every definition's table (a column named alike
// in two of them: the first's), for `NodeRelation::column_types`.
let mut column_types = std::collections::HashMap::new();
for def in defs.iter() {
for (column, ty) in &def.column_types {
column_types
.entry(column.clone())
.or_insert_with(|| ty.clone());
}
}
let node = NodeSchema {
column_types: Default::default(),
database: String::new(),
Expand Down Expand Up @@ -1104,6 +1114,7 @@ impl GraphSchema {
id,
properties,
sources,
column_types,
},
node,
))
Expand Down Expand Up @@ -1363,6 +1374,11 @@ pub struct NodeRelation {
/// Every property its sources hold (the id's among them), sorted.
pub properties: Vec<String>,
pub sources: Vec<NodeSource>,
/// Column types of the underlying table(s), keyed by column name
/// (VALUE_TYPES.md T1). Used to tell complex (Array/Map/Tuple/Dynamic)
/// properties apart from scalar ones for embedded nodes. (#1490)
pub column_types:
std::collections::HashMap<String, crate::graph_catalog::schema_types::KeyPartType>,
}

/// `a` and `b` (either may be absent) as one schema filter.
Expand Down
1 change: 1 addition & 0 deletions src/graph_catalog/node_property_check.rs
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ mod tests {
id: vec!["code".to_string()],
properties: vec!["city".to_string(), "code".to_string(), "tz".to_string()],
sources,
column_types: std::collections::HashMap::new(),
}
}

Expand Down
34 changes: 34 additions & 0 deletions src/graph_catalog/schema_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,15 @@ impl KeyPartType {
Some(KeyPartType::parse(inner))
}

/// Whether this is an `Array`, `Map`, `Tuple`, `Dynamic` or `Variant`
/// column: a type ClickHouse 25.8 cannot put inside `Nullable` (#1490).
pub fn is_complex(&self) -> bool {
let raw = self.raw.to_ascii_lowercase();
["array(", "map(", "tuple(", "dynamic", "variant("]
.iter()
.any(|p| raw.starts_with(p))
}

/// Whether this is an integer of more than 64 bits (`Int128`, `UInt256`).
pub fn is_wider_than_64_bits(&self) -> bool {
let raw = self.raw.to_ascii_lowercase();
Expand Down Expand Up @@ -864,6 +873,31 @@ mod tests {
assert_eq!(KeyPartType::parse("String").array_element(), None);
}

/// #1490: the types ClickHouse 25.8 cannot put inside `Nullable`.
#[test]
fn a_complex_column_is_an_array_map_tuple_dynamic_or_variant() {
for ty in [
"Array(String)",
"Map(String, UInt8)",
"Tuple(UInt8, String)",
"Dynamic",
"Dynamic(max_types=8)",
"Variant(Array(Int64), String)",
] {
assert!(KeyPartType::parse(ty).is_complex(), "{ty}");
}
for ty in [
"String",
"Nullable(Int64)",
"LowCardinality(String)",
"DateTime64(3)",
"Bool",
"UUID",
] {
assert!(!KeyPartType::parse(ty).is_complex(), "{ty}");
}
}

/// A space before a parenthesis is ClickHouse's same type.
#[test]
fn a_spaced_spelling_parses_as_its_type() {
Expand Down
29 changes: 21 additions & 8 deletions src/sql_generator/function_mapper/clickhouse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,16 @@ impl FunctionMapper for ClickhouseFunctionMapper {
"any"
}

fn any_non_null(&self, value: &str) -> String {
// `any` skips the NULLs of a `Nullable` column but not a `Variant`'s
// NULL: `NULL UNION ALL` an `Array` column is a `Variant`, and `any`
// could pick the arm without the property (#1426). `-OrNull`: a
// group of NULLs only is NULL, not the type's default (`anyIf` of a
// `Nullable(Tuple)` would be `(0, '')`).
format!("anyIfOrNull({value}, NOT isNull({value}))")
fn any_non_null(&self, value: &str, is_complex: bool) -> String {
if is_complex {
// Array/Map/Tuple/Dynamic cannot be inside Nullable on 25.8;
// `anyIf` returns the type's default for an all-NULL group.
format!("anyIf({value}, NOT isNull({value}))")
} else {
// `any` skips the NULLs of a `Nullable` column but not a
// `Variant`'s NULL (#1426). `-OrNull`: NULL for all-NULL.
format!("anyIfOrNull({value}, NOT isNull({value}))")
}
}

fn one_value(&self, value: &str, message: &str) -> Option<String> {
Expand All @@ -83,7 +86,7 @@ impl FunctionMapper for ClickhouseFunctionMapper {
let message = message.replace('\\', "\\\\").replace('\'', "\\'");
Some(format!(
"arrayElement([{}], 1 + throwIf(uniqUpToIf(1)(cityHash64({key}), NOT isNull({value})) > 1, '{message}'))",
self.any_non_null(value)
self.any_non_null(value, false)
))
}

Expand Down Expand Up @@ -585,6 +588,16 @@ mod tests {
);
}

/// #1490: a complex value (`Array` / `Map` / `Tuple` / `Dynamic`) cannot be
/// inside `Nullable` on ClickHouse 25.8, so its aggregate is `anyIf`;
/// every other is `anyIfOrNull`, which is NULL for a group of NULLs only.
#[test]
fn any_non_null_is_any_if_for_a_complex_value_1490() {
let m = ClickhouseFunctionMapper;
assert_eq!(m.any_non_null("v", true), "anyIf(v, NOT isNull(v))");
assert_eq!(m.any_non_null("v", false), "anyIfOrNull(v, NOT isNull(v))");
}

#[test]
fn cast_bool_uses_nullable_bool_for_null_safety() {
// #1057: Nullable(Bool), not bare Bool — CAST(NULL>2 AS Bool) throws
Expand Down
4 changes: 2 additions & 2 deletions src/sql_generator/function_mapper/databricks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -457,7 +457,7 @@ impl FunctionMapper for DatabricksFunctionMapper {
format!("array_size({list})")
}

fn any_non_null(&self, value: &str) -> String {
fn any_non_null(&self, value: &str, _is_complex: bool) -> String {
format!("any_value({value}, true)")
}

Expand All @@ -472,7 +472,7 @@ impl FunctionMapper for DatabricksFunctionMapper {
let message = message.replace('\\', "\\\\").replace('\'', "\\'");
Some(format!(
"CASE WHEN min({key}) <> max({key}) THEN raise_error('{message}') ELSE {} END",
self.any_non_null(value)
self.any_non_null(value, false)
))
}

Expand Down
7 changes: 5 additions & 2 deletions src/sql_generator/function_mapper/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,11 @@ pub(crate) trait FunctionMapper: Send + Sync {

/// An aggregate of any of the values `{0}` that are not NULL (NULL when
/// all are): CH `any` skips a `Nullable`'s NULLs but not a `Variant`'s;
/// Spark's `any_value` needs its `ignoreNulls` argument.
fn any_non_null(&self, value: &str) -> String {
/// Spark's `any_value` needs its `ignoreNulls` argument. `is_complex`: the
/// value is an `Array` / `Map` / `Tuple` / `Dynamic`, which ClickHouse 25.8
/// cannot put inside `Nullable` (`anyIfOrNull` returns one, #1490); only
/// ClickHouse acts on it.
fn any_non_null(&self, value: &str, _is_complex: bool) -> String {
format!("{}({value})", self.any())
}

Expand Down
Loading