From 74879426f7f18deff173a27a7117284fc350a3f0 Mon Sep 17 00:00:00 2001 From: ila Date: Fri, 30 Oct 2026 08:25:43 +0100 Subject: [PATCH 01/32] Draw each released cell's noise and NULL decision from its own generator The noised aggregates drew from one generator per finalize batch (or per vector), seeded from the query seed. Every cell of a query therefore shared the start of the same random sequence: NULL decisions were all-or-nothing across groups (TPC-H Q18: 0 or 57 of 57 NULL per seed) and the noise of all groups fell on the same side of the exact value. A cell's draws also depended on its position in the batch. PacCellRng seeds a generator per cell from hash(seed, key_hash, counters). Cells are now independent, and a cell's release no longer depends on row order, thread or batch. Used by sum, clip_sum, count, min/max, clip_min/max, priv_finalize, as_noised and as_noised_div. Add test/sql/pac_noise_independence.test: NULL rates and noise signs across 1000 groups, and order independence over one and several vectors. Co-Authored-By: Claude Opus 5.5 --- src/aggregates/as_aggregate.cpp | 6 ++ src/aggregates/as_clip_min_max.cpp | 11 +- src/aggregates/as_clip_sum.cpp | 12 ++- src/aggregates/as_count.cpp | 13 +-- src/aggregates/as_finalize.cpp | 6 +- src/aggregates/as_min_max.cpp | 13 +-- src/aggregates/as_sum.cpp | 14 +-- src/categorical/pac_categorical.cpp | 25 ++--- src/include/aggregates/as_aggregate.hpp | 6 ++ test/sql/pac_noise_independence.test | 133 ++++++++++++++++++++++++ 10 files changed, 195 insertions(+), 44 deletions(-) create mode 100644 test/sql/pac_noise_independence.test diff --git a/src/aggregates/as_aggregate.cpp b/src/aggregates/as_aggregate.cpp index b2f00c8b..5fbdabaa 100644 --- a/src/aggregates/as_aggregate.cpp +++ b/src/aggregates/as_aggregate.cpp @@ -133,6 +133,12 @@ bool PacNoiseInNull(uint64_t key_hash, double mi, double correction, std::mt1993 return pac_popcount64(~key_hash) > threshold; } +std::mt19937_64 PacCellRng(uint64_t seed, uint64_t key_hash, const PAC_FLOAT counters[64]) { + hash_t h = CombineHash(Hash(seed), Hash(key_hash)); + h = CombineHash(h, Hash(reinterpret_cast(counters), 64 * sizeof(PAC_FLOAT))); + return std::mt19937_64(h); +} + // ============================================================================ // Utility NULLing: probabilistically NULL low-SNR cells (post-processing) // ============================================================================ diff --git a/src/aggregates/as_clip_min_max.cpp b/src/aggregates/as_clip_min_max.cpp index 5e217eba..5c21bac1 100644 --- a/src/aggregates/as_clip_min_max.cpp +++ b/src/aggregates/as_clip_min_max.cpp @@ -365,11 +365,6 @@ static void PacClipMinMaxFinalize(Vector &states, AggregateInputData &input, Vec continue; } uint64_t key_hash = (pos ? pos->key_hash : 0) | (neg ? neg->key_hash : 0); - std::mt19937_64 gen(bind.seed); - if (PacNoiseInNull(key_hash, mi, correction, gen)) { - result_mask.SetInvalid(offset + i); - continue; - } uint64_t update_count = 0; if (pos) { pos->GetTotals(buf, clip_support, clip_scale); @@ -392,6 +387,12 @@ static void PacClipMinMaxFinalize(Vector &states, AggregateInputData &input, Vec } update_count += neg->update_count; } + // per-cell generator: independent draws across groups, the same whatever the processing order + auto gen = PacCellRng(bind.seed, key_hash, buf); + if (PacNoiseInNull(key_hash, mi, correction, gen)) { + result_mask.SetInvalid(offset + i); + continue; + } CheckPacSampleDiversity(key_hash, buf, update_count, IS_MAX ? "as_noised_clip_max" : "as_noised_clip_min", bind); double noise_var = 0.0; diff --git a/src/aggregates/as_clip_sum.cpp b/src/aggregates/as_clip_sum.cpp index 30b460a1..67e8f6ae 100644 --- a/src/aggregates/as_clip_sum.cpp +++ b/src/aggregates/as_clip_sum.cpp @@ -567,11 +567,7 @@ static void PacClipSumFinalize(Vector &states, AggregateInputData &input, Vector continue; } uint64_t key_hash = pos->key_hash; - std::mt19937_64 gen(bind.seed); - if (PacNoiseInNull(key_hash, mi, correction, gen)) { - result_mask.SetInvalid(offset + i); - continue; - } + uint64_t null_key_hash = key_hash; // the NULL decision considers the worlds of the positive state // Non-mutating: just read totals with clip_support filtering pos->GetTotals(buf, clip_support, clip_scale); @@ -588,6 +584,12 @@ static void PacClipSumFinalize(Vector &states, AggregateInputData &input, Vector } update_count += neg->update_count; } + // per-cell generator: independent draws across groups, the same whatever the processing order + auto gen = PacCellRng(bind.seed, key_hash, buf); + if (PacNoiseInNull(null_key_hash, mi, correction, gen)) { + result_mask.SetInvalid(offset + i); + continue; + } CheckPacSampleDiversity(key_hash, buf, update_count, "as_clip_sum", bind); double noise_var = 0.0; diff --git a/src/aggregates/as_count.cpp b/src/aggregates/as_count.cpp index 8aa069b5..a9d0aed6 100644 --- a/src/aggregates/as_count.cpp +++ b/src/aggregates/as_count.cpp @@ -222,7 +222,6 @@ void PacCountFinalize(Vector &states, AggregateInputData &input, Vector &result, auto data = FlatVector::GetData(result); auto &result_mask = FlatVector::Validity(result); auto &bind = input.bind_data->Cast(); - std::mt19937_64 gen(bind.seed); double mi = bind.mi; double correction = bind.correction; uint64_t query_hash = bind.query_hash; @@ -234,16 +233,18 @@ void PacCountFinalize(Vector &states, AggregateInputData &input, Vector &result, #endif PacCountState *s = aggs[i]->GetState(); uint64_t key_hash = s ? s->key_hash : 0; - // Check if we should return NULL based on key_hash (uses mi and correction) - if (PacNoiseInNull(key_hash, mi, correction, gen)) { - result_mask.SetInvalid(offset + i); - continue; - } if (s) { s->GetTotalsWithSWAR(buf); } else { memset(buf, 0, sizeof(buf)); } + // per-cell generator: independent draws across groups, the same whatever the processing order + auto gen = PacCellRng(bind.seed, key_hash, buf); + // Check if we should return NULL based on key_hash (uses mi and correction) + if (PacNoiseInNull(key_hash, mi, correction, gen)) { + result_mask.SetInvalid(offset + i); + continue; + } CheckPacSampleDiversity(key_hash, buf, s ? s->GetUpdateCount() : 0, "as_noised_count", input.bind_data->Cast()); // Multiply by 2 to compensate for 50% sampling, then apply correction diff --git a/src/aggregates/as_finalize.cpp b/src/aggregates/as_finalize.cpp index 65915eb1..cedf3709 100644 --- a/src/aggregates/as_finalize.cpp +++ b/src/aggregates/as_finalize.cpp @@ -63,8 +63,8 @@ static void PacFinalizeFunction(DataChunk &args, ExpressionState &state, Vector counters[j] = child_data.validity.RowIsValid(child_idx) ? child_values[child_idx] : 0; } - // RNG seeded per row using seed + list offset (stable across DataChunk batches) - std::mt19937_64 gen(seed ^ (entry.offset * PAC_MAGIC_HASH)); + // per-cell generator: independent draws across rows, the same whatever the batch + auto gen = PacCellRng(seed, ~0ULL, counters); result_data[i] = PacNoisySampleFrom64Counters(counters, mi, 1.0, gen, 0ULL, query_hash, nullptr); } @@ -113,7 +113,7 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader) { auto child_idx = child_data.sel->get_index(entry.offset + j); counters[j] = child_data.validity.RowIsValid(child_idx) ? child_values[child_idx] : 0; } - std::mt19937_64 gen(seed ^ (entry.offset * PAC_MAGIC_HASH)); + auto gen = PacCellRng(seed, ~0ULL, counters); tmp[i] = PacNoisySampleFrom64Counters(counters, mi, 1.0, gen, 0ULL, query_hash, nullptr); } diff --git a/src/aggregates/as_min_max.cpp b/src/aggregates/as_min_max.cpp index 3412c7c6..493b0926 100644 --- a/src/aggregates/as_min_max.cpp +++ b/src/aggregates/as_min_max.cpp @@ -113,18 +113,19 @@ static void PacMinMaxFinalize(Vector &states, AggregateInputData &input, Vector auto *s = state_ptrs[i]->GetState(); // Check if we should return NULL based on key_hash uint64_t key_hash = s ? s->key_hash : 0; - std::mt19937_64 gen(input.bind_data->Cast().seed); - // mi controls noise mode, correction reduces NULL probability (but does NOT scale min/max values) - if (PacNoiseInNull(key_hash, mi, correction, gen)) { - result_mask.SetInvalid(offset + i); - continue; - } PAC_FLOAT buf[64]; if (s && s->initialized) { s->GetTotals(buf); } else { memset(buf, 0, sizeof(buf)); } + // per-cell generator: independent draws across groups, the same whatever the processing order + auto gen = PacCellRng(input.bind_data->Cast().seed, key_hash, buf); + // mi controls noise mode, correction reduces NULL probability (but does NOT scale min/max values) + if (PacNoiseInNull(key_hash, mi, correction, gen)) { + result_mask.SetInvalid(offset + i); + continue; + } CheckPacSampleDiversity(key_hash, buf, s ? s->update_count : 0, IS_MAX ? "as_noised_max" : "as_noised_min", input.bind_data->Cast()); // Pass mi for noise, 1.0 as correction (no value scaling for min/max) diff --git a/src/aggregates/as_sum.cpp b/src/aggregates/as_sum.cpp index 9653a47c..6d4aff87 100644 --- a/src/aggregates/as_sum.cpp +++ b/src/aggregates/as_sum.cpp @@ -445,14 +445,8 @@ void PacSumFinalize(Vector &states, AggregateInputData &input, Vector &result, i result_mask.SetInvalid(offset + i); // return NULL (no values seen) continue; } - // Use per-group deterministic RNG seeded by both privacy_seed and key_hash - // This ensures each group gets the same noise regardless of processing order uint64_t key_hash = pos->key_hash; - std::mt19937_64 gen(input.bind_data->Cast().seed); - if (PacNoiseInNull(key_hash, mi, correction, gen)) { - result_mask.SetInvalid(offset + i); // return NULL (probabilistic decision) - continue; - } + uint64_t null_key_hash = key_hash; // the NULL decision considers the worlds of the positive state #ifndef PAC_SIGNEDSUM // Double-sided mode: use SFINAE helpers (cast to unsigned for int states only) FlushPosState(pos, input.allocator); @@ -464,6 +458,12 @@ void PacSumFinalize(Vector &states, AggregateInputData &input, Vector &result, i pos->GetTotals(buf); uint64_t update_count = pos->update_count; #endif + // per-cell generator: independent draws across groups, the same whatever the processing order + auto gen = PacCellRng(input.bind_data->Cast().seed, key_hash, buf); + if (PacNoiseInNull(null_key_hash, mi, correction, gen)) { + result_mask.SetInvalid(offset + i); // return NULL (probabilistic decision) + continue; + } CheckPacSampleDiversity(key_hash, buf, update_count, "as_noised_sum", input.bind_data->Cast()); double noise_var = 0.0; PAC_FLOAT result_val = diff --git a/src/categorical/pac_categorical.cpp b/src/categorical/pac_categorical.cpp index 3211cfc0..2aefca1d 100644 --- a/src/categorical/pac_categorical.cpp +++ b/src/categorical/pac_categorical.cpp @@ -587,8 +587,6 @@ static void PacNoisedFunction(DataChunk &args, ExpressionState &state, Vector &r auto &function = state.expr.Cast(); auto &bind_data = function.bind_info->Cast(); - std::mt19937_64 gen(bind_data.seed); - UnifiedVectorFormat list_data; list_vec.ToUnifiedFormat(count, list_data); @@ -619,17 +617,19 @@ static void PacNoisedFunction(DataChunk &args, ExpressionState &state, Vector &r // Assume all 64 worlds are populated (all bits set) uint64_t key_hash = ~uint64_t(0); - // Check if we should return NULL based on key_hash (uses mi and correction) - if (PacNoiseInNull(key_hash, bind_data.mi, bind_data.correction, gen)) { - result_validity.SetInvalid(i); - continue; - } // Extract counter values from the list PAC_FLOAT counters[64]; for (idx_t j = 0; j < 64; j++) { auto child_idx = child_data.sel->get_index(entry.offset + j); counters[j] = child_values[child_idx]; } + // per-cell generator: independent draws across rows, the same whatever the batch + auto gen = PacCellRng(bind_data.seed, key_hash, counters); + // Check if we should return NULL based on key_hash (uses mi and correction) + if (PacNoiseInNull(key_hash, bind_data.mi, bind_data.correction, gen)) { + result_validity.SetInvalid(i); + continue; + } double noise_var = 0.0; result_data[i] = PacNoisySampleFrom64Counters(counters, bind_data.mi, bind_data.correction, gen, ~key_hash, bind_data.query_hash, bind_data.pstate, &noise_var); @@ -733,7 +733,6 @@ static void PacNoisedDivFunction(DataChunk &args, ExpressionState &state, Vector auto &function = state.expr.Cast(); auto &bind_data = function.bind_info->Cast(); - std::mt19937_64 gen(bind_data.seed); UnifiedVectorFormat sum_data, cnt_data; sum_vec.ToUnifiedFormat(count, sum_data); @@ -768,10 +767,6 @@ static void PacNoisedDivFunction(DataChunk &args, ExpressionState &state, Vector continue; } uint64_t key_hash = ~uint64_t(0); - if (PacNoiseInNull(key_hash, bind_data.mi, bind_data.correction, gen)) { - result_validity.SetInvalid(i); - continue; - } // Element-wise division: sum[j] / cnt[j] for each of the 64 counters PAC_FLOAT counters[64]; for (idx_t j = 0; j < 64; j++) { @@ -781,6 +776,12 @@ static void PacNoisedDivFunction(DataChunk &args, ExpressionState &state, Vector PAC_FLOAT c = cnt_values[c_idx]; counters[j] = s / c; // 0/0 → NaN, which propagates correctly } + // per-cell generator: independent draws across rows, the same whatever the batch + auto gen = PacCellRng(bind_data.seed, key_hash, counters); + if (PacNoiseInNull(key_hash, bind_data.mi, bind_data.correction, gen)) { + result_validity.SetInvalid(i); + continue; + } double noise_var = 0.0; result_data[i] = static_cast(PacNoisySampleFrom64Counters(counters, bind_data.mi, bind_data.correction, gen, ~key_hash, bind_data.query_hash, diff --git a/src/include/aggregates/as_aggregate.hpp b/src/include/aggregates/as_aggregate.hpp index 1dc10ee9..414e7f9f 100644 --- a/src/include/aggregates/as_aggregate.hpp +++ b/src/include/aggregates/as_aggregate.hpp @@ -263,6 +263,12 @@ bool PacNoiseInNull(uint64_t key_hash, double mi, double correction, std::mt1993 // noise_variance: the PAC noise variance used for this cell (delta = sigma^2 / (2*mi)) bool PacUtilityNull(double noised_value, double noise_variance, double threshold, std::mt19937_64 &gen); +// PacCellRng: the generator of one released cell, seeded by the query's seed and by the cell itself (its world +// mask and its 64 counters). Different cells of a query draw independent NULL decisions and noise, and a cell +// draws the same values whatever the order, thread or batch in which it is finalized. Cells with the same mask +// and counters draw the same values, so averaging their releases does not reduce the noise. +std::mt19937_64 PacCellRng(uint64_t seed, uint64_t key_hash, const PAC_FLOAT counters[64]); + // Minimum total rows across all groups before the diversity check applies // With hash32_32 (hash_repair), single-PU groups always have exactly 32 zero bits // (in the [29,35] suspicious range), so small test datasets need a higher threshold. diff --git a/test/sql/pac_noise_independence.test b/test/sql/pac_noise_independence.test new file mode 100644 index 00000000..1c0dc4fa --- /dev/null +++ b/test/sql/pac_noise_independence.test @@ -0,0 +1,133 @@ +# name: test/sql/pac_noise_independence.test +# description: Every released cell draws its own NULL decision and noise (PacCellRng): the cells of one query are +# independent, and a cell's release does not depend on the order or batch in which it is finalized +# group: [sql] + +require privacy + +statement ok +PRAGMA clear_privacy_metadata; + +statement ok +SET privacy_seed = 42 + +statement ok +SET threads = 1 + +statement ok +SET pac_mi = 0.0078125 + +statement ok +SET pac_ptracking = false + +statement ok +SET pac_sample_diversity_check = false + +# 1000 groups of one entity: each is NULL with probability popcount(~key_hash)/64 = 1/2 +statement ok +CREATE TABLE solo AS SELECT i AS g, i AS e, 1 + i % 5 AS v FROM range(1000) t(i) + +# 1000 groups of 20 entities (no world empty: never NULL), the same data inserted in two orders +statement ok +CREATE TABLE cells AS SELECT i // 20 AS g, i AS e, 1 + i % 7 AS v FROM range(20000) t(i) + +statement ok +CREATE TABLE cells_rev AS SELECT * FROM cells ORDER BY e DESC + +# ============================================================================ +# NULL decisions are independent across groups (not all-or-nothing) +# ============================================================================ + +query I +SELECT count(*) FILTER (WHERE s IS NULL) BETWEEN 400 AND 600 +FROM (SELECT g, as_noised_sum(priv_hash(hash(e)), v) AS s FROM solo GROUP BY g) +---- +true + +query I +SELECT count(*) FILTER (WHERE c IS NULL) BETWEEN 400 AND 600 +FROM (SELECT g, as_noised_count(priv_hash(hash(e))) AS c FROM solo GROUP BY g) +---- +true + +query II +SELECT count(*) FILTER (WHERE mx IS NULL) BETWEEN 400 AND 600, count(*) FILTER (WHERE mn IS NULL) BETWEEN 400 AND 600 +FROM (SELECT g, as_noised_max(priv_hash(hash(e)), v) AS mx, as_noised_min(priv_hash(hash(e)), v) AS mn + FROM solo GROUP BY g) +---- +true true + +query II +SELECT count(*) FILTER (WHERE s IS NULL) BETWEEN 400 AND 600, count(*) FILTER (WHERE mx IS NULL) BETWEEN 400 AND 600 +FROM (SELECT g, as_noised_clip_sum(priv_hash(hash(e)), v) AS s, as_noised_clip_max(priv_hash(hash(e)), v) AS mx + FROM solo GROUP BY g) +---- +true true + +# ============================================================================ +# Noise is independent across groups: released values fall on either side of the exact ones about equally +# often (500 +- 16 of 1000; with one draw shared by all groups, nearly all fall on the same side) +# ============================================================================ + +query II +SELECT count(*) FILTER (WHERE s > x) BETWEEN 400 AND 600, count(*) FILTER (WHERE s < x) BETWEEN 400 AND 600 +FROM (SELECT g, as_noised_sum(priv_hash(hash(e)), v) AS s, sum(v) AS x FROM cells GROUP BY g) +---- +true true + +query II +SELECT count(*) FILTER (WHERE c > x) BETWEEN 400 AND 600, count(*) FILTER (WHERE c < x) BETWEEN 400 AND 600 +FROM (SELECT g, as_noised_count(priv_hash(hash(e))) AS c, count(*) AS x FROM cells GROUP BY g) +---- +true true + +query II +SELECT count(*) FILTER (WHERE s > x) BETWEEN 400 AND 600, count(*) FILTER (WHERE s < x) BETWEEN 400 AND 600 +FROM (SELECT g, as_noised_clip_sum(priv_hash(hash(e)), v) AS s, sum(v) AS x FROM cells GROUP BY g) +---- +true true + +query II +SELECT count(*) FILTER (WHERE s > x) BETWEEN 400 AND 600, count(*) FILTER (WHERE s < x) BETWEEN 400 AND 600 +FROM (SELECT g, as_noised(as_sum(priv_hash(hash(e)), v)) AS s, sum(v) AS x FROM cells GROUP BY g) +---- +true true + +# ============================================================================ +# A cell's release depends on the cell, not on the order or batch in which it is finalized: +# the same groups aggregated from rows inserted in opposite orders release the same values +# ============================================================================ + +query I +SELECT count(*) FROM ( + SELECT g, as_noised_sum(priv_hash(hash(e)), v) AS s, as_noised_count(priv_hash(hash(e))) AS c, + as_noised_max(priv_hash(hash(e)), v) AS mx, as_noised(as_sum(priv_hash(hash(e)), v)) AS l, + as_noised_div(as_sum(priv_hash(hash(e)), v), as_count(priv_hash(hash(e)))) AS d + FROM cells GROUP BY g) a +FULL JOIN ( + SELECT g, as_noised_sum(priv_hash(hash(e)), v) AS s, as_noised_count(priv_hash(hash(e))) AS c, + as_noised_max(priv_hash(hash(e)), v) AS mx, as_noised(as_sum(priv_hash(hash(e)), v)) AS l, + as_noised_div(as_sum(priv_hash(hash(e)), v), as_count(priv_hash(hash(e)))) AS d + FROM cells_rev GROUP BY g) b USING (g) +WHERE a.s IS DISTINCT FROM b.s OR a.c IS DISTINCT FROM b.c OR a.mx IS DISTINCT FROM b.mx + OR a.l IS DISTINCT FROM b.l OR a.d IS DISTINCT FROM b.d +---- +0 + +# the same over more than one vector of groups (4096 groups of 8 entities: as_count, as_noised, as_noised_div and +# priv_finalize used to draw one sequence per vector, so a cell's draws depended on its position in the vector) +query I +SELECT count(*) FROM ( + SELECT e // 8 AS g, as_noised_count(priv_hash(hash(e))) AS c, as_noised_sum(priv_hash(hash(e)), e) AS s, + as_noised(as_sum(priv_hash(hash(e)), e)) AS l, priv_finalize(as_sum(priv_hash(hash(e)), e)) AS f, + as_noised_div(as_sum(priv_hash(hash(e)), e), as_count(priv_hash(hash(e)))) AS d + FROM range(32768) t(e) GROUP BY g) a +FULL JOIN ( + SELECT e // 8 AS g, as_noised_count(priv_hash(hash(e))) AS c, as_noised_sum(priv_hash(hash(e)), e) AS s, + as_noised(as_sum(priv_hash(hash(e)), e)) AS l, priv_finalize(as_sum(priv_hash(hash(e)), e)) AS f, + as_noised_div(as_sum(priv_hash(hash(e)), e), as_count(priv_hash(hash(e)))) AS d + FROM (SELECT * FROM range(32768) t(e) ORDER BY e DESC) GROUP BY g) b USING (g) +WHERE a.c IS DISTINCT FROM b.c OR a.s IS DISTINCT FROM b.s OR a.l IS DISTINCT FROM b.l + OR a.f IS DISTINCT FROM b.f OR a.d IS DISTINCT FROM b.d +---- +0 From e64bdc1e7b37038bb60b02ac33a6083932642b8d Mon Sep 17 00:00:00 2001 From: ila Date: Fri, 30 Oct 2026 10:26:06 +0100 Subject: [PATCH 02/32] Use lightweight independent noise streams per finalization batch Replace per-cell MT initialization and counter hashing with PCG32-backed 64-bit streams. Give each batch a unique stream and advance it across cells. Mark scalar releases volatile and test identical counters across groups and vectors. Draw assignment may vary with execution order. --- src/aggregates/as_aggregate.cpp | 18 ++++----- src/aggregates/as_clip_min_max.cpp | 11 +++-- src/aggregates/as_clip_sum.cpp | 12 +++--- src/aggregates/as_count.cpp | 12 +++--- src/aggregates/as_finalize.cpp | 7 ++-- src/aggregates/as_min_max.cpp | 12 +++--- src/aggregates/as_sum.cpp | 12 +++--- src/categorical/pac_categorical.cpp | 18 ++++----- src/include/aggregates/as_aggregate.hpp | 37 +++++++++++++---- test/sql/pac_noise_independence.test | 53 ++++++++----------------- 10 files changed, 91 insertions(+), 101 deletions(-) diff --git a/src/aggregates/as_aggregate.cpp b/src/aggregates/as_aggregate.cpp index 5fbdabaa..a93bcad5 100644 --- a/src/aggregates/as_aggregate.cpp +++ b/src/aggregates/as_aggregate.cpp @@ -9,6 +9,7 @@ #include "duckdb/planner/expression/bound_function_expression.hpp" #include +#include #ifndef M_PI #define M_PI 3.14159265358979323846 #endif @@ -114,7 +115,7 @@ static double ComputeSecondMomentVariance(const vector &values); // PacNoiseInNull: probabilistically returns true based on bit count in key_hash // mi: controls probabilistic (mi>0) vs deterministic (mi<=0) mode // correction: reduces NULL probability by this factor (considers correction times more non-nulls) -bool PacNoiseInNull(uint64_t key_hash, double mi, double correction, std::mt19937_64 &gen) { +bool PacNoiseInNull(uint64_t key_hash, double mi, double correction, PacRandomEngine &gen) { int popcount = pac_popcount64(key_hash); // Effective popcount considering correction factor (correction times more non-nulls) double effective_popcount = popcount * correction; @@ -133,16 +134,15 @@ bool PacNoiseInNull(uint64_t key_hash, double mi, double correction, std::mt1993 return pac_popcount64(~key_hash) > threshold; } -std::mt19937_64 PacCellRng(uint64_t seed, uint64_t key_hash, const PAC_FLOAT counters[64]) { - hash_t h = CombineHash(Hash(seed), Hash(key_hash)); - h = CombineHash(h, Hash(reinterpret_cast(counters), 64 * sizeof(PAC_FLOAT))); - return std::mt19937_64(h); +PacRandomEngine PacRandomStream(uint64_t seed) { + static std::atomic next_stream {0}; + return PacRandomEngine(seed, next_stream.fetch_add(1, std::memory_order_relaxed)); } // ============================================================================ // Utility NULLing: probabilistically NULL low-SNR cells (post-processing) // ============================================================================ -bool PacUtilityNull(double noised_value, double noise_variance, double threshold, std::mt19937_64 &gen) { +bool PacUtilityNull(double noised_value, double noise_variance, double threshold, PacRandomEngine &gen) { if (std::isnan(threshold)) { return false; // disabled (setting is NULL) } @@ -175,7 +175,7 @@ bool PacUtilityNull(double noised_value, double noise_variance, double threshold // correction: factor to multiply values by after compacting but before noising // pstate: optional p-tracking state for persistent secret composition (Bayesian posterior over worlds) // Returns: correction*yJ + noise where yJ is a randomly selected counter -PAC_FLOAT PacNoisySampleFrom64Counters(const PAC_FLOAT counters[64], double mi, double correction, std::mt19937_64 &gen, +PAC_FLOAT PacNoisySampleFrom64Counters(const PAC_FLOAT counters[64], double mi, double correction, PacRandomEngine &gen, uint64_t is_null, uint64_t counter_selector, const std::shared_ptr &pstate, double *out_noise_variance) { D_ASSERT(~is_null != 0); // at least one bit must be valid @@ -356,9 +356,9 @@ uint64_t PacRepairHash32(uint64_t num) { // k : minimum per-subsample support; cells below it are released as NULL // ============================================================================ struct PacAggregateLocalState : public FunctionLocalState { - explicit PacAggregateLocalState(uint64_t seed) : gen(seed) { + explicit PacAggregateLocalState(uint64_t seed) : gen(PacRandomStream(seed)) { } - std::mt19937_64 gen; + PacRandomEngine gen; }; static unique_ptr PacAggregateInit(ExpressionState &, const BoundFunctionExpression &, diff --git a/src/aggregates/as_clip_min_max.cpp b/src/aggregates/as_clip_min_max.cpp index 5c21bac1..3aa42edc 100644 --- a/src/aggregates/as_clip_min_max.cpp +++ b/src/aggregates/as_clip_min_max.cpp @@ -354,6 +354,7 @@ static void PacClipMinMaxFinalize(Vector &states, AggregateInputData &input, Vec int clip_support = bind.clip_support_threshold; bool clip_scale = bind.clip_scale; + auto gen = PacRandomStream(bind.seed); for (idx_t i = 0; i < count; i++) { PacClipMinMaxFlushBuffer(*state_ptrs[i], *state_ptrs[i], input.allocator); @@ -365,6 +366,10 @@ static void PacClipMinMaxFinalize(Vector &states, AggregateInputData &input, Vec continue; } uint64_t key_hash = (pos ? pos->key_hash : 0) | (neg ? neg->key_hash : 0); + if (PacNoiseInNull(key_hash, mi, correction, gen)) { + result_mask.SetInvalid(offset + i); + continue; + } uint64_t update_count = 0; if (pos) { pos->GetTotals(buf, clip_support, clip_scale); @@ -387,12 +392,6 @@ static void PacClipMinMaxFinalize(Vector &states, AggregateInputData &input, Vec } update_count += neg->update_count; } - // per-cell generator: independent draws across groups, the same whatever the processing order - auto gen = PacCellRng(bind.seed, key_hash, buf); - if (PacNoiseInNull(key_hash, mi, correction, gen)) { - result_mask.SetInvalid(offset + i); - continue; - } CheckPacSampleDiversity(key_hash, buf, update_count, IS_MAX ? "as_noised_clip_max" : "as_noised_clip_min", bind); double noise_var = 0.0; diff --git a/src/aggregates/as_clip_sum.cpp b/src/aggregates/as_clip_sum.cpp index 67e8f6ae..d450bf5a 100644 --- a/src/aggregates/as_clip_sum.cpp +++ b/src/aggregates/as_clip_sum.cpp @@ -557,6 +557,7 @@ static void PacClipSumFinalize(Vector &states, AggregateInputData &input, Vector int clip_support = bind.clip_support_threshold; bool clip_scale = bind.clip_scale; + auto gen = PacRandomStream(bind.seed); for (idx_t i = 0; i < count; i++) { PacClipSumFlushBuffer(*state_ptrs[i], *state_ptrs[i], input.allocator, bind.sample_lanes); @@ -567,7 +568,10 @@ static void PacClipSumFinalize(Vector &states, AggregateInputData &input, Vector continue; } uint64_t key_hash = pos->key_hash; - uint64_t null_key_hash = key_hash; // the NULL decision considers the worlds of the positive state + if (PacNoiseInNull(key_hash, mi, correction, gen)) { + result_mask.SetInvalid(offset + i); + continue; + } // Non-mutating: just read totals with clip_support filtering pos->GetTotals(buf, clip_support, clip_scale); @@ -584,12 +588,6 @@ static void PacClipSumFinalize(Vector &states, AggregateInputData &input, Vector } update_count += neg->update_count; } - // per-cell generator: independent draws across groups, the same whatever the processing order - auto gen = PacCellRng(bind.seed, key_hash, buf); - if (PacNoiseInNull(null_key_hash, mi, correction, gen)) { - result_mask.SetInvalid(offset + i); - continue; - } CheckPacSampleDiversity(key_hash, buf, update_count, "as_clip_sum", bind); double noise_var = 0.0; diff --git a/src/aggregates/as_count.cpp b/src/aggregates/as_count.cpp index a9d0aed6..a0b4e3ae 100644 --- a/src/aggregates/as_count.cpp +++ b/src/aggregates/as_count.cpp @@ -227,24 +227,22 @@ void PacCountFinalize(Vector &states, AggregateInputData &input, Vector &result, uint64_t query_hash = bind.query_hash; auto pstate = bind.pstate; PAC_FLOAT buf[64]; + auto gen = PacRandomStream(bind.seed); for (idx_t i = 0; i < count; i++) { #if !defined(PAC_NOBUFFERING) && !defined(PAC_NOCASCADING) aggs[i]->FlushBuffer(*aggs[i], input.allocator); // flush values into yourself #endif PacCountState *s = aggs[i]->GetState(); uint64_t key_hash = s ? s->key_hash : 0; + if (PacNoiseInNull(key_hash, mi, correction, gen)) { + result_mask.SetInvalid(offset + i); + continue; + } if (s) { s->GetTotalsWithSWAR(buf); } else { memset(buf, 0, sizeof(buf)); } - // per-cell generator: independent draws across groups, the same whatever the processing order - auto gen = PacCellRng(bind.seed, key_hash, buf); - // Check if we should return NULL based on key_hash (uses mi and correction) - if (PacNoiseInNull(key_hash, mi, correction, gen)) { - result_mask.SetInvalid(offset + i); - continue; - } CheckPacSampleDiversity(key_hash, buf, s ? s->GetUpdateCount() : 0, "as_noised_count", input.bind_data->Cast()); // Multiply by 2 to compensate for 50% sampling, then apply correction diff --git a/src/aggregates/as_finalize.cpp b/src/aggregates/as_finalize.cpp index cedf3709..d3f3c7c5 100644 --- a/src/aggregates/as_finalize.cpp +++ b/src/aggregates/as_finalize.cpp @@ -26,6 +26,7 @@ static void PacFinalizeFunction(DataChunk &args, ExpressionState &state, Vector seed ^= PAC_MAGIC_HASH * static_cast(context.ActiveTransaction().GetActiveQuery()); } uint64_t query_hash = (seed * PAC_MAGIC_HASH) ^ PAC_MAGIC_HASH; + auto gen = PacRandomStream(seed); auto &list_vec = args.data[0]; idx_t count = args.size(); @@ -63,9 +64,6 @@ static void PacFinalizeFunction(DataChunk &args, ExpressionState &state, Vector counters[j] = child_data.validity.RowIsValid(child_idx) ? child_values[child_idx] : 0; } - // per-cell generator: independent draws across rows, the same whatever the batch - auto gen = PacCellRng(seed, ~0ULL, counters); - result_data[i] = PacNoisySampleFrom64Counters(counters, mi, 1.0, gen, 0ULL, query_hash, nullptr); } } @@ -74,6 +72,7 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader) { auto list_type = LogicalType::LIST(PacFloatLogicalType()); auto scalar_type = PacFloatLogicalType(); ScalarFunction priv_finalize("priv_finalize", {list_type}, scalar_type, PacFinalizeFunction); + priv_finalize.stability = FunctionStability::VOLATILE; loader.RegisterFunction(priv_finalize); // Register implicit casts LIST → FLOAT and LIST → DOUBLE so the binder @@ -83,6 +82,7 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader) { double mi = 0.0; uint64_t seed = 42; uint64_t query_hash = (seed * PAC_MAGIC_HASH) ^ PAC_MAGIC_HASH; + auto gen = PacRandomStream(seed); UnifiedVectorFormat list_data; source.ToUnifiedFormat(count, list_data); @@ -113,7 +113,6 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader) { auto child_idx = child_data.sel->get_index(entry.offset + j); counters[j] = child_data.validity.RowIsValid(child_idx) ? child_values[child_idx] : 0; } - auto gen = PacCellRng(seed, ~0ULL, counters); tmp[i] = PacNoisySampleFrom64Counters(counters, mi, 1.0, gen, 0ULL, query_hash, nullptr); } diff --git a/src/aggregates/as_min_max.cpp b/src/aggregates/as_min_max.cpp index 493b0926..c0926409 100644 --- a/src/aggregates/as_min_max.cpp +++ b/src/aggregates/as_min_max.cpp @@ -104,6 +104,7 @@ static void PacMinMaxFinalize(Vector &states, AggregateInputData &input, Vector double correction = input.bind_data ? input.bind_data->Cast().correction : 1.0; uint64_t query_hash = input.bind_data ? input.bind_data->Cast().query_hash : 0; auto pstate = input.bind_data ? input.bind_data->Cast().pstate : nullptr; + auto gen = PacRandomStream(input.bind_data ? input.bind_data->Cast().seed : 0); for (idx_t i = 0; i < count; i++) { #ifndef PAC_NOBUFFERING @@ -113,19 +114,16 @@ static void PacMinMaxFinalize(Vector &states, AggregateInputData &input, Vector auto *s = state_ptrs[i]->GetState(); // Check if we should return NULL based on key_hash uint64_t key_hash = s ? s->key_hash : 0; + if (PacNoiseInNull(key_hash, mi, correction, gen)) { + result_mask.SetInvalid(offset + i); + continue; + } PAC_FLOAT buf[64]; if (s && s->initialized) { s->GetTotals(buf); } else { memset(buf, 0, sizeof(buf)); } - // per-cell generator: independent draws across groups, the same whatever the processing order - auto gen = PacCellRng(input.bind_data->Cast().seed, key_hash, buf); - // mi controls noise mode, correction reduces NULL probability (but does NOT scale min/max values) - if (PacNoiseInNull(key_hash, mi, correction, gen)) { - result_mask.SetInvalid(offset + i); - continue; - } CheckPacSampleDiversity(key_hash, buf, s ? s->update_count : 0, IS_MAX ? "as_noised_max" : "as_noised_min", input.bind_data->Cast()); // Pass mi for noise, 1.0 as correction (no value scaling for min/max) diff --git a/src/aggregates/as_sum.cpp b/src/aggregates/as_sum.cpp index 6d4aff87..07adc910 100644 --- a/src/aggregates/as_sum.cpp +++ b/src/aggregates/as_sum.cpp @@ -434,6 +434,7 @@ void PacSumFinalize(Vector &states, AggregateInputData &input, Vector &result, i double correction = input.bind_data ? input.bind_data->Cast().correction : 1.0; uint64_t query_hash = input.bind_data ? input.bind_data->Cast().query_hash : 0; auto pstate = input.bind_data ? input.bind_data->Cast().pstate : nullptr; + auto gen = PacRandomStream(input.bind_data ? input.bind_data->Cast().seed : 0); for (idx_t i = 0; i < count; i++) { #ifndef PAC_NOBUFFERING @@ -446,7 +447,10 @@ void PacSumFinalize(Vector &states, AggregateInputData &input, Vector &result, i continue; } uint64_t key_hash = pos->key_hash; - uint64_t null_key_hash = key_hash; // the NULL decision considers the worlds of the positive state + if (PacNoiseInNull(key_hash, mi, correction, gen)) { + result_mask.SetInvalid(offset + i); + continue; + } #ifndef PAC_SIGNEDSUM // Double-sided mode: use SFINAE helpers (cast to unsigned for int states only) FlushPosState(pos, input.allocator); @@ -458,12 +462,6 @@ void PacSumFinalize(Vector &states, AggregateInputData &input, Vector &result, i pos->GetTotals(buf); uint64_t update_count = pos->update_count; #endif - // per-cell generator: independent draws across groups, the same whatever the processing order - auto gen = PacCellRng(input.bind_data->Cast().seed, key_hash, buf); - if (PacNoiseInNull(null_key_hash, mi, correction, gen)) { - result_mask.SetInvalid(offset + i); // return NULL (probabilistic decision) - continue; - } CheckPacSampleDiversity(key_hash, buf, update_count, "as_noised_sum", input.bind_data->Cast()); double noise_var = 0.0; PAC_FLOAT result_val = diff --git a/src/categorical/pac_categorical.cpp b/src/categorical/pac_categorical.cpp index 2aefca1d..b27f1a3a 100644 --- a/src/categorical/pac_categorical.cpp +++ b/src/categorical/pac_categorical.cpp @@ -102,8 +102,8 @@ struct PacCategoricalBindData : public FunctionData { // Local state for PAC categorical functions (RNG) // ============================================================================ struct PacCategoricalLocalState : public FunctionLocalState { - std::mt19937_64 gen; - explicit PacCategoricalLocalState(uint64_t seed) : gen(seed) { + PacRandomEngine gen; + explicit PacCategoricalLocalState(uint64_t seed) : gen(PacRandomStream(seed)) { } }; @@ -141,7 +141,7 @@ static inline uint64_t BoolListToMask(UnifiedVectorFormat &list_data, UnifiedVec // Helper: Common filter logic for mask-based filtering // ============================================================================ static inline bool FilterFromCount(idx_t true_count, idx_t lane_count, double mi, double correction, - std::mt19937_64 &gen) { + PacRandomEngine &gen) { if (lane_count == 0) { return false; } @@ -168,7 +168,7 @@ static inline bool FilterFromCount(idx_t true_count, idx_t lane_count, double mi // correction: considers correction times more non-nulls (increases true probability) // When mi <= 0: deterministic, true when popcount(mask) * correction >= 32 // When mi > 0: probabilistic, P(true) = popcount(mask) * correction / 64 -static inline bool FilterFromMask(uint64_t mask, double mi, double correction, std::mt19937_64 &gen) { +static inline bool FilterFromMask(uint64_t mask, double mi, double correction, PacRandomEngine &gen) { int popcount = Popcount64(mask); return FilterFromCount(popcount, 64, mi, correction, gen); } @@ -223,7 +223,7 @@ static void PacSelectFunction(DataChunk &args, ExpressionState &state, Vector &r struct PacFilterParams { double mi; double correction; - std::mt19937_64 &gen; + PacRandomEngine &gen; }; static PacFilterParams GetPacFilterParams(ExpressionState &state) { @@ -586,6 +586,7 @@ static void PacNoisedFunction(DataChunk &args, ExpressionState &state, Vector &r auto &function = state.expr.Cast(); auto &bind_data = function.bind_info->Cast(); + auto gen = PacRandomStream(bind_data.seed); UnifiedVectorFormat list_data; list_vec.ToUnifiedFormat(count, list_data); @@ -623,8 +624,6 @@ static void PacNoisedFunction(DataChunk &args, ExpressionState &state, Vector &r auto child_idx = child_data.sel->get_index(entry.offset + j); counters[j] = child_values[child_idx]; } - // per-cell generator: independent draws across rows, the same whatever the batch - auto gen = PacCellRng(bind_data.seed, key_hash, counters); // Check if we should return NULL based on key_hash (uses mi and correction) if (PacNoiseInNull(key_hash, bind_data.mi, bind_data.correction, gen)) { result_validity.SetInvalid(i); @@ -733,6 +732,7 @@ static void PacNoisedDivFunction(DataChunk &args, ExpressionState &state, Vector auto &function = state.expr.Cast(); auto &bind_data = function.bind_info->Cast(); + auto gen = PacRandomStream(bind_data.seed); UnifiedVectorFormat sum_data, cnt_data; sum_vec.ToUnifiedFormat(count, sum_data); @@ -776,8 +776,6 @@ static void PacNoisedDivFunction(DataChunk &args, ExpressionState &state, Vector PAC_FLOAT c = cnt_values[c_idx]; counters[j] = s / c; // 0/0 → NaN, which propagates correctly } - // per-cell generator: independent draws across rows, the same whatever the batch - auto gen = PacCellRng(bind_data.seed, key_hash, counters); if (PacNoiseInNull(key_hash, bind_data.mi, bind_data.correction, gen)) { result_validity.SetInvalid(i); continue; @@ -1217,6 +1215,7 @@ void RegisterPacCategoricalFunctions(ExtensionLoader &loader) { // as_noised(list) -> PAC_FLOAT ScalarFunction as_noised("as_noised", {list_double_type}, PacFloatLogicalType(), PacNoisedFunction, PacCategoricalBind, nullptr, nullptr, PacCategoricalInitLocal); + as_noised.stability = FunctionStability::VOLATILE; loader.RegisterFunction( make_scalar_info(as_noised, "[INTERNAL] Applies PAC noise to 64-element counter list, returns scalar.")); @@ -1227,6 +1226,7 @@ void RegisterPacCategoricalFunctions(ExtensionLoader &loader) { // as_noised_div(list sum, list cnt) -> DOUBLE ScalarFunction as_noised_div("as_noised_div", {list_double_type, list_double_type}, LogicalType::DOUBLE, PacNoisedDivFunction, PacCategoricalBind, nullptr, nullptr, PacCategoricalInitLocal); + as_noised_div.stability = FunctionStability::VOLATILE; loader.RegisterFunction(make_scalar_info(as_noised_div, "[INTERNAL] Fused counter-list division + noise for AVG.")); // priv_filter_: optimized comparison + filter in a single pass diff --git a/src/include/aggregates/as_aggregate.hpp b/src/include/aggregates/as_aggregate.hpp index 414e7f9f..119420b6 100644 --- a/src/include/aggregates/as_aggregate.hpp +++ b/src/include/aggregates/as_aggregate.hpp @@ -5,6 +5,7 @@ #include "duckdb/common/random_engine.hpp" #include "duckdb/execution/expression_executor.hpp" #include "duckdb/transaction/meta_transaction.hpp" +#include "pcg_random.hpp" #include #include #include @@ -184,6 +185,27 @@ static inline int pac_ctzll(uint64_t x) { namespace duckdb { +// DuckDB's PCG32 core, exposed as a 64-bit URBG for the standard distributions. +class PacRandomEngine { +public: + using result_type = uint64_t; + explicit PacRandomEngine(uint64_t seed, uint64_t stream) : engine(seed, stream) { + } + static constexpr result_type min() { + return 0; + } + static constexpr result_type max() { + return ~result_type(0); + } + result_type operator()() { + const auto high = engine(); + return (result_type(high) << 32) | engine(); + } + +private: + pcg32 engine; +}; + // Returns the DuckDB LogicalType corresponding to PAC_FLOAT (FLOAT or DOUBLE). static inline LogicalType PacFloatLogicalType() { return std::is_same::value ? LogicalType::FLOAT : LogicalType::DOUBLE; @@ -237,7 +259,7 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader); // correction: factor to multiply values by after compacting NULLs but before adding noise // pstate: optional shared p-tracking state for persistent secret composition (may be nullptr) // out_noise_variance: if non-null, receives the noise variance used (for utility NULLing) -PAC_FLOAT PacNoisySampleFrom64Counters(const PAC_FLOAT counters[64], double mi, double correction, std::mt19937_64 &gen, +PAC_FLOAT PacNoisySampleFrom64Counters(const PAC_FLOAT counters[64], double mi, double correction, PacRandomEngine &gen, uint64_t is_null = 0, uint64_t counter_selector = 0, const std::shared_ptr &pstate = nullptr, double *out_noise_variance = nullptr); @@ -253,7 +275,7 @@ static inline bool PacNoisedSelect(uint64_t key_hash, uint64_t rnd) { // correction: reduces NULL probability by this factor (considers correction times more non-nulls) // Probabilistic: P(NULL) = popcount(~key_hash) / (64 * correction) // Deterministic: NULL when popcount(key_hash) * correction < 1 -bool PacNoiseInNull(uint64_t key_hash, double mi, double correction, std::mt19937_64 &gen); +bool PacNoiseInNull(uint64_t key_hash, double mi, double correction, PacRandomEngine &gen); // PacUtilityNull: post-processing NULLing of low-SNR cells. // Returns true (NULL the cell) probabilistically based on z-score = |noised_value| / noise_std_dev. @@ -261,13 +283,12 @@ bool PacNoiseInNull(uint64_t key_hash, double mi, double correction, std::mt1993 // Safe by data processing inequality: this is a function of the already-noised output + independent coins. // threshold: z-score center (e.g., 4.0 means ~20% expected relative error cutoff). NaN = disabled. // noise_variance: the PAC noise variance used for this cell (delta = sigma^2 / (2*mi)) -bool PacUtilityNull(double noised_value, double noise_variance, double threshold, std::mt19937_64 &gen); +bool PacUtilityNull(double noised_value, double noise_variance, double threshold, PacRandomEngine &gen); -// PacCellRng: the generator of one released cell, seeded by the query's seed and by the cell itself (its world -// mask and its 64 counters). Different cells of a query draw independent NULL decisions and noise, and a cell -// draws the same values whatever the order, thread or batch in which it is finalized. Cells with the same mask -// and counters draw the same values, so averaging their releases does not reduce the noise. -std::mt19937_64 PacCellRng(uint64_t seed, uint64_t key_hash, const PAC_FLOAT counters[64]); +// One lightweight stream per finalization batch, advanced across its cells. +// Stream IDs distinguish batches and workers without hashing counters or adding per-group state. +// Which draw a cell receives depends on execution order; bit-identical per-cell releases are not promised. +PacRandomEngine PacRandomStream(uint64_t seed); // Minimum total rows across all groups before the diversity check applies // With hash32_32 (hash_repair), single-PU groups always have exactly 32 zero bits diff --git a/test/sql/pac_noise_independence.test b/test/sql/pac_noise_independence.test index 1c0dc4fa..2ec63d02 100644 --- a/test/sql/pac_noise_independence.test +++ b/test/sql/pac_noise_independence.test @@ -1,6 +1,5 @@ # name: test/sql/pac_noise_independence.test -# description: Every released cell draws its own NULL decision and noise (PacCellRng): the cells of one query are -# independent, and a cell's release does not depend on the order or batch in which it is finalized +# description: Independent noise draws across groups and finalization batches # group: [sql] require privacy @@ -27,13 +26,10 @@ SET pac_sample_diversity_check = false statement ok CREATE TABLE solo AS SELECT i AS g, i AS e, 1 + i % 5 AS v FROM range(1000) t(i) -# 1000 groups of 20 entities (no world empty: never NULL), the same data inserted in two orders +# 1000 groups of 20 entities (no world empty: never NULL) statement ok CREATE TABLE cells AS SELECT i // 20 AS g, i AS e, 1 + i % 7 AS v FROM range(20000) t(i) -statement ok -CREATE TABLE cells_rev AS SELECT * FROM cells ORDER BY e DESC - # ============================================================================ # NULL decisions are independent across groups (not all-or-nothing) # ============================================================================ @@ -94,40 +90,23 @@ FROM (SELECT g, as_noised(as_sum(priv_hash(hash(e)), v)) AS s, sum(v) AS x FROM true true # ============================================================================ -# A cell's release depends on the cell, not on the order or batch in which it is finalized: -# the same groups aggregated from rows inserted in opposite orders release the same values +# Equal counter values in different groups still receive independent draws. +# Reinitializing the generator from counters would make all 1000 NULL decisions identical. # ============================================================================ query I -SELECT count(*) FROM ( - SELECT g, as_noised_sum(priv_hash(hash(e)), v) AS s, as_noised_count(priv_hash(hash(e))) AS c, - as_noised_max(priv_hash(hash(e)), v) AS mx, as_noised(as_sum(priv_hash(hash(e)), v)) AS l, - as_noised_div(as_sum(priv_hash(hash(e)), v), as_count(priv_hash(hash(e)))) AS d - FROM cells GROUP BY g) a -FULL JOIN ( - SELECT g, as_noised_sum(priv_hash(hash(e)), v) AS s, as_noised_count(priv_hash(hash(e))) AS c, - as_noised_max(priv_hash(hash(e)), v) AS mx, as_noised(as_sum(priv_hash(hash(e)), v)) AS l, - as_noised_div(as_sum(priv_hash(hash(e)), v), as_count(priv_hash(hash(e)))) AS d - FROM cells_rev GROUP BY g) b USING (g) -WHERE a.s IS DISTINCT FROM b.s OR a.c IS DISTINCT FROM b.c OR a.mx IS DISTINCT FROM b.mx - OR a.l IS DISTINCT FROM b.l OR a.d IS DISTINCT FROM b.d +SELECT count(*) FILTER (WHERE c IS NULL) BETWEEN 400 AND 600 +FROM (SELECT i, as_noised_count(4294967295::UBIGINT) AS c FROM range(1000) t(i) GROUP BY i) ---- -0 +true -# the same over more than one vector of groups (4096 groups of 8 entities: as_count, as_noised, as_noised_div and -# priv_finalize used to draw one sequence per vector, so a cell's draws depended on its position in the vector) -query I -SELECT count(*) FROM ( - SELECT e // 8 AS g, as_noised_count(priv_hash(hash(e))) AS c, as_noised_sum(priv_hash(hash(e)), e) AS s, - as_noised(as_sum(priv_hash(hash(e)), e)) AS l, priv_finalize(as_sum(priv_hash(hash(e)), e)) AS f, - as_noised_div(as_sum(priv_hash(hash(e)), e), as_count(priv_hash(hash(e)))) AS d - FROM range(32768) t(e) GROUP BY g) a -FULL JOIN ( - SELECT e // 8 AS g, as_noised_count(priv_hash(hash(e))) AS c, as_noised_sum(priv_hash(hash(e)), e) AS s, - as_noised(as_sum(priv_hash(hash(e)), e)) AS l, priv_finalize(as_sum(priv_hash(hash(e)), e)) AS f, - as_noised_div(as_sum(priv_hash(hash(e)), e), as_count(priv_hash(hash(e)))) AS d - FROM (SELECT * FROM range(32768) t(e) ORDER BY e DESC) GROUP BY g) b USING (g) -WHERE a.c IS DISTINCT FROM b.c OR a.s IS DISTINCT FROM b.s OR a.l IS DISTINCT FROM b.l - OR a.f IS DISTINCT FROM b.f OR a.d IS DISTINCT FROM b.d +# Constant counter arrays across several vectors must not be constant-folded, +# or restart the same RNG stream in each vector. Check all scalar release forms. +query III +SELECT count(DISTINCT l) > 3000, count(DISTINCT f) > 3000, count(DISTINCT d) > 3000 +FROM (SELECT as_noised(range(64)::FLOAT[]) AS l, + priv_finalize(range(64)::FLOAT[]) AS f, + as_noised_div(range(64)::FLOAT[], list_transform(range(64), x -> 1::FLOAT)) AS d + FROM range(4096)) t ---- -0 +true true true From 29c8a3c33ea236e417aec4b2804bf653e16e01af Mon Sep 17 00:00:00 2001 From: ila Date: Fri, 30 Oct 2026 10:48:02 +0100 Subject: [PATCH 03/32] Reset PAC noise streams per query for seeded reproducibility --- README.md | 13 ++++ docs/syntax.md | 4 ++ src/aggregates/as_aggregate.cpp | 32 +++++++-- src/aggregates/as_clip_min_max.cpp | 2 +- src/aggregates/as_clip_sum.cpp | 2 +- src/aggregates/as_count.cpp | 2 +- src/aggregates/as_finalize.cpp | 12 +--- src/aggregates/as_min_max.cpp | 4 +- src/aggregates/as_sum.cpp | 4 +- src/categorical/pac_categorical.cpp | 24 +++---- src/include/aggregates/as_aggregate.hpp | 29 ++++----- test/sql/pac_noise_independence.test | 87 +++++++++++++++++++++++++ 12 files changed, 163 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 5a03c8b9..aee7ff98 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,19 @@ This gives a formal (ε,δ)-DP guarantee at the **user level** — neighboring d | `dp_delta` | `1e-6` | dp_elastic | Failure probability δ (0 = pure DP) | | `dp_sum_bound` | required | dp_elastic | Clipping bound for `SUM`/`AVG` inputs: values are clipped to `[-bound, bound]` | +For PAC, a fixed `privacy_seed` repeats the random draws when the query, data, +settings, execution order, and initial p-tracking state are unchanged on the same +build. Stream allocation resets for every query, including prepared executions, +so unrelated queries do not shift later seeded draws. Streams advance across cells: +identical aggregate counters in different groups still receive independent draws. + +For reproducible tests, use `SET threads = 1` and a stable input order. Parallel +hash aggregation can emit groups in a different order even at a fixed thread count, +so bit-identical per-group results are not guaranteed across parallel executions, +row/batch reordering, or builds. P-tracking updates its calibration state after +releases; comparisons must start from the same state (the RNG regression tests use +`SET pac_ptracking = false`). + ## SQL Reference ### Supported Aggregates and Operators diff --git a/docs/syntax.md b/docs/syntax.md index 7bd9e989..bf3f3494 100644 --- a/docs/syntax.md +++ b/docs/syntax.md @@ -158,3 +158,7 @@ There is some preliminary support for using multiple PU tables inside the same s | `privacy_diffcols` | `NULL` | [Utility diff](pac/utility.md): compare noised vs exact results | | `privacy_min_group_count` | `NULL` (disabled) | Admin-set minimum support threshold for DP groups. In PAC aggregates, this remains the z-score threshold for [utility NULLing](pac/runtime_checks.md#utility-nulling). | | `priv_clip_support` | `NULL` (disabled) | Enable clipping: aggregate per-PU values with outlier elimination at the given support threshold | + +PAC seeded reproducibility requires stable execution order; use `SET threads = 1` +for repeatable tests. Parallel group order and p-tracking state can change results; +see the [seed reproducibility contract](../README.md#settings). diff --git a/src/aggregates/as_aggregate.cpp b/src/aggregates/as_aggregate.cpp index a93bcad5..debd56a1 100644 --- a/src/aggregates/as_aggregate.cpp +++ b/src/aggregates/as_aggregate.cpp @@ -7,6 +7,7 @@ #include "duckdb/common/exception.hpp" #include "duckdb/function/aggregate_function.hpp" #include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/main/client_context_state.hpp" #include #include @@ -134,9 +135,28 @@ bool PacNoiseInNull(uint64_t key_hash, double mi, double correction, PacRandomEn return pac_popcount64(~key_hash) > threshold; } -PacRandomEngine PacRandomStream(uint64_t seed) { - static std::atomic next_stream {0}; - return PacRandomEngine(seed, next_stream.fetch_add(1, std::memory_order_relaxed)); +struct PacRandomState : public ClientContextState { + std::atomic next_stream {0}; + + void QueryBegin(ClientContext &) override { + next_stream.store(0, std::memory_order_relaxed); + } +}; + +shared_ptr GetPacRandomState(ClientContext &context) { + return context.registered_state->GetOrCreate("privacy_random_streams"); +} + +uint64_t GetPacSeed(ClientContext &context, double mi) { + Value value; + if (context.TryGetCurrentSetting("privacy_seed", value) && !value.IsNull()) { + return uint64_t(value.GetValue()); + } + return mi == 0.0 ? 42 : 42 ^ (PAC_MAGIC_HASH * uint64_t(context.ActiveTransaction().GetActiveQuery())); +} + +PacRandomEngine PacRandomStream(uint64_t seed, PacRandomState &state) { + return PacRandomEngine(seed, state.next_stream.fetch_add(1, std::memory_order_relaxed)); } // ============================================================================ @@ -356,19 +376,19 @@ uint64_t PacRepairHash32(uint64_t num) { // k : minimum per-subsample support; cells below it are released as NULL // ============================================================================ struct PacAggregateLocalState : public FunctionLocalState { - explicit PacAggregateLocalState(uint64_t seed) : gen(PacRandomStream(seed)) { + explicit PacAggregateLocalState(PacRandomEngine gen_p) : gen(std::move(gen_p)) { } PacRandomEngine gen; }; -static unique_ptr PacAggregateInit(ExpressionState &, const BoundFunctionExpression &, +static unique_ptr PacAggregateInit(ExpressionState &state, const BoundFunctionExpression &, FunctionData *bind_data) { // Single source-of-truth: prefer seed supplied via bind_data (PrivBindData). uint64_t seed = std::random_device {}(); if (bind_data) { seed = bind_data->Cast().seed; } - return make_uniq(seed); + return make_uniq(PacRandomStream(seed, *GetPacRandomState(state.GetContext()))); } // Templated over values element type T and counts element type C so we can accept diff --git a/src/aggregates/as_clip_min_max.cpp b/src/aggregates/as_clip_min_max.cpp index 3aa42edc..3579238f 100644 --- a/src/aggregates/as_clip_min_max.cpp +++ b/src/aggregates/as_clip_min_max.cpp @@ -354,7 +354,7 @@ static void PacClipMinMaxFinalize(Vector &states, AggregateInputData &input, Vec int clip_support = bind.clip_support_threshold; bool clip_scale = bind.clip_scale; - auto gen = PacRandomStream(bind.seed); + auto gen = PacRandomStream(bind.seed, *bind.random_state); for (idx_t i = 0; i < count; i++) { PacClipMinMaxFlushBuffer(*state_ptrs[i], *state_ptrs[i], input.allocator); diff --git a/src/aggregates/as_clip_sum.cpp b/src/aggregates/as_clip_sum.cpp index d450bf5a..acd891e1 100644 --- a/src/aggregates/as_clip_sum.cpp +++ b/src/aggregates/as_clip_sum.cpp @@ -557,7 +557,7 @@ static void PacClipSumFinalize(Vector &states, AggregateInputData &input, Vector int clip_support = bind.clip_support_threshold; bool clip_scale = bind.clip_scale; - auto gen = PacRandomStream(bind.seed); + auto gen = PacRandomStream(bind.seed, *bind.random_state); for (idx_t i = 0; i < count; i++) { PacClipSumFlushBuffer(*state_ptrs[i], *state_ptrs[i], input.allocator, bind.sample_lanes); diff --git a/src/aggregates/as_count.cpp b/src/aggregates/as_count.cpp index a0b4e3ae..c07b5b07 100644 --- a/src/aggregates/as_count.cpp +++ b/src/aggregates/as_count.cpp @@ -227,7 +227,7 @@ void PacCountFinalize(Vector &states, AggregateInputData &input, Vector &result, uint64_t query_hash = bind.query_hash; auto pstate = bind.pstate; PAC_FLOAT buf[64]; - auto gen = PacRandomStream(bind.seed); + auto gen = PacRandomStream(bind.seed, *bind.random_state); for (idx_t i = 0; i < count; i++) { #if !defined(PAC_NOBUFFERING) && !defined(PAC_NOCASCADING) aggs[i]->FlushBuffer(*aggs[i], input.allocator); // flush values into yourself diff --git a/src/aggregates/as_finalize.cpp b/src/aggregates/as_finalize.cpp index d3f3c7c5..e8e88fb7 100644 --- a/src/aggregates/as_finalize.cpp +++ b/src/aggregates/as_finalize.cpp @@ -18,15 +18,9 @@ static void PacFinalizeFunction(DataChunk &args, ExpressionState &state, Vector auto &context = state.GetContext(); double mi = GetPacMiFromSetting(context); - Value pac_seed_val; - uint64_t seed = (context.TryGetCurrentSetting("privacy_seed", pac_seed_val) && !pac_seed_val.IsNull()) - ? uint64_t(pac_seed_val.GetValue()) - : 42; - if (mi != 0.0) { - seed ^= PAC_MAGIC_HASH * static_cast(context.ActiveTransaction().GetActiveQuery()); - } + uint64_t seed = GetPacSeed(context, mi); uint64_t query_hash = (seed * PAC_MAGIC_HASH) ^ PAC_MAGIC_HASH; - auto gen = PacRandomStream(seed); + auto gen = PacRandomStream(seed, *GetPacRandomState(context)); auto &list_vec = args.data[0]; idx_t count = args.size(); @@ -82,7 +76,7 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader) { double mi = 0.0; uint64_t seed = 42; uint64_t query_hash = (seed * PAC_MAGIC_HASH) ^ PAC_MAGIC_HASH; - auto gen = PacRandomStream(seed); + PacRandomEngine gen(seed, 0); UnifiedVectorFormat list_data; source.ToUnifiedFormat(count, list_data); diff --git a/src/aggregates/as_min_max.cpp b/src/aggregates/as_min_max.cpp index c0926409..a8c073a5 100644 --- a/src/aggregates/as_min_max.cpp +++ b/src/aggregates/as_min_max.cpp @@ -104,7 +104,9 @@ static void PacMinMaxFinalize(Vector &states, AggregateInputData &input, Vector double correction = input.bind_data ? input.bind_data->Cast().correction : 1.0; uint64_t query_hash = input.bind_data ? input.bind_data->Cast().query_hash : 0; auto pstate = input.bind_data ? input.bind_data->Cast().pstate : nullptr; - auto gen = PacRandomStream(input.bind_data ? input.bind_data->Cast().seed : 0); + auto gen = input.bind_data ? PacRandomStream(input.bind_data->Cast().seed, + *input.bind_data->Cast().random_state) + : PacRandomEngine(0, 0); for (idx_t i = 0; i < count; i++) { #ifndef PAC_NOBUFFERING diff --git a/src/aggregates/as_sum.cpp b/src/aggregates/as_sum.cpp index 07adc910..a0633cd4 100644 --- a/src/aggregates/as_sum.cpp +++ b/src/aggregates/as_sum.cpp @@ -434,7 +434,9 @@ void PacSumFinalize(Vector &states, AggregateInputData &input, Vector &result, i double correction = input.bind_data ? input.bind_data->Cast().correction : 1.0; uint64_t query_hash = input.bind_data ? input.bind_data->Cast().query_hash : 0; auto pstate = input.bind_data ? input.bind_data->Cast().pstate : nullptr; - auto gen = PacRandomStream(input.bind_data ? input.bind_data->Cast().seed : 0); + auto gen = input.bind_data ? PacRandomStream(input.bind_data->Cast().seed, + *input.bind_data->Cast().random_state) + : PacRandomEngine(0, 0); for (idx_t i = 0; i < count; i++) { #ifndef PAC_NOBUFFERING diff --git a/src/categorical/pac_categorical.cpp b/src/categorical/pac_categorical.cpp index b27f1a3a..0ae56f10 100644 --- a/src/categorical/pac_categorical.cpp +++ b/src/categorical/pac_categorical.cpp @@ -54,25 +54,19 @@ struct PacCategoricalBindData : public FunctionData { uint64_t query_hash; // derived from seed: used as counter selector for NoisySample std::shared_ptr pstate; // p-tracking state shared across query (may be nullptr) double utility_threshold; // z-score threshold for utility NULLing (NaN = disabled) + shared_ptr random_state; // Primary constructor - reads seed from privacy_seed setting, or uses default 42 if not set. - // When mi > 0, seed is randomized per query via the query number. + // Without an explicit seed, mi > 0 mixes in the query number. explicit PacCategoricalBindData(ClientContext &ctx, double mi_val = 0.0, double correction_val = 1.0) - : mi(mi_val), correction(correction_val), utility_threshold(std::numeric_limits::quiet_NaN()) { + : mi(mi_val), correction(correction_val), utility_threshold(std::numeric_limits::quiet_NaN()), + random_state(GetPacRandomState(ctx)) { // Read utility threshold setting Value ut_val; if (ctx.TryGetCurrentSetting("privacy_min_group_count", ut_val) && !ut_val.IsNull()) { utility_threshold = ut_val.GetValue(); } - Value pac_seed_val; - if (ctx.TryGetCurrentSetting("privacy_seed", pac_seed_val) && !pac_seed_val.IsNull()) { - seed = uint64_t(pac_seed_val.GetValue()); - } else { - seed = 42; - } - if (mi != 0.0) { // randomize seed per query (not in deterministic mode aka mi==0) - seed ^= PAC_MAGIC_HASH * static_cast(ctx.ActiveTransaction().GetActiveQuery()); - } + seed = GetPacSeed(ctx, mi); query_hash = (seed * PAC_MAGIC_HASH) ^ PAC_MAGIC_HASH; // Initialize p-tracking if mi > 0 and pac_ptracking is enabled if (mi > 0.0) { @@ -103,7 +97,7 @@ struct PacCategoricalBindData : public FunctionData { // ============================================================================ struct PacCategoricalLocalState : public FunctionLocalState { PacRandomEngine gen; - explicit PacCategoricalLocalState(uint64_t seed) : gen(PacRandomStream(seed)) { + explicit PacCategoricalLocalState(PacRandomEngine gen_p) : gen(std::move(gen_p)) { } }; @@ -113,7 +107,7 @@ PacCategoricalInitLocal(ExpressionState &state, const BoundFunctionExpression &e if (bind_data) { seed = bind_data->Cast().seed; } - return make_uniq(seed); + return make_uniq(PacRandomStream(seed, *GetPacRandomState(state.GetContext()))); } // ============================================================================ @@ -586,7 +580,7 @@ static void PacNoisedFunction(DataChunk &args, ExpressionState &state, Vector &r auto &function = state.expr.Cast(); auto &bind_data = function.bind_info->Cast(); - auto gen = PacRandomStream(bind_data.seed); + auto gen = PacRandomStream(bind_data.seed, *bind_data.random_state); UnifiedVectorFormat list_data; list_vec.ToUnifiedFormat(count, list_data); @@ -732,7 +726,7 @@ static void PacNoisedDivFunction(DataChunk &args, ExpressionState &state, Vector auto &function = state.expr.Cast(); auto &bind_data = function.bind_info->Cast(); - auto gen = PacRandomStream(bind_data.seed); + auto gen = PacRandomStream(bind_data.seed, *bind_data.random_state); UnifiedVectorFormat sum_data, cnt_data; sum_vec.ToUnifiedFormat(count, sum_data); diff --git a/src/include/aggregates/as_aggregate.hpp b/src/include/aggregates/as_aggregate.hpp index 119420b6..249f4a32 100644 --- a/src/include/aggregates/as_aggregate.hpp +++ b/src/include/aggregates/as_aggregate.hpp @@ -285,10 +285,12 @@ bool PacNoiseInNull(uint64_t key_hash, double mi, double correction, PacRandomEn // noise_variance: the PAC noise variance used for this cell (delta = sigma^2 / (2*mi)) bool PacUtilityNull(double noised_value, double noise_variance, double threshold, PacRandomEngine &gen); -// One lightweight stream per finalization batch, advanced across its cells. -// Stream IDs distinguish batches and workers without hashing counters or adding per-group state. -// Which draw a cell receives depends on execution order; bit-identical per-cell releases are not promised. -PacRandomEngine PacRandomStream(uint64_t seed); +// Streams advance across cells and reset for each query execution. Fixed seeds +// reproduce draws for the same execution order, not after row/worker reordering. +struct PacRandomState; +shared_ptr GetPacRandomState(ClientContext &context); +uint64_t GetPacSeed(ClientContext &context, double mi); +PacRandomEngine PacRandomStream(uint64_t seed, PacRandomState &state); // Minimum total rows across all groups before the diversity check applies // With hash32_32 (hash_repair), single-PU groups always have exactly 32 zero bits @@ -318,13 +320,13 @@ inline double GetPacMiFromSetting(ClientContext &ctx) { } // Bind data used by privacy aggregates to carry `mi`, `correction`, and sampling parameters. -// Reads seed from privacy_seed setting (or uses query-id if not set) and computes query_hash. +// Reads an explicit privacy_seed, otherwise mixes in query-id when noise is enabled. // query_hash is XOR'd with per-row key_hash inside priv_hash() (centralized) and used as // the counter selector for PacNoisySampleFrom64Counters in finalize functions. struct PrivBindData : public FunctionData { double mi; // mutual information parameter from pac_mi setting (controls noise/NULL probability) double correction; // correction factor: multiplies sum/avg/count results, reduces NULL prob for min/max - uint64_t seed; // RNG seed: privacy_seed setting value, or query-id if not set + uint64_t seed; // RNG seed: explicit privacy_seed, otherwise default seed mixed with query-id uint64_t query_hash; // derived from seed: used inside priv_hash() for XOR and as counter selector double scale_divisor; // for DECIMAL as_avg: divide result by 10^scale (default 1.0) bool hash_repair; // if true, priv_hash() repairs hash to exactly 32 bits set @@ -333,6 +335,7 @@ struct PrivBindData : public FunctionData { int sample_count; // SASS mode: total number of sample lanes (m); 64 uses the SIMD bitmask path bool sample_rescale; // SASS mode: rescale SUM/COUNT samples to full-dataset-estimator scale double utility_threshold; // z-score threshold for utility NULLing (NaN = disabled, any value = enabled) + shared_ptr random_state; // query-local stream allocation, shared across aggregate bindings // Persistent secret p-tracking: shared across all aggregates in the same query (same query_hash). // When active (mi > 0 and pac_ptracking enabled), noise calibration uses p-weighted variance @@ -344,7 +347,7 @@ struct PrivBindData : public FunctionData { mutable uint64_t suspicious_count; // groups with [29..35] zero bits AND all-same-value mutable uint64_t nonsuspicious_count; // groups that are not suspicious - // Primary constructor - reads seed from privacy_seed setting, or uses query-id if not set. + // Primary constructor - an explicit privacy_seed is stable across queries. // All aggregates in the same query get the same seed and query_hash. explicit PrivBindData(ClientContext &ctx, double mi_val, double correction_val = 1.0, double scale_div = 1.0, bool hash_repair_val = false, int sample_lanes_val = 0, @@ -352,7 +355,7 @@ struct PrivBindData : public FunctionData { : mi(mi_val), correction(correction_val), scale_divisor(scale_div), hash_repair(hash_repair_val), sample_diversity_check(true), sample_lanes(sample_lanes_val), sample_count(sample_count_val), sample_rescale(sample_rescale_val), utility_threshold(std::numeric_limits::quiet_NaN()), - total_update_count(0), suspicious_count(0), nonsuspicious_count(0) { + random_state(GetPacRandomState(ctx)), total_update_count(0), suspicious_count(0), nonsuspicious_count(0) { Value sd_val; if (ctx.TryGetCurrentSetting("pac_sample_diversity_check", sd_val) && !sd_val.IsNull()) { sample_diversity_check = sd_val.GetValue(); @@ -362,15 +365,7 @@ struct PrivBindData : public FunctionData { if (ctx.TryGetCurrentSetting("privacy_min_group_count", ut_val) && !ut_val.IsNull()) { utility_threshold = ut_val.GetValue(); } - Value pac_seed_val; - if (ctx.TryGetCurrentSetting("privacy_seed", pac_seed_val) && !pac_seed_val.IsNull()) { - seed = uint64_t(pac_seed_val.GetValue()); - } else { - seed = 42; - } - if (mi != 0.0) { // randomize seed per query (not in deterministic mode aka mi==0) - seed ^= PAC_MAGIC_HASH * static_cast(ctx.ActiveTransaction().GetActiveQuery()); - } + seed = GetPacSeed(ctx, mi); query_hash = (seed * PAC_MAGIC_HASH) ^ PAC_MAGIC_HASH; // Initialize p-tracking if mi > 0 and pac_ptracking is enabled diff --git a/test/sql/pac_noise_independence.test b/test/sql/pac_noise_independence.test index 2ec63d02..a097cf6c 100644 --- a/test/sql/pac_noise_independence.test +++ b/test/sql/pac_noise_independence.test @@ -110,3 +110,90 @@ FROM (SELECT as_noised(range(64)::FLOAT[]) AS l, FROM range(4096)) t ---- true true true + +# A fixed seed repeats for the same execution setup, even after unrelated draws. +statement ok +CREATE TABLE seeded_first AS +SELECT g, as_noised_sum(priv_hash(hash(e)), v) AS s, + as_noised_count(priv_hash(hash(e))) AS c, + as_noised_min(priv_hash(hash(e)), v) AS mn, + as_noised_max(priv_hash(hash(e)), v) AS mx, + as_noised_clip_sum(priv_hash(hash(e)), v) AS cs, + as_noised_clip_max(priv_hash(hash(e)), v) AS cm +FROM solo GROUP BY g + +statement ok +CREATE TABLE unrelated_draws AS SELECT as_noised(range(64)::FLOAT[]) AS v FROM range(8192) + +statement ok +CREATE TABLE seeded_second AS +SELECT g, as_noised_sum(priv_hash(hash(e)), v) AS s, + as_noised_count(priv_hash(hash(e))) AS c, + as_noised_min(priv_hash(hash(e)), v) AS mn, + as_noised_max(priv_hash(hash(e)), v) AS mx, + as_noised_clip_sum(priv_hash(hash(e)), v) AS cs, + as_noised_clip_max(priv_hash(hash(e)), v) AS cm +FROM solo GROUP BY g + +query I +SELECT count(*) FROM seeded_first a JOIN seeded_second b USING (g) +WHERE a.s IS DISTINCT FROM b.s OR a.c IS DISTINCT FROM b.c + OR a.mn IS DISTINCT FROM b.mn OR a.mx IS DISTINCT FROM b.mx + OR a.cs IS DISTINCT FROM b.cs OR a.cm IS DISTINCT FROM b.cm +---- +0 + +# Prepared scalar releases must reset streams on every execution and advance +# between vectors; bind data can survive many executions of the statement. +statement ok +CREATE TABLE seeded_prepared(i BIGINT, l FLOAT, f FLOAT, d DOUBLE) + +statement ok +PREPARE seeded_release AS INSERT INTO seeded_prepared +SELECT i, as_noised(range(64)::FLOAT[]), priv_finalize(range(64)::FLOAT[]), + as_noised_div(range(64)::FLOAT[], list_transform(range(64), x -> 1::FLOAT)) +FROM range(4096) t(i) + +statement ok +EXECUTE seeded_release + +statement ok +EXECUTE seeded_release + +query I +SELECT count(*) FROM seeded_prepared a JOIN seeded_prepared b USING (i) +WHERE a.rowid < 4096 AND b.rowid >= 4096 +AND (a.l IS DISTINCT FROM b.l OR a.f IS DISTINCT FROM b.f OR a.d IS DISTINCT FROM b.d) +---- +0 + +query III +SELECT count(DISTINCT l) > 3000, count(DISTINCT f) > 3000, count(DISTINCT d) > 3000 +FROM seeded_prepared WHERE rowid < 4096 +---- +true true true + +statement ok +DEALLOCATE seeded_release + +statement ok +CREATE TABLE seeded_counts(g BIGINT, c BIGINT) + +statement ok +PREPARE seeded_count AS INSERT INTO seeded_counts +SELECT g, as_noised_count(priv_hash(hash(e))) FROM solo GROUP BY g + +statement ok +EXECUTE seeded_count + +statement ok +EXECUTE seeded_count + +query I +SELECT count(*) FROM seeded_counts a JOIN seeded_counts b USING (g) +WHERE a.rowid < 1000 AND b.rowid >= 1000 AND a.c IS DISTINCT FROM b.c +---- +0 + +statement ok +DEALLOCATE seeded_count From 66c9006d9889cdea2e97c8fa2016e5c8abbef074 Mon Sep 17 00:00:00 2001 From: ila Date: Sun, 1 Nov 2026 08:32:24 +0100 Subject: [PATCH 04/32] Port privacy to DuckDB 2.0 with native CTE decorrelation --- .../workflows/MainDistributionPipeline.yml | 12 +- CMakeLists.txt | 120 ++-- Makefile | 3 +- README.md | 2 +- benchmark/meps_brr/meps_brr_poc.cpp | 8 +- benchmark/ssb/as_ssb_benchmark.cpp | 28 +- docs/build/README.md | 33 +- duckdb | 2 +- extension-ci-tools | 2 +- extension_config.cmake | 3 +- src/aggregates/as_aggregate.cpp | 29 +- src/aggregates/as_clip_min_max.cpp | 200 +++--- src/aggregates/as_clip_sum.cpp | 365 ++++++----- src/aggregates/as_count.cpp | 494 +++++++------- src/aggregates/as_finalize.cpp | 21 +- src/aggregates/as_min_max.cpp | 226 ++++--- src/aggregates/as_sum.cpp | 599 +++++++++-------- src/aggregates/dp_approx_bounds.cpp | 131 ++-- src/aggregates/dp_laplace_noise.cpp | 71 +- src/aggregates/filterless_aggregate.cpp | 289 ++++---- src/categorical/pac_categorical.cpp | 120 ++-- src/categorical/pac_categorical_detection.cpp | 245 +++---- src/categorical/pac_categorical_lambdas.cpp | 297 ++++----- src/categorical/pac_categorical_rewriter.cpp | 458 ++++++------- src/compiler/privacy_bitslice_add_fkjoins.cpp | 22 +- src/compiler/privacy_compiler.cpp | 134 ++-- src/compiler/privacy_compiler_helpers.cpp | 45 +- src/compiler/privacy_mechanisms.cpp | 618 ++++++++++-------- src/core/privacy_extension.cpp | 3 +- src/core/privacy_optimizer.cpp | 24 +- src/diff/pac_utility_diff.cpp | 115 ++-- src/include/aggregates/as_aggregate.hpp | 47 +- src/include/aggregates/as_clip_aggr.hpp | 19 +- src/include/aggregates/as_sum.hpp | 8 +- .../categorical/pac_categorical_detection.hpp | 61 +- .../categorical/pac_categorical_rewriter.hpp | 14 +- src/include/parser/privacy_parser.hpp | 3 +- .../query_processing/pac_subquery_handler.hpp | 2 +- .../privacy_expression_builder.hpp | 2 +- .../query_processing/privacy_fk_resolver.hpp | 2 +- src/include/utils/privacy_helpers.hpp | 3 + src/metadata/privacy_compatibility_check.cpp | 61 +- src/metadata/privacy_metadata_manager.cpp | 4 +- src/parser/privacy_parser.cpp | 43 +- src/parser/privacy_parser_helpers.cpp | 1 + src/query_processing/pac_avg_rewriter.cpp | 32 +- src/query_processing/pac_derived_rewriter.cpp | 264 ++++---- .../pac_projection_propagation.cpp | 124 +++- src/query_processing/pac_subquery_handler.cpp | 70 +- src/query_processing/pac_topk_rewriter.cpp | 363 +++++----- .../privacy_expression_builder.cpp | 310 +++++---- src/query_processing/privacy_fk_resolver.cpp | 4 +- .../privacy_plan_traversal.cpp | 77 ++- .../dp_sass_stability_query.cpp | 21 +- src/test/test_compiler_functions.cpp | 159 +++-- src/test/test_plan_traversal.cpp | 4 +- src/utils/privacy_helpers.cpp | 240 ++++--- test/sql/dp_filterless.test | 4 +- test/sql/dp_sass_m.test | 2 +- test/sql/dp_standard.test | 2 +- ...lice_compiler_uncorrelated_subqueries.test | 8 +- test/sql/pac_categorical_subqueries.test | 78 ++- test/sql/pac_noise_independence.test | 4 +- test/sql/pac_plan.test | 2 +- 64 files changed, 3670 insertions(+), 3087 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index ae93e1c6..7b01c057 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,17 +14,17 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.5.4 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@ebdcc1961780b0ce1272a0d7637dc58447c34885 with: - duckdb_version: v1.5.4 - ci_tools_version: v1.5.4 + duckdb_version: v2.0.0 + ci_tools_version: ebdcc1961780b0ce1272a0d7637dc58447c34885 extension_name: privacy code-quality-check: name: Code Quality Check - uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@v1.5.4 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@ebdcc1961780b0ce1272a0d7637dc58447c34885 with: - duckdb_version: v1.5.4 - ci_tools_version: v1.5.4 + duckdb_version: v2.0.0 + ci_tools_version: ebdcc1961780b0ce1272a0d7637dc58447c34885 extension_name: privacy format_checks: 'format;tidy' diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c3325ad..1cc4b438 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,10 +3,11 @@ cmake_minimum_required(VERSION 3.5) # Set extension name here set(TARGET_NAME privacy) -# DuckDB's extension distribution supports vcpkg. As such, dependencies can be added in ./vcpkg.json and then -# used in cmake with find_package. Feel free to remove or replace with other dependencies. -# Note that it should also be removed from vcpkg.json to prevent needlessly installing it.. -#find_package(OpensSL REQUIRED) +# DuckDB's extension distribution supports vcpkg. As such, dependencies can be +# added in ./vcpkg.json and then used in cmake with find_package. Feel free to +# remove or replace with other dependencies. Note that it should also be removed +# from vcpkg.json to prevent needlessly installing it.. find_package(OpensSL +# REQUIRED) set(EXTENSION_NAME ${TARGET_NAME}_extension) set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) @@ -15,15 +16,16 @@ project(${TARGET_NAME}) include_directories(src/include) include_directories(benchmark/include) -# Option to disable approximate sum algorithm (use exact cascading instead) -# By default, pac_sum/pac_avg use the approx algorithm (16-bit cascading counters) +# Option to disable approximate sum algorithm (use exact cascading instead) By +# default, pac_sum/pac_avg use the approx algorithm (16-bit cascading counters) # Set PAC_NOAPPROX=ON to use exact cascading instead -option(PAC_NOAPPROX "Disable approximate sum algorithm, use exact cascading" OFF) +option(PAC_NOAPPROX "Disable approximate sum algorithm, use exact cascading" + OFF) if(PAC_NOAPPROX) - add_definitions(-DPAC_EXACTSUM) - message(STATUS "privacy: Using exact cascading sum algorithm (PAC_NOAPPROX)") + add_definitions(-DPAC_EXACTSUM) + message(STATUS "privacy: Using exact cascading sum algorithm (PAC_NOAPPROX)") else() - message(STATUS "privacy: Using approximate sum algorithm (default)") + message(STATUS "privacy: Using approximate sum algorithm (default)") endif() # Extension sources @@ -69,8 +71,8 @@ build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) if(WIN32) - target_link_libraries(${EXTENSION_NAME} bcrypt) - target_link_libraries(${LOADABLE_EXTENSION_NAME} bcrypt) + target_link_libraries(${EXTENSION_NAME} PRIVATE bcrypt) + target_link_libraries(${LOADABLE_EXTENSION_NAME} PRIVATE bcrypt) endif() install( @@ -79,51 +81,42 @@ install( LIBRARY DESTINATION "${INSTALL_LIB_DIR}" ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") -# Test runners and benchmarks - only build for native platforms, not WASM or Windows -# (Windows lacks unistd.h and other POSIX features used by the benchmark code) -set(PRIVACY_LINK_LIBS - duckdb_static - duckdb_generated_extension_loader - ${EXTENSION_NAME} - core_functions_extension - parquet_extension - duckdb_static) - -# jemalloc is only built on Linux 64-bit; link it when the extension was loaded -# (can't use if(TARGET ...) here because the target isn't defined yet at this point) -if(DUCKDB_EXTENSION_JEMALLOC_SHOULD_LINK) - list(APPEND PRIVACY_LINK_LIBS jemalloc_extension) -endif() +# Test runners and benchmarks - only build for native platforms, not WASM or +# Windows (Windows lacks unistd.h and other POSIX features used by the benchmark +# code) +set(PRIVACY_LINK_LIBS duckdb_static) if(NOT EMSCRIPTEN AND NOT WIN32) - # Build a single unified test executable that invokes all tests via src/test/test_runner.cpp - add_executable(pac_test_runner - src/test/test_runner.cpp - src/test/test_plan_traversal.cpp - src/test/test_schema_metadata.cpp - src/test/test_compiler_functions.cpp - src/test/test_privacy_parser.cpp) - target_link_libraries(pac_test_runner ${PRIVACY_LINK_LIBS}) - - # TPCH + AS benchmark runner - add_executable(as_tpch_benchmark benchmark/tpch/as_tpch_benchmark.cpp) - target_link_libraries(as_tpch_benchmark ${PRIVACY_LINK_LIBS}) - - # WIP rebuttal experiment: four-way MEPS balanced repeated replication benchmark - add_executable(meps_brr_poc benchmark/meps_brr/meps_brr_poc.cpp) - target_link_libraries(meps_brr_poc ${PRIVACY_LINK_LIBS}) - - # WIP rebuttal experiment: standard joined SSB queries and 64 sampled worlds - add_executable(as_ssb_benchmark benchmark/ssb/as_ssb_benchmark.cpp) - target_link_libraries(as_ssb_benchmark ${PRIVACY_LINK_LIBS}) - - # ClickBench + AS benchmark runner - add_executable(as_clickbench_benchmark benchmark/clickbench/as_clickbench_benchmark.cpp) - target_link_libraries(as_clickbench_benchmark ${PRIVACY_LINK_LIBS}) + # Build a single unified test executable that invokes all tests via + # src/test/test_runner.cpp + add_executable( + pac_test_runner + src/test/test_runner.cpp src/test/test_plan_traversal.cpp + src/test/test_schema_metadata.cpp src/test/test_compiler_functions.cpp + src/test/test_privacy_parser.cpp) + target_link_libraries(pac_test_runner PRIVATE ${PRIVACY_LINK_LIBS}) + + # TPCH + AS benchmark runner + add_executable(as_tpch_benchmark benchmark/tpch/as_tpch_benchmark.cpp) + target_link_libraries(as_tpch_benchmark PRIVATE ${PRIVACY_LINK_LIBS}) + + # WIP rebuttal experiment: four-way MEPS balanced repeated replication + # benchmark + add_executable(meps_brr_poc benchmark/meps_brr/meps_brr_poc.cpp) + target_link_libraries(meps_brr_poc PRIVATE ${PRIVACY_LINK_LIBS}) + + # WIP rebuttal experiment: standard joined SSB queries and 64 sampled worlds + add_executable(as_ssb_benchmark benchmark/ssb/as_ssb_benchmark.cpp) + target_link_libraries(as_ssb_benchmark PRIVATE ${PRIVACY_LINK_LIBS}) + + # ClickBench + AS benchmark runner + add_executable(as_clickbench_benchmark + benchmark/clickbench/as_clickbench_benchmark.cpp) + target_link_libraries(as_clickbench_benchmark PRIVATE ${PRIVACY_LINK_LIBS}) # TPC-DS automatic-rewriter coverage and end-to-end performance add_executable(as_tpcds_benchmark benchmark/tpcds/as_tpcds_benchmark.cpp) - target_link_libraries(as_tpcds_benchmark ${PRIVACY_LINK_LIBS}) + target_link_libraries(as_tpcds_benchmark PRIVATE ${PRIVACY_LINK_LIBS}) # Unified DP utility benchmark runner set(TPCH_STOCK_DP_QUERY_NAMES q01 q05 q06 q14 q19) @@ -146,9 +139,22 @@ if(NOT EMSCRIPTEN AND NOT WIN32) @ONLY) add_executable(dp_benchmark_runner benchmark/dp/dp_benchmark_runner.cpp) target_include_directories(dp_benchmark_runner PRIVATE "${TPCH_QUERY_GENERATED_DIR}") - target_link_libraries(dp_benchmark_runner ${PRIVACY_LINK_LIBS}) - - # IMDb/JOB benchmark runner - add_executable(imdb_benchmark benchmark/imdb/imdb_benchmark.cpp) - target_link_libraries(imdb_benchmark ${PRIVACY_LINK_LIBS}) + target_link_libraries(dp_benchmark_runner PRIVATE ${PRIVACY_LINK_LIBS}) + + # IMDb/JOB benchmark runner + add_executable(imdb_benchmark benchmark/imdb/imdb_benchmark.cpp) + target_link_libraries(imdb_benchmark PRIVATE ${PRIVACY_LINK_LIBS}) + + foreach( + PRIVACY_RUNNER IN + ITEMS pac_test_runner + as_tpch_benchmark + meps_brr_poc + as_ssb_benchmark + as_clickbench_benchmark + as_tpcds_benchmark + dp_benchmark_runner + imdb_benchmark) + link_extension_libraries(${PRIVACY_RUNNER} PRIVATE) + endforeach() endif() diff --git a/Makefile b/Makefile index df0bff53..c9ca9741 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ PROJ_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) # Configuration of extension EXT_NAME=privacy EXT_CONFIG=${PROJ_DIR}extension_config.cmake +OVERRIDE_GIT_DESCRIBE ?= v2.0.0 # Include the Makefile from extension-ci-tools -include extension-ci-tools/makefiles/duckdb_extension.Makefile \ No newline at end of file +include extension-ci-tools/makefiles/duckdb_extension.Makefile diff --git a/README.md b/README.md index aee7ff98..dcd37d8c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Both modes share the same DDL and require no changes to query syntax — the extension rewrites aggregate query plans transparently. -This works on DuckDB v1.5 and beyond. See https://duckdb.org/install to install. Or if you do not want to install anything: this extension is also distributed in WASM, so you can run the examples also in https://shell.duckdb.org in a browser. +This branch targets DuckDB v2.0.0. See https://duckdb.org/install to install. Or if you do not want to install anything: this extension is also distributed in WASM, so you can run the examples also in https://shell.duckdb.org in a browser. ## Install diff --git a/benchmark/meps_brr/meps_brr_poc.cpp b/benchmark/meps_brr/meps_brr_poc.cpp index 8fc9f100..2edd7b66 100644 --- a/benchmark/meps_brr/meps_brr_poc.cpp +++ b/benchmark/meps_brr/meps_brr_poc.cpp @@ -38,14 +38,14 @@ static void Save(const string &path, const string &sql) { throw std::runtime_error("Cannot write " + path); f << sql; } -static unique_ptr Query(Connection &con, const string &sql) { +static unique_ptr Query(Connection &con, const string &sql) { auto r = con.Query(sql); if (!r || r->HasError()) throw std::runtime_error(r ? r->GetError() : "No query result"); return r; } static void Require(Connection &con, const string &sql, const string &message) { - if (!Query(con, sql)->GetValue(0, 0).GetValue()) + if (!Query(con, sql)->Collection().GetValue(0, 0).GetValue()) throw std::runtime_error(message); } static string ReplicateSQL(string sql, int rep) { @@ -78,8 +78,8 @@ static Results Run(Connection &con, const string &sql, const string &mode, int w if (r->RowCount() != 1) throw std::runtime_error("Expected one row of named scalar/list estimates"); for (idx_t col = 0; col < r->ColumnCount(); col++) { - string metric = r->names[col]; - auto value = r->GetValue(col, 0); + string metric = r->GetNames()[col].GetIdentifierName(); + auto value = r->Collection().GetValue(col, 0); if (mode == "duckdb" || mode == "naive_as") { out[metric].push_back(Number(value)); continue; diff --git a/benchmark/ssb/as_ssb_benchmark.cpp b/benchmark/ssb/as_ssb_benchmark.cpp index 9899859f..ec223c73 100644 --- a/benchmark/ssb/as_ssb_benchmark.cpp +++ b/benchmark/ssb/as_ssb_benchmark.cpp @@ -37,14 +37,14 @@ static string Csv(const string &s) { out += c == '"' ? "\"\"" : string(1, c); return out + "\""; } -static unique_ptr Query(Connection &con, const string &sql) { +static unique_ptr Query(Connection &con, const string &sql) { auto r = con.Query(sql); if (!r || r->HasError()) throw std::runtime_error(r ? r->GetError() : "No result"); return r; } static void Require(Connection &con, const string &sql, const string &why) { - if (!Query(con, sql)->GetValue(0, 0).GetValue()) + if (!Query(con, sql)->Collection().GetValue(0, 0).GetValue()) throw std::runtime_error(why); } static string InputQuery(string sql, int world) { @@ -66,10 +66,11 @@ static string QuerySQL(const string &sql) { } using Results = std::map; enum class Mode { FULL, REPEATED, ASA }; -static void Collect(MaterializedQueryResult &r, Results &out, int world, bool is_as, bool grouped) { +static void Collect(QueryResult &r, Results &out, int world, bool is_as, bool grouped) { + auto rows = r.Collection().GetRows(); idx_t aggregate = r.ColumnCount(); for (idx_t col = 0; col < r.ColumnCount(); col++) - if (r.names[col] == "revenue" || r.names[col] == "profit") + if (r.GetNames()[col] == "revenue" || r.GetNames()[col] == "profit") aggregate = col; if (aggregate == r.ColumnCount()) throw std::runtime_error("Missing aggregate column"); @@ -78,13 +79,13 @@ static void Collect(MaterializedQueryResult &r, Results &out, int world, bool is for (idx_t col = 0; col < r.ColumnCount() - (is_as ? 1 : 0); col++) { if (col == aggregate) continue; - auto v = r.GetValue(col, row); + auto v = rows.GetValue(col, row); string s = v.IsNull() ? "NULL" : v.type().ToString() + ":" + v.ToString(); key += std::to_string(s.size()) + ":" + s; } - Value value = r.GetValue(aggregate, row); + Value value = rows.GetValue(aggregate, row); if (is_as) { - auto pv = r.GetValue(r.ColumnCount() - 1, row); + auto pv = rows.GetValue(r.ColumnCount() - 1, row); bool present = false; if (!pv.IsNull()) { const auto &presence = ListValue::GetChildren(pv); @@ -227,17 +228,20 @@ int main(int argc, char **argv) { if (const char *memory_limit = std::getenv("SSB_MEMORY_LIMIT")) { Query(con, "SET memory_limit=" + Quote(memory_limit)); } - string memory_limit = Query(con, "SELECT current_setting('memory_limit')")->GetValue(0, 0).ToString(); + string memory_limit = + Query(con, "SELECT current_setting('memory_limit')")->Collection().GetValue(0, 0).ToString(); std::cerr << "DuckDB memory limit: " << memory_limit << '\n'; Query(con, "SET priv_rewrite=false; SET pac_sample_diversity_check=false; SET pac_mi=0; SET privacy_seed=42; " "SET pac_hash_repair=true"); Query(con, Read(ssb + "/schema.sql")); auto manifest = Query(con, "SELECT \"table\", rows::UBIGINT FROM read_csv(" + Quote(data + "/row_counts.csv") + ", header=true)"); + auto manifest_rows = manifest->Collection().GetRows(); std::map row_counts; for (idx_t row = 0; row < manifest->RowCount(); row++) { if (!row_counts - .emplace(manifest->GetValue(0, row).ToString(), manifest->GetValue(1, row).GetValue()) + .emplace(manifest_rows.GetValue(0, row).ToString(), + manifest_rows.GetValue(1, row).GetValue()) .second) throw std::runtime_error("Duplicate table in row-count manifest"); } @@ -281,9 +285,10 @@ int main(int argc, char **argv) { for (const auto &variant : {std::make_pair("full", aggregate_sql), std::make_pair("asa", as_aggregate_sql), std::make_pair("world0", InputQuery(aggregate_sql, 0))}) { auto explain = Query(con, "EXPLAIN " + variant.second); + auto explain_rows = explain->Collection().GetRows(); plans << "\n--- " << name << " timed " << variant.first << " ---\n"; for (idx_t row = 0; row < explain->RowCount(); row++) { - plans << explain->GetValue(1, row).ToString() << '\n'; + plans << explain_rows.GetValue(1, row).ToString() << '\n'; } } bool rewritten = false, executed = false; @@ -291,9 +296,10 @@ int main(int argc, char **argv) { Query(con, "SET priv_rewrite=true"); try { auto explain = Query(con, "EXPLAIN " + sql); + auto explain_rows = explain->Collection().GetRows(); string plan; for (idx_t row = 0; row < explain->RowCount(); row++) - plan += explain->GetValue(1, row).ToString(); + plan += explain_rows.GetValue(1, row).ToString(); plans << "\n--- " << name << " ---\n" << plan << '\n'; rewritten = plan.find("as_noised_sum") != string::npos || plan.find("as_sum(") != string::npos; if (!rewritten) diff --git a/docs/build/README.md b/docs/build/README.md index 38492142..aee454c7 100644 --- a/docs/build/README.md +++ b/docs/build/README.md @@ -3,9 +3,9 @@ ## Prerequisites - **git** (with submodule support) -- **cmake** (>= 3.11) +- **cmake** (>= 3.14) - **ninja** (optional but recommended — significantly faster builds) -- **C++ compiler** with C++11 support (clang recommended, especially for benchmarks) +- **C++ compiler** with C++17 support (clang recommended, especially for benchmarks) ## Clone @@ -28,7 +28,22 @@ make # release build without ninja make debug # debug build ``` -The extension is built into `build/release/extension/pac/pac.duckdb_extension`. +The extension is built into `build/release/extension/privacy/privacy.duckdb_extension`. + +The Makefile sets `OVERRIDE_GIT_DESCRIBE=v2.0.0` so the loadable extension +uses the released DuckDB ABI label. When configuring CMake directly, pass +`-DOVERRIDE_GIT_DESCRIBE=v2.0.0` as well. DuckDB 2.0 requires explicit static +extension registration; `extension_config.cmake` enables it for the CLI, +tests, and benchmark runners. + +The compiler uses DuckDB 2.0's default CTE-based decorrelation. No legacy +`delim_join_as_cte` setting is required. CTE readers carry the corresponding +mask bindings and counter types, and intermediate expressions remain per-world +until the output projection finalizes them. + +DuckDB 2.0 can infer wider DECIMAL types for arithmetic than DuckDB 1.5. +The inferred width selects the SUM state (including the INT128 path), so +approximate outputs can differ across engine versions even with the same seed. ## Run Tests @@ -40,7 +55,7 @@ make test ./build/release/test/unittest --test-dir . "test/sql/pac*" # C++ unit tests -./build/release/extension/pac/pac_test_runner +./build/release/extension/privacy/pac_test_runner ``` ## Build Flags @@ -63,11 +78,13 @@ The DuckDB version is pinned via the `duckdb` and `extension-ci-tools` submodule To update to a new release: ```bash -cd duckdb && git fetch --tags && git checkout v1.X.0 -cd ../extension-ci-tools && git fetch && git checkout origin/v1.X.0 +cd duckdb && git fetch --tags && git checkout v2.0.0 +cd ../extension-ci-tools && git fetch origin && git checkout ebdcc1961780b0ce1272a0d7637dc58447c34885 cd .. git add duckdb extension-ci-tools -git commit -m "Bump DuckDB to v1.X.0" +git commit -m "Bump DuckDB to v2.0.0" ``` -Replace `v1.X.0` with the target version tag. +The upgrade pins DuckDB 2.0.0 and the CI-tools commit shown above. +When changing versions, update both submodule pins and the references in +`.github/workflows/MainDistributionPipeline.yml`. diff --git a/duckdb b/duckdb index 08e34c44..d54cbbe3 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 08e34c447bae34eaee3723cac61f2878b6bdf787 +Subproject commit d54cbbe3df17e8e5ce68312f483b4155836112aa diff --git a/extension-ci-tools b/extension-ci-tools index 1d3a50b3..ebdcc196 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit 1d3a50b3414534f6787e28c1c5efaafb4beb4e8f +Subproject commit ebdcc1961780b0ce1272a0d7637dc58447c34885 diff --git a/extension_config.cmake b/extension_config.cmake index 2a11adbb..de2d3703 100644 --- a/extension_config.cmake +++ b/extension_config.cmake @@ -5,6 +5,7 @@ duckdb_extension_load(privacy SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR} LOAD_TESTS ) +duckdb_extension_statically_link(privacy) # Any extra extensions that should be built -# e.g.: duckdb_extension_load(json) \ No newline at end of file +# e.g.: duckdb_extension_load(json) diff --git a/src/aggregates/as_aggregate.cpp b/src/aggregates/as_aggregate.cpp index debd56a1..23aa01cb 100644 --- a/src/aggregates/as_aggregate.cpp +++ b/src/aggregates/as_aggregate.cpp @@ -1,3 +1,4 @@ +#include "duckdb/common/vector/list_vector.hpp" #include "aggregates/as_aggregate.hpp" #include "duckdb.hpp" @@ -149,7 +150,7 @@ shared_ptr GetPacRandomState(ClientContext &context) { uint64_t GetPacSeed(ClientContext &context, double mi) { Value value; - if (context.TryGetCurrentSetting("privacy_seed", value) && !value.IsNull()) { + if (context.TryGetCurrentSetting(Identifier("privacy_seed"), value) && !value.IsNull()) { return uint64_t(value.GetValue()); } return mi == 0.0 ? 42 : 42 ^ (PAC_MAGIC_HASH * uint64_t(context.ActiveTransaction().GetActiveQuery())); @@ -402,8 +403,8 @@ static void PacAggregateScalar(DataChunk &args, ExpressionState &state, Vector & idx_t count = args.size(); result.SetVectorType(VectorType::FLAT_VECTOR); - auto res = FlatVector::GetData(result); - FlatVector::Validity(result).SetAllValid(count); + auto res = FlatVector::GetDataMutable(result); + FlatVector::ValidityMutable(result).SetAllValid(count); auto &local = ExecuteFunctionState::GetFunctionState(state)->Cast(); auto &gen = local.gen; @@ -412,8 +413,8 @@ static void PacAggregateScalar(DataChunk &args, ExpressionState &state, Vector & uint64_t query_hash = 0; std::shared_ptr pstate; auto &bound_expr = state.expr.Cast(); - if (bound_expr.bind_info) { - auto &bd = bound_expr.bind_info->Cast(); + if (bound_expr.BindInfo()) { + auto &bd = bound_expr.BindInfo()->Cast(); query_hash = bd.query_hash; pstate = bd.pstate; } @@ -547,8 +548,8 @@ static void PacAggregateScalar(DataChunk &args, ExpressionState &state, Vector & } } -static unique_ptr PacAggregateBind(ClientContext &ctx, ScalarFunction &, - vector> &) { +static unique_ptr PacAggregateBind(BindScalarFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); // mi is supplied per-row; the bind-time value is unused for noise but seeds the RNG. return make_uniq(ctx, 128.0); } @@ -559,8 +560,9 @@ void RegisterPacAggregateFunctions(ExtensionLoader &loader) { auto fun = ScalarFunction( "pac_aggregate", {LogicalType::LIST(vals_elem), LogicalType::LIST(cnts_elem), LogicalType::DOUBLE, LogicalType::INTEGER}, - LogicalType::DOUBLE, fn, PacAggregateBind, nullptr, nullptr, PacAggregateInit); - fun.null_handling = FunctionNullHandling::SPECIAL_HANDLING; + LogicalType::DOUBLE, fn, PacAggregateBind, nullptr, PacAggregateInit); + fun.SetNullHandling(FunctionNullHandling::SPECIAL_HANDLING); + fun.SetFallible(); loader.RegisterFunction(fun); }; @@ -595,11 +597,12 @@ void RegisterPacAggregateFunctions(ExtensionLoader &loader) { #undef REG_COUNTS_FOR } -static unique_ptr PacHashBind(ClientContext &ctx, ScalarFunction &, vector> &) { +static unique_ptr PacHashBind(BindScalarFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); double mi = GetPacMiFromSetting(ctx); bool hash_repair = true; Value hr_val; - if (ctx.TryGetCurrentSetting("pac_hash_repair", hr_val) && !hr_val.IsNull()) { + if (ctx.TryGetCurrentSetting(Identifier("pac_hash_repair"), hr_val) && !hr_val.IsNull()) { hash_repair = hr_val.GetValue(); } return make_uniq(ctx, mi, 1.0, 1.0, hash_repair); @@ -611,8 +614,8 @@ static void PacHashFunction(DataChunk &args, ExpressionState &state, Vector &res auto &function = state.expr.Cast(); uint64_t query_hash = 0; bool hash_repair = true; - if (function.bind_info) { - auto &bind_data = function.bind_info->Cast(); + if (function.BindInfo()) { + auto &bind_data = function.BindInfo()->Cast(); query_hash = bind_data.query_hash; hash_repair = bind_data.hash_repair; } diff --git a/src/aggregates/as_clip_min_max.cpp b/src/aggregates/as_clip_min_max.cpp index 3579238f..5154effb 100644 --- a/src/aggregates/as_clip_min_max.cpp +++ b/src/aggregates/as_clip_min_max.cpp @@ -1,3 +1,4 @@ +#include "duckdb/common/vector/list_vector.hpp" #include "aggregates/as_clip_min_max.hpp" #include "categorical/pac_categorical.hpp" #include "duckdb/common/types/decimal.hpp" @@ -180,11 +181,13 @@ static void PacClipMinMaxScatterUpdate(Vector inputs[], Vector &states, idx_t co // Generate for IS_MAX=true (MAX) #define X(NAME, VALUE_T, INPUT_T, SIGNED_VAL) \ - static void PacClipMaxUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, \ - idx_t count) { \ - auto &state = *reinterpret_cast *>(state_p); \ - PacClipMinMaxUpdate(inputs, state, count, aggr.allocator, \ - GetPrivSampleLanes(aggr)); \ + static void PacClipMaxUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t input_count, \ + const ClusteredAggr &clusters, idx_t count) { \ + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { \ + auto &state = *reinterpret_cast *>(state_p); \ + PacClipMinMaxUpdate(inputs, state, count, aggr.allocator, \ + GetPrivSampleLanes(aggr)); \ + }); \ } \ static void PacClipMaxScatterUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, \ idx_t count) { \ @@ -197,11 +200,13 @@ PCMM_INT_TYPES_UNSIGNED // Generate for IS_MAX=false (MIN) #define X(NAME, VALUE_T, INPUT_T, SIGNED_VAL) \ - static void PacClipMinUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, \ - idx_t count) { \ - auto &state = *reinterpret_cast *>(state_p); \ - PacClipMinMaxUpdate(inputs, state, count, aggr.allocator, \ - GetPrivSampleLanes(aggr)); \ + static void PacClipMinUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t input_count, \ + const ClusteredAggr &clusters, idx_t count) { \ + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { \ + auto &state = *reinterpret_cast *>(state_p); \ + PacClipMinMaxUpdate(inputs, state, count, aggr.allocator, \ + GetPrivSampleLanes(aggr)); \ + }); \ } \ static void PacClipMinScatterUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, \ idx_t count) { \ @@ -273,22 +278,26 @@ static void PacClipMinMaxScatterUpdateFloat(Vector inputs[], Vector &states, idx XF(SingleDouble, double, CLIP_DOUBLE_SHIFT) #define XF(NAME, FLOAT_T, SHIFT_VAL) \ - static void PacClipMaxUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, \ - idx_t count) { \ - auto &state = *reinterpret_cast *>(state_p); \ - PacClipMinMaxUpdateFloat(inputs, state, count, aggr.allocator, \ - GetPrivSampleLanes(aggr)); \ + static void PacClipMaxUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t input_count, \ + const ClusteredAggr &clusters, idx_t count) { \ + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { \ + auto &state = *reinterpret_cast *>(state_p); \ + PacClipMinMaxUpdateFloat(inputs, state, count, aggr.allocator, \ + GetPrivSampleLanes(aggr)); \ + }); \ } \ static void PacClipMaxScatterUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, \ idx_t count) { \ PacClipMinMaxScatterUpdateFloat(inputs, states, count, aggr.allocator, \ GetPrivSampleLanes(aggr)); \ } \ - static void PacClipMinUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, \ - idx_t count) { \ - auto &state = *reinterpret_cast *>(state_p); \ - PacClipMinMaxUpdateFloat(inputs, state, count, aggr.allocator, \ - GetPrivSampleLanes(aggr)); \ + static void PacClipMinUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t input_count, \ + const ClusteredAggr &clusters, idx_t count) { \ + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { \ + auto &state = *reinterpret_cast *>(state_p); \ + PacClipMinMaxUpdateFloat(inputs, state, count, aggr.allocator, \ + GetPrivSampleLanes(aggr)); \ + }); \ } \ static void PacClipMinScatterUpdate##NAME(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, \ idx_t count) { \ @@ -341,11 +350,11 @@ static void PacClipMinCombine(Vector &src, Vector &dst, AggregateInputData &aggr // Finalize: noised scalar output // ============================================================================ template -static void PacClipMinMaxFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void PacClipMinMaxFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData *>(states); - auto data = FlatVector::GetData(result); - auto &result_mask = FlatVector::Validity(result); + auto data = FlatVector::GetDataMutable(result); + auto &result_mask = FlatVector::ValidityMutable(result); auto &bind = static_cast(*input.bind_data); double mi = bind.mi; double correction = bind.correction; @@ -422,10 +431,12 @@ static void PacClipMinMaxFinalize(Vector &states, AggregateInputData &input, Vec XFIN(HugeInt, hugeint_t) #define XFIN(NAME, ACC_T) \ - static void PacClipMaxNoisedFinalize##NAME(Vector &s, AggregateInputData &i, Vector &r, idx_t c, idx_t o) { \ + static void PacClipMaxNoisedFinalize##NAME(Vector &s, AggregateFinalizeInputData &i, Vector &r, idx_t c, \ + idx_t o) { \ PacClipMinMaxFinalize(s, i, r, c, o); \ } \ - static void PacClipMinNoisedFinalize##NAME(Vector &s, AggregateInputData &i, Vector &r, idx_t c, idx_t o) { \ + static void PacClipMinNoisedFinalize##NAME(Vector &s, AggregateFinalizeInputData &i, Vector &r, idx_t c, \ + idx_t o) { \ PacClipMinMaxFinalize(s, i, r, c, o); \ } PCMM_FINALIZE_TYPES @@ -435,8 +446,8 @@ PCMM_FINALIZE_TYPES // Counters finalize (LIST output) // ============================================================================ template -static void PacClipMinMaxFinalizeCounters(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacClipMinMaxFinalizeCounters(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData *>(states); auto &bind = static_cast(*input.bind_data); int clip_support = bind.clip_support_threshold; @@ -444,14 +455,14 @@ static void PacClipMinMaxFinalizeCounters(Vector &states, AggregateInputData &in double float_scale = bind.float_scale; bool clip_scale = bind.clip_scale; - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * 64; ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); for (idx_t i = 0; i < count; i++) { PacClipMinMaxFlushBuffer(*state_ptrs[i], *state_ptrs[i], input.allocator); @@ -499,10 +510,10 @@ static void PacClipMinMaxFinalizeCounters(Vector &states, AggregateInputData &in } } -static void PacClipMaxFinalizeCounters(Vector &s, AggregateInputData &i, Vector &r, idx_t c, idx_t o) { +static void PacClipMaxFinalizeCounters(Vector &s, AggregateFinalizeInputData &i, Vector &r, idx_t c, idx_t o) { PacClipMinMaxFinalizeCounters(s, i, r, c, o); } -static void PacClipMinFinalizeCounters(Vector &s, AggregateInputData &i, Vector &r, idx_t c, idx_t o) { +static void PacClipMinFinalizeCounters(Vector &s, AggregateFinalizeInputData &i, Vector &r, idx_t c, idx_t o) { PacClipMinMaxFinalizeCounters(s, i, r, c, o); } @@ -510,13 +521,16 @@ static void PacClipMinFinalizeCounters(Vector &s, AggregateInputData &i, Vector // State size / init / bind // ============================================================================ template -static idx_t PacClipMinMaxStateSize(const AggregateFunction &) { +static idx_t PacClipMinMaxStateSize(AggregateStateInput &) { return sizeof(PacClipMinMaxStateWrapper); } template -static void PacClipMinMaxInitialize(const AggregateFunction &, data_ptr_t state_p) { - memset(state_p, 0, sizeof(PacClipMinMaxStateWrapper)); +static void PacClipMinMaxInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_p = states[i]; + memset(state_p, 0, sizeof(PacClipMinMaxStateWrapper)); + } } // PacClipBind, PacClipBindFloat, PacClipBindDouble are defined in as_clip_aggr.hpp @@ -534,40 +548,42 @@ static AggregateFunction GetPacClipMinMaxNoisedAggregate(PhysicalType type) { switch (type) { case PhysicalType::INT16: - return AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::SMALLINT}, LogicalType::HUGEINT, state_size, - init, IS_MAX ? PacClipMaxScatterUpdateSmallInt : PacClipMinScatterUpdateSmallInt, - combine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, - IS_MAX ? PacClipMaxUpdateSmallInt : PacClipMinUpdateSmallInt); + return AggregateFunction( + Identifier(name), {LogicalType::UBIGINT, LogicalType::SMALLINT}, LogicalType::HUGEINT, state_size, init, + IS_MAX ? PacClipMaxScatterUpdateSmallInt : PacClipMinScatterUpdateSmallInt, combine, finalize, + FunctionNullHandling::DEFAULT_NULL_HANDLING, IS_MAX ? PacClipMaxUpdateSmallInt : PacClipMinUpdateSmallInt); case PhysicalType::INT32: - return AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::INTEGER}, LogicalType::HUGEINT, state_size, - init, IS_MAX ? PacClipMaxScatterUpdateInteger : PacClipMinScatterUpdateInteger, - combine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, - IS_MAX ? PacClipMaxUpdateInteger : PacClipMinUpdateInteger); + return AggregateFunction( + Identifier(name), {LogicalType::UBIGINT, LogicalType::INTEGER}, LogicalType::HUGEINT, state_size, init, + IS_MAX ? PacClipMaxScatterUpdateInteger : PacClipMinScatterUpdateInteger, combine, finalize, + FunctionNullHandling::DEFAULT_NULL_HANDLING, IS_MAX ? PacClipMaxUpdateInteger : PacClipMinUpdateInteger); case PhysicalType::INT64: - return AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::BIGINT}, LogicalType::HUGEINT, state_size, - init, IS_MAX ? PacClipMaxScatterUpdateBigInt : PacClipMinScatterUpdateBigInt, combine, - finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, - IS_MAX ? PacClipMaxUpdateBigInt : PacClipMinUpdateBigInt); + return AggregateFunction( + Identifier(name), {LogicalType::UBIGINT, LogicalType::BIGINT}, LogicalType::HUGEINT, state_size, init, + IS_MAX ? PacClipMaxScatterUpdateBigInt : PacClipMinScatterUpdateBigInt, combine, finalize, + FunctionNullHandling::DEFAULT_NULL_HANDLING, IS_MAX ? PacClipMaxUpdateBigInt : PacClipMinUpdateBigInt); case PhysicalType::INT128: - return AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::HUGEINT}, LogicalType::HUGEINT, state_size, - init, IS_MAX ? PacClipMaxScatterUpdateBigInt : PacClipMinScatterUpdateBigInt, combine, - IS_MAX ? PacClipMaxNoisedFinalizeHugeInt : PacClipMinNoisedFinalizeHugeInt, - FunctionNullHandling::DEFAULT_NULL_HANDLING, - IS_MAX ? PacClipMaxUpdateBigInt : PacClipMinUpdateBigInt); + return AggregateFunction( + Identifier(name), {LogicalType::UBIGINT, LogicalType::HUGEINT}, LogicalType::HUGEINT, state_size, init, + IS_MAX ? PacClipMaxScatterUpdateBigInt : PacClipMinScatterUpdateBigInt, combine, + IS_MAX ? PacClipMaxNoisedFinalizeHugeInt : PacClipMinNoisedFinalizeHugeInt, + FunctionNullHandling::DEFAULT_NULL_HANDLING, IS_MAX ? PacClipMaxUpdateBigInt : PacClipMinUpdateBigInt); default: throw InternalException("as_noised_clip_min/max: unsupported decimal physical type"); } } template -static unique_ptr BindDecimalPacNoisedClipMinMax(ClientContext &ctx, AggregateFunction &function, - vector> &args) { - auto decimal_type = args[1]->return_type; - function = GetPacClipMinMaxNoisedAggregate(decimal_type.InternalType()); - function.name = IS_MAX ? "as_noised_clip_max" : "as_noised_clip_min"; - function.arguments[1] = decimal_type; - function.return_type = LogicalType::DECIMAL(Decimal::MAX_WIDTH_DECIMAL, DecimalType::GetScale(decimal_type)); - return PacClipBind(ctx, function, args); +static unique_ptr BindDecimalPacNoisedClipMinMax(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &function = bind_input.GetBoundFunction(); + auto &args = bind_input.GetArguments(); + auto decimal_type = args[1]->GetReturnType(); + function.ReplaceImplementation(GetPacClipMinMaxNoisedAggregate(decimal_type.InternalType())); + function.SetName(IS_MAX ? "as_noised_clip_max" : "as_noised_clip_min"); + function.GetArguments()[1] = decimal_type; + function.SetReturnType(LogicalType::DECIMAL(Decimal::MAX_WIDTH_DECIMAL, DecimalType::GetScale(decimal_type))); + return PacClipBind(bind_input); } // ============================================================================ @@ -576,29 +592,29 @@ static unique_ptr BindDecimalPacNoisedClipMinMax(ClientContext &ct template static void AddClipMinMaxCountersFcn(AggregateFunctionSet &set, const string &name, const LogicalType &value_type, aggregate_update_t scatter, aggregate_finalize_t finalize, - aggregate_simple_update_t update) { + aggregate_cluster_update_t update) { auto list_type = LogicalType::LIST(PacFloatLogicalType()); - set.AddFunction(AggregateFunction(name, {LogicalType::UBIGINT, value_type}, list_type, + set.AddFunction(AggregateFunction(Identifier(name), {LogicalType::UBIGINT, value_type}, list_type, PacClipMinMaxStateSize, PacClipMinMaxInitialize, scatter, IS_MAX ? PacClipMaxCombine : PacClipMinCombine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacClipBind)); - set.AddFunction(AggregateFunction(name, {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, list_type, - PacClipMinMaxStateSize, PacClipMinMaxInitialize, scatter, - IS_MAX ? PacClipMaxCombine : PacClipMinCombine, finalize, + set.AddFunction(AggregateFunction(Identifier(name), {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, + list_type, PacClipMinMaxStateSize, PacClipMinMaxInitialize, + scatter, IS_MAX ? PacClipMaxCombine : PacClipMinCombine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacClipBind)); } template static void AddNoisedClipMinMaxFcn(AggregateFunctionSet &set, const string &name, const LogicalType &value_type, const LogicalType &result_type, aggregate_update_t scatter, - aggregate_finalize_t finalize, aggregate_simple_update_t update) { - set.AddFunction(AggregateFunction(name, {LogicalType::UBIGINT, value_type}, result_type, + aggregate_finalize_t finalize, aggregate_cluster_update_t update) { + set.AddFunction(AggregateFunction(Identifier(name), {LogicalType::UBIGINT, value_type}, result_type, PacClipMinMaxStateSize, PacClipMinMaxInitialize, scatter, IS_MAX ? PacClipMaxCombine : PacClipMinCombine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacClipBind)); - set.AddFunction(AggregateFunction(name, {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, result_type, - PacClipMinMaxStateSize, PacClipMinMaxInitialize, scatter, - IS_MAX ? PacClipMaxCombine : PacClipMinCombine, finalize, + set.AddFunction(AggregateFunction(Identifier(name), {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, + result_type, PacClipMinMaxStateSize, PacClipMinMaxInitialize, + scatter, IS_MAX ? PacClipMaxCombine : PacClipMinCombine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacClipBind)); } @@ -690,25 +706,25 @@ static void AddFloatDoubleOverloads(AggregateFunctionSet &set, const string &nam // FLOAT set.AddFunction( - AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::FLOAT}, list_type, state_size, init, + AggregateFunction(Identifier(name), {LogicalType::UBIGINT, LogicalType::FLOAT}, list_type, state_size, init, IS_MAX ? PacClipMaxScatterUpdateSingleFloat : PacClipMinScatterUpdateSingleFloat, combine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, IS_MAX ? PacClipMaxUpdateSingleFloat : PacClipMinUpdateSingleFloat, PacClipBindFloat)); set.AddFunction(AggregateFunction( - name, {LogicalType::UBIGINT, LogicalType::FLOAT, LogicalType::DOUBLE}, list_type, state_size, init, - IS_MAX ? PacClipMaxScatterUpdateSingleFloat : PacClipMinScatterUpdateSingleFloat, combine, finalize, + Identifier(name), {LogicalType::UBIGINT, LogicalType::FLOAT, LogicalType::DOUBLE}, list_type, state_size, + init, IS_MAX ? PacClipMaxScatterUpdateSingleFloat : PacClipMinScatterUpdateSingleFloat, combine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, IS_MAX ? PacClipMaxUpdateSingleFloat : PacClipMinUpdateSingleFloat, PacClipBindFloat)); // DOUBLE set.AddFunction( - AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::DOUBLE}, list_type, state_size, init, - IS_MAX ? PacClipMaxScatterUpdateSingleDouble : PacClipMinScatterUpdateSingleDouble, + AggregateFunction(Identifier(name), {LogicalType::UBIGINT, LogicalType::DOUBLE}, list_type, state_size, + init, IS_MAX ? PacClipMaxScatterUpdateSingleDouble : PacClipMinScatterUpdateSingleDouble, combine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, IS_MAX ? PacClipMaxUpdateSingleDouble : PacClipMinUpdateSingleDouble, PacClipBindDouble)); set.AddFunction(AggregateFunction( - name, {LogicalType::UBIGINT, LogicalType::DOUBLE, LogicalType::DOUBLE}, list_type, state_size, init, - IS_MAX ? PacClipMaxScatterUpdateSingleDouble : PacClipMinScatterUpdateSingleDouble, combine, finalize, + Identifier(name), {LogicalType::UBIGINT, LogicalType::DOUBLE, LogicalType::DOUBLE}, list_type, state_size, + init, IS_MAX ? PacClipMaxScatterUpdateSingleDouble : PacClipMinScatterUpdateSingleDouble, combine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, IS_MAX ? PacClipMaxUpdateSingleDouble : PacClipMinUpdateSingleDouble, PacClipBindDouble)); } else { @@ -716,28 +732,28 @@ static void AddFloatDoubleOverloads(AggregateFunctionSet &set, const string &nam auto double_finalize = IS_MAX ? PacClipMaxNoisedFinalizeDouble : PacClipMinNoisedFinalizeDouble; // FLOAT → FLOAT - set.AddFunction( - AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::FLOAT}, LogicalType::FLOAT, state_size, init, - IS_MAX ? PacClipMaxScatterUpdateSingleFloat : PacClipMinScatterUpdateSingleFloat, combine, - float_finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, - IS_MAX ? PacClipMaxUpdateSingleFloat : PacClipMinUpdateSingleFloat, PacClipBindFloat)); set.AddFunction(AggregateFunction( - name, {LogicalType::UBIGINT, LogicalType::FLOAT, LogicalType::DOUBLE}, LogicalType::FLOAT, state_size, init, + Identifier(name), {LogicalType::UBIGINT, LogicalType::FLOAT}, LogicalType::FLOAT, state_size, init, IS_MAX ? PacClipMaxScatterUpdateSingleFloat : PacClipMinScatterUpdateSingleFloat, combine, float_finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, IS_MAX ? PacClipMaxUpdateSingleFloat : PacClipMinUpdateSingleFloat, PacClipBindFloat)); + set.AddFunction(AggregateFunction( + Identifier(name), {LogicalType::UBIGINT, LogicalType::FLOAT, LogicalType::DOUBLE}, LogicalType::FLOAT, + state_size, init, IS_MAX ? PacClipMaxScatterUpdateSingleFloat : PacClipMinScatterUpdateSingleFloat, combine, + float_finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, + IS_MAX ? PacClipMaxUpdateSingleFloat : PacClipMinUpdateSingleFloat, PacClipBindFloat)); // DOUBLE → DOUBLE - set.AddFunction( - AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::DOUBLE}, LogicalType::DOUBLE, state_size, init, - IS_MAX ? PacClipMaxScatterUpdateSingleDouble : PacClipMinScatterUpdateSingleDouble, - combine, double_finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, - IS_MAX ? PacClipMaxUpdateSingleDouble : PacClipMinUpdateSingleDouble, PacClipBindDouble)); set.AddFunction(AggregateFunction( - name, {LogicalType::UBIGINT, LogicalType::DOUBLE, LogicalType::DOUBLE}, LogicalType::DOUBLE, state_size, - init, IS_MAX ? PacClipMaxScatterUpdateSingleDouble : PacClipMinScatterUpdateSingleDouble, combine, + Identifier(name), {LogicalType::UBIGINT, LogicalType::DOUBLE}, LogicalType::DOUBLE, state_size, init, + IS_MAX ? PacClipMaxScatterUpdateSingleDouble : PacClipMinScatterUpdateSingleDouble, combine, double_finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, IS_MAX ? PacClipMaxUpdateSingleDouble : PacClipMinUpdateSingleDouble, PacClipBindDouble)); + set.AddFunction(AggregateFunction( + Identifier(name), {LogicalType::UBIGINT, LogicalType::DOUBLE, LogicalType::DOUBLE}, LogicalType::DOUBLE, + state_size, init, IS_MAX ? PacClipMaxScatterUpdateSingleDouble : PacClipMinScatterUpdateSingleDouble, + combine, double_finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, + IS_MAX ? PacClipMaxUpdateSingleDouble : PacClipMinUpdateSingleDouble, PacClipBindDouble)); } } @@ -748,7 +764,7 @@ template static void RegisterPacClipMinMaxCountersFunctions(ExtensionLoader &loader) { const string name = IS_MAX ? "as_clip_max" : "as_clip_min"; const string short_name = IS_MAX ? "clip_max" : "clip_min"; - AggregateFunctionSet fcn_set(name); + AggregateFunctionSet fcn_set {Identifier(name)}; RegisterClipMinMaxTypeOverloads(fcn_set, name, true); // DECIMAL overloads @@ -771,7 +787,7 @@ static void RegisterPacClipMinMaxCountersFunctions(ExtensionLoader &loader) { template static void RegisterPacNoisedClipMinMaxFunctions(ExtensionLoader &loader) { const string name = IS_MAX ? "as_noised_clip_max" : "as_noised_clip_min"; - AggregateFunctionSet fcn_set(name); + AggregateFunctionSet fcn_set {Identifier(name)}; RegisterClipMinMaxTypeOverloads(fcn_set, name, false); // DECIMAL overloads diff --git a/src/aggregates/as_clip_sum.cpp b/src/aggregates/as_clip_sum.cpp index acd891e1..3ea6a073 100644 --- a/src/aggregates/as_clip_sum.cpp +++ b/src/aggregates/as_clip_sum.cpp @@ -1,3 +1,4 @@ +#include "duckdb/common/vector/list_vector.hpp" #include "aggregates/as_clip_sum.hpp" #include "categorical/pac_categorical.hpp" #include "duckdb/common/types/decimal.hpp" @@ -266,11 +267,13 @@ static void PacClipSumScatterUpdate(Vector inputs[], Vector &states, idx_t count X(UBigInt, uint64_t, uint64_t, false) #define X(NAME, VALUE_T, INPUT_T, SIGNED) \ - static void PacClipSumUpdate##NAME(Vector input[], AggregateInputData &agg, idx_t, data_ptr_t state_p, \ - idx_t cnt) { \ - auto &state = *reinterpret_cast *>(state_p); \ - PacClipSumUpdate(input, state, cnt, agg.allocator, \ - GetPrivSampleLanes(agg)); \ + static void PacClipSumUpdate##NAME(Vector input[], AggregateInputData &agg, idx_t input_count, \ + const ClusteredAggr &clusters, idx_t cnt) { \ + ExecutePrivClusters(input, input_count, clusters, [&](Vector input[], data_ptr_t state_p, idx_t cnt) { \ + auto &state = *reinterpret_cast *>(state_p); \ + PacClipSumUpdate(input, state, cnt, agg.allocator, \ + GetPrivSampleLanes(agg)); \ + }); \ } \ static void PacClipSumScatterUpdate##NAME(Vector input[], AggregateInputData &agg, idx_t, Vector &sts, \ idx_t cnt) { \ @@ -282,22 +285,26 @@ CLIP_SUM_INT_TYPES_UNSIGNED #undef X // HugeInt update (signed, via hugeint routing — 128-bit needs full 62 levels) -static void PacClipSumUpdateHugeInt(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, idx_t count) { - auto &state = *reinterpret_cast *>(state_p); - int sample_lanes = GetPrivSampleLanes(aggr); - UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto values = UnifiedVectorFormat::GetData(value_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); - if (!hash_data.validity.RowIsValid(h_idx) || !value_data.validity.RowIsValid(v_idx)) { - continue; +static void PacClipSumUpdateHugeInt(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto &state = *reinterpret_cast *>(state_p); + int sample_lanes = GetPrivSampleLanes(aggr); + UnifiedVectorFormat hash_data, value_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, value_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto values = UnifiedVectorFormat::GetData(value_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto v_idx = value_data.sel->get_index(i); + if (!hash_data.validity.RowIsValid(h_idx) || !value_data.validity.RowIsValid(v_idx)) { + continue; + } + PacClipSumUpdateOne(state, hashes[h_idx], values[v_idx], aggr.allocator, + sample_lanes); } - PacClipSumUpdateOne(state, hashes[h_idx], values[v_idx], aggr.allocator, sample_lanes); - } + }); } static void PacClipSumScatterUpdateHugeInt(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -321,48 +328,50 @@ static void PacClipSumScatterUpdateHugeInt(Vector inputs[], AggregateInputData & } // UHugeInt update (unsigned, convert to hugeint for routing — 128-bit needs full 62 levels) -static void PacClipSumUpdateUHugeInt(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, - idx_t count) { - auto &state = *reinterpret_cast *>(state_p); - int sample_lanes = GetPrivSampleLanes(aggr); - UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto values = UnifiedVectorFormat::GetData(value_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); - if (!hash_data.validity.RowIsValid(h_idx) || !value_data.validity.RowIsValid(v_idx)) { - continue; - } - // uhugeint_t is always positive; treat as 128-bit unsigned - auto &v = values[v_idx]; - auto *pos_state = state.EnsureState(aggr.allocator); - if (DUCKDB_LIKELY(hashes[h_idx])) { - uint64_t counter_hash = TransformPacUpdateHash(hashes[h_idx], sample_lanes); - uint64_t upper = static_cast(v.upper); - uint64_t lower = v.lower; - int level = PacClipSumIntState::GetLevel128(upper, lower); - uint64_t shift = level << 1; - uint16_t shifted_val; - if (shift >= 64) { - shifted_val = static_cast(upper >> (shift - 64)); - } else if (shift > 0) { - shifted_val = static_cast((lower >> shift) | (upper << (64 - shift))); - } else { - shifted_val = static_cast(lower); +static void PacClipSumUpdateUHugeInt(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto &state = *reinterpret_cast *>(state_p); + int sample_lanes = GetPrivSampleLanes(aggr); + UnifiedVectorFormat hash_data, value_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, value_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto values = UnifiedVectorFormat::GetData(value_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto v_idx = value_data.sel->get_index(i); + if (!hash_data.validity.RowIsValid(h_idx) || !value_data.validity.RowIsValid(v_idx)) { + continue; + } + // uhugeint_t is always positive; treat as 128-bit unsigned + auto &v = values[v_idx]; + auto *pos_state = state.EnsureState(aggr.allocator); + if (DUCKDB_LIKELY(hashes[h_idx])) { + uint64_t counter_hash = TransformPacUpdateHash(hashes[h_idx], sample_lanes); + uint64_t upper = static_cast(v.upper); + uint64_t lower = v.lower; + int level = PacClipSumIntState::GetLevel128(upper, lower); + uint64_t shift = level << 1; + uint16_t shifted_val; + if (shift >= 64) { + shifted_val = static_cast(upper >> (shift - 64)); + } else if (shift > 0) { + shifted_val = static_cast((lower >> shift) | (upper << (64 - shift))); + } else { + shifted_val = static_cast(lower); + } + shifted_val &= 0xFF; + pos_state->key_hash |= counter_hash; + pos_state->EnsureLevelAllocated(aggr.allocator, level); + uint64_t *buf = pos_state->levels[level]; + buf[17] |= (1ULL << (hashes[h_idx] >> 58)); + pos_state->AddToExactCount(buf, shifted_val, aggr.allocator); + Pac2AddToTotalsSWAR16(buf, shifted_val, counter_hash); + pos_state->update_count++; } - shifted_val &= 0xFF; - pos_state->key_hash |= counter_hash; - pos_state->EnsureLevelAllocated(aggr.allocator, level); - uint64_t *buf = pos_state->levels[level]; - buf[17] |= (1ULL << (hashes[h_idx] >> 58)); - pos_state->AddToExactCount(buf, shifted_val, aggr.allocator); - Pac2AddToTotalsSWAR16(buf, shifted_val, counter_hash); - pos_state->update_count++; } - } + }); } static void PacClipSumScatterUpdateUHugeInt(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -413,35 +422,38 @@ static void PacClipSumScatterUpdateUHugeInt(Vector inputs[], AggregateInputData // Float/Double update: scale to int64, route through signed path // ============================================================================ template -static void PacClipSumUpdateFloat(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, idx_t count) { - auto &state = *reinterpret_cast *>(state_p); - int sample_lanes = GetPrivSampleLanes(aggr); - UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto values = UnifiedVectorFormat::GetData(value_data); - - if (hash_data.validity.AllValid() && value_data.validity.AllValid()) { - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); - PacClipSumUpdateOne( - state, hashes[h_idx], ScaleFloatToInt64(values[v_idx]), aggr.allocator, - sample_lanes); - } - } else { - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); - if (!hash_data.validity.RowIsValid(h_idx) || !value_data.validity.RowIsValid(v_idx)) { - continue; +static void PacClipSumUpdateFloat(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto &state = *reinterpret_cast *>(state_p); + int sample_lanes = GetPrivSampleLanes(aggr); + UnifiedVectorFormat hash_data, value_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, value_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto values = UnifiedVectorFormat::GetData(value_data); + + if (hash_data.validity.AllValid() && value_data.validity.AllValid()) { + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto v_idx = value_data.sel->get_index(i); + PacClipSumUpdateOne( + state, hashes[h_idx], ScaleFloatToInt64(values[v_idx]), aggr.allocator, + sample_lanes); + } + } else { + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto v_idx = value_data.sel->get_index(i); + if (!hash_data.validity.RowIsValid(h_idx) || !value_data.validity.RowIsValid(v_idx)) { + continue; + } + PacClipSumUpdateOne( + state, hashes[h_idx], ScaleFloatToInt64(values[v_idx]), aggr.allocator, + sample_lanes); } - PacClipSumUpdateOne( - state, hashes[h_idx], ScaleFloatToInt64(values[v_idx]), aggr.allocator, - sample_lanes); } - } + }); } template @@ -469,26 +481,26 @@ static void PacClipSumScatterUpdateFloat(Vector inputs[], AggregateInputData &ag } // Instantiate float/double update functions -static void PacClipSumUpdateSingleFloat(Vector inputs[], AggregateInputData &aggr, idx_t n, data_ptr_t state_p, - idx_t count) { - PacClipSumUpdateFloat(inputs, aggr, n, state_p, count); +static void PacClipSumUpdateSingleFloat(Vector inputs[], AggregateInputData &aggr, idx_t n, + const ClusteredAggr &clusters, idx_t count) { + PacClipSumUpdateFloat(inputs, aggr, n, clusters, count); } static void PacClipSumScatterUpdateSingleFloat(Vector inputs[], AggregateInputData &aggr, idx_t n, Vector &states, idx_t count) { PacClipSumScatterUpdateFloat(inputs, aggr, n, states, count); } -static void PacClipSumUpdateSingleDouble(Vector inputs[], AggregateInputData &aggr, idx_t n, data_ptr_t state_p, - idx_t count) { - PacClipSumUpdateFloat(inputs, aggr, n, state_p, count); +static void PacClipSumUpdateSingleDouble(Vector inputs[], AggregateInputData &aggr, idx_t n, + const ClusteredAggr &clusters, idx_t count) { + PacClipSumUpdateFloat(inputs, aggr, n, clusters, count); } static void PacClipSumScatterUpdateSingleDouble(Vector inputs[], AggregateInputData &aggr, idx_t n, Vector &states, idx_t count) { PacClipSumScatterUpdateFloat(inputs, aggr, n, states, count); } -static void DpSampleClipSumUpdateDouble(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, - idx_t count) { - PacClipSumUpdateFloat(inputs, aggr, 0, state_p, count); +static void DpSampleClipSumUpdateDouble(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + PacClipSumUpdateFloat(inputs, aggr, input_count, clusters, count); } static void DpSampleClipSumScatterUpdateDouble(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, @@ -545,10 +557,11 @@ static void DpSampleClipSumCombine(Vector &src, Vector &dst, AggregateInputData // Finalize // ============================================================================ template -static void PacClipSumFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +static void PacClipSumFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, + idx_t offset) { auto state_ptrs = FlatVector::GetData *>(states); - auto data = FlatVector::GetData(result); - auto &result_mask = FlatVector::Validity(result); + auto data = FlatVector::GetDataMutable(result); + auto &result_mask = FlatVector::ValidityMutable(result); auto &bind = static_cast(*input.bind_data); double mi = bind.mi; double correction = bind.correction; @@ -607,35 +620,35 @@ static void PacClipSumFinalize(Vector &states, AggregateInputData &input, Vector // Instantiate noised finalize (scalar output for as_noised_clip_sum) // 64-bit types -static void PacClipSumNoisedFinalizeSigned(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacClipSumNoisedFinalizeSigned(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacClipSumFinalize(states, input, result, count, offset); } -static void PacClipSumNoisedFinalizeUnsigned(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacClipSumNoisedFinalizeUnsigned(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacClipSumFinalize(states, input, result, count, offset); } // 128-bit types -static void PacClipSumNoisedFinalizeSigned128(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacClipSumNoisedFinalizeSigned128(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacClipSumFinalize(states, input, result, count, offset); } -static void PacClipSumNoisedFinalizeUnsigned128(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacClipSumNoisedFinalizeUnsigned128(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacClipSumFinalize(states, input, result, count, offset); } // BIGINT output variant — used for count→sum conversion where the original returned BIGINT -static void PacClipSumNoisedFinalizeBigInt(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacClipSumNoisedFinalizeBigInt(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacClipSumFinalize(states, input, result, count, offset); } // Float/double output variants -static void PacClipSumNoisedFinalizeFloat(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacClipSumNoisedFinalizeFloat(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacClipSumFinalize(states, input, result, count, offset); } -static void PacClipSumNoisedFinalizeDouble(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacClipSumNoisedFinalizeDouble(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacClipSumFinalize(states, input, result, count, offset); } @@ -646,7 +659,7 @@ enum class ClipCountersFinalizeMode : uint8_t { PAC, DP_SAMPLE }; // ============================================================================ template -static void PacClipSumFinalizeCounters(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void PacClipSumFinalizeCounters(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData *>(states); auto &bind = static_cast(*input.bind_data); @@ -657,14 +670,14 @@ static void PacClipSumFinalizeCounters(Vector &states, AggregateInputData &input bool dp_sample = MODE == ClipCountersFinalizeMode::DP_SAMPLE; double dp_sample_rescale = dp_sample && bind.sample_rescale ? DpSampleRescale(bind.sample_lanes) : 1.0; - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * 64; ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); for (idx_t i = 0; i < count; i++) { PacClipSumFlushBuffer(*state_ptrs[i], *state_ptrs[i], input.allocator, bind.sample_lanes); @@ -718,25 +731,25 @@ static void PacClipSumFinalizeCounters(Vector &states, AggregateInputData &input } // 64-bit counters finalize -static void PacClipSumFinalizeCountersSigned(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacClipSumFinalizeCountersSigned(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacClipSumFinalizeCounters(states, input, result, count, offset); } -static void PacClipSumFinalizeCountersUnsigned(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacClipSumFinalizeCountersUnsigned(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacClipSumFinalizeCounters(states, input, result, count, offset); } // 128-bit counters finalize -static void PacClipSumFinalizeCountersSigned128(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacClipSumFinalizeCountersSigned128(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacClipSumFinalizeCounters(states, input, result, count, offset); } -static void PacClipSumFinalizeCountersUnsigned128(Vector &states, AggregateInputData &input, Vector &result, +static void PacClipSumFinalizeCountersUnsigned128(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { PacClipSumFinalizeCounters(states, input, result, count, offset); } -static void DpSampleClipSumFinalizeCounters(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void DpSampleClipSumFinalizeCounters(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacClipSumFinalizeCounters(states, input, result, count, offset); } @@ -744,18 +757,24 @@ static void DpSampleClipSumFinalizeCounters(Vector &states, AggregateInputData & // ============================================================================ // State size / init / bind // ============================================================================ -static idx_t PacClipSumStateSize(const AggregateFunction &) { +static idx_t PacClipSumStateSize(AggregateStateInput &) { return sizeof(PacClipSumStateWrapper<>); } -static idx_t PacClipSumStateSize128(const AggregateFunction &) { +static idx_t PacClipSumStateSize128(AggregateStateInput &) { return sizeof(PacClipSumStateWrapper); } -static void PacClipSumInitialize(const AggregateFunction &, data_ptr_t state_p) { - memset(state_p, 0, sizeof(PacClipSumStateWrapper<>)); +static void PacClipSumInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_p = states[i]; + memset(state_p, 0, sizeof(PacClipSumStateWrapper<>)); + } } -static void PacClipSumInitialize128(const AggregateFunction &, data_ptr_t state_p) { - memset(state_p, 0, sizeof(PacClipSumStateWrapper)); +static void PacClipSumInitialize128(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_p = states[i]; + memset(state_p, 0, sizeof(PacClipSumStateWrapper)); + } } // PacClipBind, PacClipBindFloat, PacClipBindDouble are defined in as_clip_aggr.hpp @@ -818,31 +837,35 @@ static AggregateFunction GetPacClipSumCountersAggregate(PhysicalType type) { } } -static unique_ptr BindDecimalPacNoisedClipSum(ClientContext &ctx, AggregateFunction &function, - vector> &args) { - auto decimal_type = args[1]->return_type; - function = GetPacClipSumNoisedAggregate(decimal_type.InternalType()); - function.name = "as_noised_clip_sum"; - function.arguments[1] = decimal_type; - function.return_type = LogicalType::DECIMAL(Decimal::MAX_WIDTH_DECIMAL, DecimalType::GetScale(decimal_type)); - return PacClipBind(ctx, function, args); +static unique_ptr BindDecimalPacNoisedClipSum(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &function = bind_input.GetBoundFunction(); + auto &args = bind_input.GetArguments(); + auto decimal_type = args[1]->GetReturnType(); + function.ReplaceImplementation(GetPacClipSumNoisedAggregate(decimal_type.InternalType())); + function.SetName("as_noised_clip_sum"); + function.GetArguments()[1] = decimal_type; + function.SetReturnType(LogicalType::DECIMAL(Decimal::MAX_WIDTH_DECIMAL, DecimalType::GetScale(decimal_type))); + return PacClipBind(bind_input); } -static unique_ptr BindDecimalPacClipSum(ClientContext &ctx, AggregateFunction &function, - vector> &args) { - auto decimal_type = args[1]->return_type; - function = GetPacClipSumCountersAggregate(decimal_type.InternalType()); - function.name = "as_clip_sum"; - function.arguments[1] = decimal_type; +static unique_ptr BindDecimalPacClipSum(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &function = bind_input.GetBoundFunction(); + auto &args = bind_input.GetArguments(); + auto decimal_type = args[1]->GetReturnType(); + function.ReplaceImplementation(GetPacClipSumCountersAggregate(decimal_type.InternalType())); + function.SetName("as_clip_sum"); + function.GetArguments()[1] = decimal_type; // counters always return LIST, no DECIMAL return type needed - return PacClipBind(ctx, function, args); + return PacClipBind(bind_input); } -static unique_ptr DpSampleClipSumBind(ClientContext &ctx, AggregateFunction &, - vector> &) { +static unique_ptr DpSampleClipSumBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); Value dc_val; int clip_support = 0; - if (ctx.TryGetCurrentSetting("priv_clip_support", dc_val) && !dc_val.IsNull()) { + if (ctx.TryGetCurrentSetting(Identifier("priv_clip_support"), dc_val) && !dc_val.IsNull()) { clip_support = static_cast(dc_val.GetValue()); } if (clip_support <= 0) { @@ -857,25 +880,26 @@ static unique_ptr DpSampleClipSumBind(ClientContext &ctx, Aggregat // ============================================================================ static void AddClipSumCountersFcn(AggregateFunctionSet &set, const string &name, const LogicalType &value_type, aggregate_update_t scatter, aggregate_finalize_t finalize, - aggregate_simple_update_t update) { + aggregate_cluster_update_t update) { auto list_type = LogicalType::LIST(PacFloatLogicalType()); - set.AddFunction(AggregateFunction(name, {LogicalType::UBIGINT, value_type}, list_type, PacClipSumStateSize, - PacClipSumInitialize, scatter, PacClipSumCombine, finalize, - FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacClipBind)); - set.AddFunction(AggregateFunction(name, {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, list_type, + set.AddFunction(AggregateFunction(Identifier(name), {LogicalType::UBIGINT, value_type}, list_type, PacClipSumStateSize, PacClipSumInitialize, scatter, PacClipSumCombine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacClipBind)); + set.AddFunction(AggregateFunction(Identifier(name), {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, + list_type, PacClipSumStateSize, PacClipSumInitialize, scatter, PacClipSumCombine, + finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacClipBind)); } static void AddNoisedClipSumFcn(AggregateFunctionSet &set, const string &name, const LogicalType &value_type, const LogicalType &result_type, aggregate_update_t scatter, - aggregate_finalize_t finalize, aggregate_simple_update_t update) { - set.AddFunction(AggregateFunction(name, {LogicalType::UBIGINT, value_type}, result_type, PacClipSumStateSize, - PacClipSumInitialize, scatter, PacClipSumCombine, finalize, - FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacClipBind)); - set.AddFunction(AggregateFunction(name, {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, result_type, + aggregate_finalize_t finalize, aggregate_cluster_update_t update) { + set.AddFunction(AggregateFunction(Identifier(name), {LogicalType::UBIGINT, value_type}, result_type, PacClipSumStateSize, PacClipSumInitialize, scatter, PacClipSumCombine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacClipBind)); + set.AddFunction(AggregateFunction(Identifier(name), {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, + result_type, PacClipSumStateSize, PacClipSumInitialize, scatter, + PacClipSumCombine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, update, + PacClipBind)); } // Helper to register all type overloads for a clip sum function set @@ -903,15 +927,16 @@ static void RegisterClipSumTypeOverloads(AggregateFunctionSet &set, const string // HUGEINT/UHUGEINT: use 128-bit state (62 levels) { auto lt = LogicalType::LIST(PacFloatLogicalType()); - set.AddFunction(AggregateFunction( - name, {LogicalType::UBIGINT, LogicalType::HUGEINT}, lt, PacClipSumStateSize128, PacClipSumInitialize128, - PacClipSumScatterUpdateHugeInt, PacClipSumCombine128, PacClipSumFinalizeCountersSigned128, - FunctionNullHandling::DEFAULT_NULL_HANDLING, PacClipSumUpdateHugeInt, PacClipBind)); set.AddFunction( - AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::UHUGEINT}, lt, PacClipSumStateSize128, - PacClipSumInitialize128, PacClipSumScatterUpdateUHugeInt, PacClipSumCombine128, - PacClipSumFinalizeCountersUnsigned128, FunctionNullHandling::DEFAULT_NULL_HANDLING, - PacClipSumUpdateUHugeInt, PacClipBind)); + AggregateFunction(Identifier(name), {LogicalType::UBIGINT, LogicalType::HUGEINT}, lt, + PacClipSumStateSize128, PacClipSumInitialize128, PacClipSumScatterUpdateHugeInt, + PacClipSumCombine128, PacClipSumFinalizeCountersSigned128, + FunctionNullHandling::DEFAULT_NULL_HANDLING, PacClipSumUpdateHugeInt, PacClipBind)); + set.AddFunction( + AggregateFunction(Identifier(name), {LogicalType::UBIGINT, LogicalType::UHUGEINT}, lt, + PacClipSumStateSize128, PacClipSumInitialize128, PacClipSumScatterUpdateUHugeInt, + PacClipSumCombine128, PacClipSumFinalizeCountersUnsigned128, + FunctionNullHandling::DEFAULT_NULL_HANDLING, PacClipSumUpdateUHugeInt, PacClipBind)); } } else { // Noised (scalar HUGEINT) variants @@ -935,12 +960,12 @@ static void RegisterClipSumTypeOverloads(AggregateFunctionSet &set, const string PacClipSumNoisedFinalizeUnsigned, PacClipSumUpdateUBigInt); // HUGEINT/UHUGEINT: use 128-bit state (62 levels) set.AddFunction( - AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::HUGEINT}, LogicalType::HUGEINT, + AggregateFunction(Identifier(name), {LogicalType::UBIGINT, LogicalType::HUGEINT}, LogicalType::HUGEINT, PacClipSumStateSize128, PacClipSumInitialize128, PacClipSumScatterUpdateHugeInt, PacClipSumCombine128, PacClipSumNoisedFinalizeSigned128, FunctionNullHandling::DEFAULT_NULL_HANDLING, PacClipSumUpdateHugeInt, PacClipBind)); set.AddFunction( - AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::UHUGEINT}, LogicalType::HUGEINT, + AggregateFunction(Identifier(name), {LogicalType::UBIGINT, LogicalType::UHUGEINT}, LogicalType::HUGEINT, PacClipSumStateSize128, PacClipSumInitialize128, PacClipSumScatterUpdateUHugeInt, PacClipSumCombine128, PacClipSumNoisedFinalizeUnsigned128, FunctionNullHandling::DEFAULT_NULL_HANDLING, PacClipSumUpdateUHugeInt, PacClipBind)); diff --git a/src/aggregates/as_count.cpp b/src/aggregates/as_count.cpp index c07b5b07..c264518f 100644 --- a/src/aggregates/as_count.cpp +++ b/src/aggregates/as_count.cpp @@ -1,3 +1,4 @@ +#include "duckdb/common/vector/list_vector.hpp" #include #include "aggregates/as_count.hpp" #include "categorical/pac_categorical.hpp" @@ -6,13 +7,16 @@ namespace duckdb { -static unique_ptr PacCountBind(ClientContext &ctx, AggregateFunction &func, - vector> &args) { +static unique_ptr PacCountBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &func = bind_input.GetBoundFunction(); + auto &args = bind_input.GetArguments(); // Correction is always the last arg when it's a foldable numeric constant idx_t corr_idx = args.size(); // sentinel: no correction if (args.size() >= 2) { auto &last = args.back(); - if (last->IsFoldable() && (last->return_type.IsNumeric() || last->return_type.id() == LogicalTypeId::UNKNOWN)) { + if (last->IsFoldable() && + (last->GetReturnType().IsNumeric() || last->GetReturnType().id() == LogicalTypeId::UNKNOWN)) { corr_idx = args.size() - 1; } } @@ -28,52 +32,61 @@ using ScatterState = PacCountStateWrapper; #endif // State functions - uses ScatterState for both simple and scatter -static idx_t PacCountStateSize(const AggregateFunction &) { +static idx_t PacCountStateSize(AggregateStateInput &) { return sizeof(ScatterState); } -static void PacCountInitialize(const AggregateFunction &, data_ptr_t state_ptr) { - memset(state_ptr, 0, sizeof(ScatterState)); +static void PacCountInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i]; + memset(state_ptr, 0, sizeof(ScatterState)); + } } -void PacCountUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - ScatterState &agg = *reinterpret_cast(state_ptr); - int sample_lanes = GetPrivSampleLanes(aggr); - UnifiedVectorFormat idata; - inputs[0].ToUnifiedFormat(count, idata); - auto input_data = UnifiedVectorFormat::GetData(idata); - for (idx_t i = 0; i < count; i++) { - auto idx = idata.sel->get_index(i); - if (idata.validity.RowIsValid(idx)) { - auto key_hash = TransformPacUpdateHash(input_data[idx], sample_lanes); +void PacCountUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, const ClusteredAggr &clusters, + idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + ScatterState &agg = *reinterpret_cast(state_ptr); + int sample_lanes = GetPrivSampleLanes(aggr); + UnifiedVectorFormat idata; + inputs[0].ToUnifiedFormat(count, idata); + auto input_data = UnifiedVectorFormat::GetData(idata); + for (idx_t i = 0; i < count; i++) { + auto idx = idata.sel->get_index(i); + if (idata.validity.RowIsValid(idx)) { + auto key_hash = TransformPacUpdateHash(input_data[idx], sample_lanes); #if defined(PAC_NOBUFFERING) || defined(PAC_NOCASCADING) - PacCountUpdateOne(agg, key_hash, aggr.allocator); + PacCountUpdateOne(agg, key_hash, aggr.allocator); #else PacCountBufferOrUpdateOne(agg, key_hash, aggr.allocator); #endif + } } - } + }); } -void PacCountColumnUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - ScatterState &agg = *reinterpret_cast(state_ptr); - int sample_lanes = GetPrivSampleLanes(aggr); - UnifiedVectorFormat hash_data, col_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, col_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto c_idx = col_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx) && col_data.validity.RowIsValid(c_idx)) { - auto key_hash = TransformPacUpdateHash(hashes[h_idx], sample_lanes); +void PacCountColumnUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, const ClusteredAggr &clusters, + idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + ScatterState &agg = *reinterpret_cast(state_ptr); + int sample_lanes = GetPrivSampleLanes(aggr); + UnifiedVectorFormat hash_data, col_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, col_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto c_idx = col_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx) && col_data.validity.RowIsValid(c_idx)) { + auto key_hash = TransformPacUpdateHash(hashes[h_idx], sample_lanes); #if defined(PAC_NOBUFFERING) || defined(PAC_NOCASCADING) - PacCountUpdateOne(agg, key_hash, aggr.allocator); + PacCountUpdateOne(agg, key_hash, aggr.allocator); #else PacCountBufferOrUpdateOne(agg, key_hash, aggr.allocator); #endif + } } - } + }); } void PacCountScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -119,22 +132,24 @@ void PacCountColumnScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_ } } -static void DpSampleCountMaskUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, - idx_t count) { - ScatterState &agg = *reinterpret_cast(state_ptr); - UnifiedVectorFormat idata; - inputs[0].ToUnifiedFormat(count, idata); - auto input_data = UnifiedVectorFormat::GetData(idata); - for (idx_t i = 0; i < count; i++) { - auto idx = idata.sel->get_index(i); - if (idata.validity.RowIsValid(idx)) { +static void DpSampleCountMaskUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + ScatterState &agg = *reinterpret_cast(state_ptr); + UnifiedVectorFormat idata; + inputs[0].ToUnifiedFormat(count, idata); + auto input_data = UnifiedVectorFormat::GetData(idata); + for (idx_t i = 0; i < count; i++) { + auto idx = idata.sel->get_index(i); + if (idata.validity.RowIsValid(idx)) { #if defined(PAC_NOBUFFERING) || defined(PAC_NOCASCADING) - PacCountUpdateOne(agg, input_data[idx], aggr.allocator); + PacCountUpdateOne(agg, input_data[idx], aggr.allocator); #else PacCountBufferOrUpdateOne(agg, input_data[idx], aggr.allocator); #endif + } } - } + }); } static void DpSampleCountMaskScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, @@ -217,10 +232,10 @@ void PacCountCombine(Vector &src, Vector &dst, AggregateInputData &aggr, idx_t c } } -void PacCountFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +void PacCountFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { auto aggs = FlatVector::GetData(states); - auto data = FlatVector::GetData(result); - auto &result_mask = FlatVector::Validity(result); + auto data = FlatVector::GetDataMutable(result); + auto &result_mask = FlatVector::ValidityMutable(result); auto &bind = input.bind_data->Cast(); double mi = bind.mi; double correction = bind.correction; @@ -270,20 +285,21 @@ void PacCountFinalize(Vector &states, AggregateInputData &input, Vector &result, // Returns LIST with exactly 64 elements (no NULLs). // Position j is 0 if key_hash bit j is 0, otherwise value * 2 * correction. -void PacCountFinalizeCounters(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +void PacCountFinalizeCounters(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, + idx_t offset) { auto aggs = FlatVector::GetData(states); double correction = input.bind_data ? input.bind_data->Cast().correction : 1.0; // Result is LIST - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); // Reserve space for all lists (64 elements each) idx_t total_elements = (offset + count) * 64; ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); PAC_FLOAT buf[64]; for (idx_t i = 0; i < count; i++) { @@ -359,21 +375,21 @@ void RegisterPacCountFunctions(ExtensionLoader &loader) { // PAC's `PacCountFinalizeCounters` zeroes positions where (key_hash >> j) & 1 == 0; // SASS samples each lane independently via DpSampleHash, so all 64 are meaningful. // ============================================================================ -static void DpSampleCountFinalizeCounters(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void DpSampleCountFinalizeCounters(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { auto aggs = FlatVector::GetData(states); auto *bind = input.bind_data ? &input.bind_data->Cast() : nullptr; int sample_lanes = bind ? bind->sample_lanes : 1; double rescale = bind && bind->sample_rescale ? DpSampleRescale(sample_lanes) : 1.0; - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * 64; ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); PAC_FLOAT buf[64]; for (idx_t i = 0; i < count; i++) { @@ -398,17 +414,17 @@ static void DpSampleCountFinalizeCounters(Vector &states, AggregateInputData &in } } -static void DpSampleCountMaskFinalizeCounters(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void DpSampleCountMaskFinalizeCounters(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { auto aggs = FlatVector::GetData(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * 64; ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); PAC_FLOAT buf[64]; for (idx_t i = 0; i < count; i++) { @@ -433,8 +449,8 @@ static void DpSampleCountMaskFinalizeCounters(Vector &states, AggregateInputData } } -static unique_ptr DpSampleCountBind(ClientContext &ctx, AggregateFunction &, - vector> &) { +static unique_ptr DpSampleCountBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); return MakeDpSampleBindData(ctx); } @@ -474,12 +490,15 @@ struct DpSampleMCountState { PAC_FLOAT *counts; }; -static idx_t DpSampleMCountStateSize(const AggregateFunction &) { +static idx_t DpSampleMCountStateSize(AggregateStateInput &) { return sizeof(DpSampleMCountState); } -static void DpSampleMCountInitialize(const AggregateFunction &, data_ptr_t state_ptr) { - memset(state_ptr, 0, sizeof(DpSampleMCountState)); +static void DpSampleMCountInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i]; + memset(state_ptr, 0, sizeof(DpSampleMCountState)); + } } static PAC_FLOAT *EnsureDpSampleMCountArray(DpSampleMCountState &state, ArenaAllocator &allocator, int sample_count) { @@ -496,35 +515,40 @@ static void DpSampleMCountUpdateOne(DpSampleMCountState &state, ArenaAllocator & counts[DpSassLaneIndex(hash, sample_count)] += PAC_FLOAT(1.0); } -static void DpSampleMCountUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - int sample_count = GetPrivSampleCount(aggr); - UnifiedVectorFormat hash_data; - inputs[0].ToUnifiedFormat(count, hash_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx)) { - DpSampleMCountUpdateOne(state, aggr.allocator, hashes[h_idx], sample_count); +static void DpSampleMCountUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + int sample_count = GetPrivSampleCount(aggr); + UnifiedVectorFormat hash_data; + inputs[0].ToUnifiedFormat(count, hash_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx)) { + DpSampleMCountUpdateOne(state, aggr.allocator, hashes[h_idx], sample_count); + } } - } + }); } -static void DpSampleMCountColumnUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, - idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - int sample_count = GetPrivSampleCount(aggr); - UnifiedVectorFormat hash_data, col_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, col_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto c_idx = col_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx) && col_data.validity.RowIsValid(c_idx)) { - DpSampleMCountUpdateOne(state, aggr.allocator, hashes[h_idx], sample_count); +static void DpSampleMCountColumnUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + int sample_count = GetPrivSampleCount(aggr); + UnifiedVectorFormat hash_data, col_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, col_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto c_idx = col_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx) && col_data.validity.RowIsValid(c_idx)) { + DpSampleMCountUpdateOne(state, aggr.allocator, hashes[h_idx], sample_count); + } } - } + }); } static void DpSampleMCountScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -562,30 +586,32 @@ static void DpSampleMCountColumnScatterUpdate(Vector inputs[], AggregateInputDat } } -static void DpSampleMCountIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, - idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - int sample_count = GetPrivSampleCount(aggr); - UnifiedVectorFormat hash_data, predicate_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, predicate_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto &predicate_child = ListVector::GetEntry(inputs[1]); - UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), predicate_child_data); - auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto p_idx = predicate_data.sel->get_index(i); - if (!hash_data.validity.RowIsValid(h_idx)) { - continue; - } - idx_t lane = DpSassLaneIndex(hashes[h_idx], sample_count); - if (DpSampleMPredicateLaneIsTrue(predicate_data, predicate_child_data, predicate_values, p_idx, lane)) { - auto counts = EnsureDpSampleMCountArray(state, aggr.allocator, sample_count); - counts[lane] += PAC_FLOAT(1.0); +static void DpSampleMCountIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + int sample_count = GetPrivSampleCount(aggr); + UnifiedVectorFormat hash_data, predicate_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, predicate_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto &predicate_child = ListVector::GetEntry(inputs[1]); + UnifiedVectorFormat predicate_child_data; + predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), predicate_child_data); + auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto p_idx = predicate_data.sel->get_index(i); + if (!hash_data.validity.RowIsValid(h_idx)) { + continue; + } + idx_t lane = DpSassLaneIndex(hashes[h_idx], sample_count); + if (DpSampleMPredicateLaneIsTrue(predicate_data, predicate_child_data, predicate_values, p_idx, lane)) { + auto counts = EnsureDpSampleMCountArray(state, aggr.allocator, sample_count); + counts[lane] += PAC_FLOAT(1.0); + } } - } + }); } static void DpSampleMCountIfScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, @@ -633,18 +659,18 @@ static void DpSampleMCountCombine(Vector &src, Vector &dst, AggregateInputData & } } -static void DpSampleMCountFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void DpSampleMCountFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { int sample_count = GetPrivSampleCount(input); auto state_ptrs = FlatVector::GetData(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * static_cast(sample_count); ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); bool sample_rescale = input.bind_data ? input.bind_data->Cast().sample_rescale : true; // A disjoint lane contains about 1/m of the PUs. Scaling COUNT by m turns each lane // answer into an estimator of the full-data answer; disabling it preserves subsample scale. @@ -664,12 +690,12 @@ static void DpSampleMCountFinalize(Vector &states, AggregateInputData &input, Ve } } -static void DpSampleMCountDistinctFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void DpSampleMCountDistinctFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { int sample_count = GetPrivSampleCount(input); auto state_ptrs = FlatVector::GetData(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * static_cast(sample_count); ListVector::Reserve(result, total_elements); @@ -678,7 +704,7 @@ static void DpSampleMCountDistinctFinalize(Vector &states, AggregateInputData &i // The compiler has already produced one row per (group, lane, distinct value). // Keep these as lane-level DISTINCT answers. Generic COUNT rescaling would instead // change the target to a full-data cardinality extrapolation. - auto child_data = FlatVector::GetData(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); for (idx_t i = 0; i < count; i++) { idx_t base = (offset + i) * static_cast(sample_count); list_entries[offset + i].offset = base; @@ -694,23 +720,25 @@ static void DpSampleMCountDistinctFinalize(Vector &states, AggregateInputData &i } } -static void DpSampleMCountDistinctIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, - idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - int sample_count = GetPrivSampleCount(aggr); - UnifiedVectorFormat hash_data, include_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, include_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto include_values = UnifiedVectorFormat::GetData(include_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto include_idx = include_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx) && include_data.validity.RowIsValid(include_idx) && - include_values[include_idx]) { - DpSampleMCountUpdateOne(state, aggr.allocator, hashes[h_idx], sample_count); +static void DpSampleMCountDistinctIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + int sample_count = GetPrivSampleCount(aggr); + UnifiedVectorFormat hash_data, include_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, include_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto include_values = UnifiedVectorFormat::GetData(include_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto include_idx = include_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx) && include_data.validity.RowIsValid(include_idx) && + include_values[include_idx]) { + DpSampleMCountUpdateOne(state, aggr.allocator, hashes[h_idx], sample_count); + } } - } + }); } static void DpSampleMCountDistinctIfScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, @@ -734,8 +762,8 @@ static void DpSampleMCountDistinctIfScatterUpdate(Vector inputs[], AggregateInpu } } -static unique_ptr DpSampleMCountBind(ClientContext &ctx, AggregateFunction &, - vector> &) { +static unique_ptr DpSampleMCountBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); int sample_lanes = GetDpSampleLanes(ctx); if (sample_lanes != 1) { throw InvalidInputException("as_sample_m_count: dp_sample_lanes must be 1 when dp_sass_m > 64"); @@ -797,12 +825,15 @@ struct DpSampleMCountDistinctValuesState { value_set_t *sets; }; -static idx_t DpSampleMCountDistinctValuesStateSize(const AggregateFunction &) { +static idx_t DpSampleMCountDistinctValuesStateSize(AggregateStateInput &) { return sizeof(DpSampleMCountDistinctValuesState); } -static void DpSampleMCountDistinctValuesInitialize(const AggregateFunction &, data_ptr_t state_ptr) { - memset(state_ptr, 0, sizeof(DpSampleMCountDistinctValuesState)); +static void DpSampleMCountDistinctValuesInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i]; + memset(state_ptr, 0, sizeof(DpSampleMCountDistinctValuesState)); + } } static value_set_t *EnsureDpSampleMCountDistinctValueSets(DpSampleMCountDistinctValuesState &state, @@ -840,25 +871,27 @@ static void DpSampleMCountDistinctValuesUpdateOne(DpSampleMCountDistinctValuesSt } } -static void DpSampleMCountDistinctValuesUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, - idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - int sample_count = GetPrivSampleCount(aggr); - UnifiedVectorFormat hash_data, list_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, list_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto &child_vec = ListVector::GetEntry(inputs[1]); - UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), child_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto list_idx = list_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx)) { - DpSampleMCountDistinctValuesUpdateOne(state, aggr.allocator, hashes[h_idx], inputs[1], list_data, - child_data, list_idx, sample_count); +static void DpSampleMCountDistinctValuesUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + int sample_count = GetPrivSampleCount(aggr); + UnifiedVectorFormat hash_data, list_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, list_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto &child_vec = ListVector::GetEntry(inputs[1]); + UnifiedVectorFormat child_data; + child_vec.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), child_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto list_idx = list_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx)) { + DpSampleMCountDistinctValuesUpdateOne(state, aggr.allocator, hashes[h_idx], inputs[1], list_data, + child_data, list_idx, sample_count); + } } - } + }); } static void DpSampleMCountDistinctValuesScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, @@ -903,19 +936,19 @@ static void DpSampleMCountDistinctValuesCombine(Vector &src, Vector &dst, Aggreg } } -static void DpSampleMCountDistinctValuesFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void DpSampleMCountDistinctValuesFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { int sample_count = GetPrivSampleCount(input); auto state_ptrs = FlatVector::GetData(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * static_cast(sample_count); ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); - auto &child_validity = FlatVector::Validity(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); + auto &child_validity = FlatVector::ValidityMutable(child_vec); child_validity.SetAllValid(total_elements); for (idx_t i = 0; i < count; i++) { idx_t base = (offset + i) * static_cast(sample_count); @@ -947,12 +980,14 @@ static void DpSampleMCountDistinctValuesDestroy(Vector &states, AggregateInputDa } } -static unique_ptr DpSampleMCountDistinctValuesBind(ClientContext &ctx, AggregateFunction &func, - vector> &args) { - if (args.size() != 2 || args[1]->return_type.id() != LogicalTypeId::LIST) { +static unique_ptr DpSampleMCountDistinctValuesBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &func = bind_input.GetBoundFunction(); + auto &args = bind_input.GetArguments(); + if (args.size() != 2 || args[1]->GetReturnType().id() != LogicalTypeId::LIST) { throw InvalidInputException("as_sample_m_count_distinct_values expects a LIST of per-PU distinct values"); } - return DpSampleMCountBind(ctx, func, args); + return DpSampleMCountBind(bind_input); } void RegisterDpSampleMCountDistinctValuesFunctions(ExtensionLoader &loader) { @@ -992,35 +1027,40 @@ static void PacSampleMCountUpdateOne(DpSampleMCountState &state, ArenaAllocator ForEachPacMSelectedLane(hash, bind, [&](idx_t lane) { counts[lane] += PAC_FLOAT(1.0); }); } -static void PacSampleMCountUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - auto &bind = aggr.bind_data->Cast(); - UnifiedVectorFormat hash_data; - inputs[0].ToUnifiedFormat(count, hash_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx)) { - PacSampleMCountUpdateOne(state, aggr.allocator, hashes[h_idx], bind); +static void PacSampleMCountUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + auto &bind = aggr.bind_data->Cast(); + UnifiedVectorFormat hash_data; + inputs[0].ToUnifiedFormat(count, hash_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx)) { + PacSampleMCountUpdateOne(state, aggr.allocator, hashes[h_idx], bind); + } } - } + }); } -static void PacSampleMCountColumnUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, - idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - auto &bind = aggr.bind_data->Cast(); - UnifiedVectorFormat hash_data, col_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, col_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto c_idx = col_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx) && col_data.validity.RowIsValid(c_idx)) { - PacSampleMCountUpdateOne(state, aggr.allocator, hashes[h_idx], bind); +static void PacSampleMCountColumnUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + auto &bind = aggr.bind_data->Cast(); + UnifiedVectorFormat hash_data, col_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, col_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto c_idx = col_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx) && col_data.validity.RowIsValid(c_idx)) { + PacSampleMCountUpdateOne(state, aggr.allocator, hashes[h_idx], bind); + } } - } + }); } static void PacSampleMCountScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, @@ -1059,31 +1099,33 @@ static void PacSampleMCountColumnScatterUpdate(Vector inputs[], AggregateInputDa } } -static void PacSampleMCountIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, - idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - auto &bind = aggr.bind_data->Cast(); - UnifiedVectorFormat hash_data, predicate_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, predicate_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto &predicate_child = ListVector::GetEntry(inputs[1]); - UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), predicate_child_data); - auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto p_idx = predicate_data.sel->get_index(i); - if (!hash_data.validity.RowIsValid(h_idx)) { - continue; - } - auto counts = EnsureDpSampleMCountArray(state, aggr.allocator, bind.sample_count); - ForEachPacMSelectedLane(hashes[h_idx], bind, [&](idx_t lane) { - if (DpSampleMPredicateLaneIsTrue(predicate_data, predicate_child_data, predicate_values, p_idx, lane)) { - counts[lane] += PAC_FLOAT(1.0); +static void PacSampleMCountIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + auto &bind = aggr.bind_data->Cast(); + UnifiedVectorFormat hash_data, predicate_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, predicate_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto &predicate_child = ListVector::GetEntry(inputs[1]); + UnifiedVectorFormat predicate_child_data; + predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), predicate_child_data); + auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto p_idx = predicate_data.sel->get_index(i); + if (!hash_data.validity.RowIsValid(h_idx)) { + continue; } - }); - } + auto counts = EnsureDpSampleMCountArray(state, aggr.allocator, bind.sample_count); + ForEachPacMSelectedLane(hashes[h_idx], bind, [&](idx_t lane) { + if (DpSampleMPredicateLaneIsTrue(predicate_data, predicate_child_data, predicate_values, p_idx, lane)) { + counts[lane] += PAC_FLOAT(1.0); + } + }); + } + }); } static void PacSampleMCountIfScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, @@ -1115,18 +1157,18 @@ static void PacSampleMCountIfScatterUpdate(Vector inputs[], AggregateInputData & } } -static void PacSampleMCountFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void PacSampleMCountFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { int sample_count = GetPrivSampleCount(input); auto state_ptrs = FlatVector::GetData(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * static_cast(sample_count); ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); for (idx_t i = 0; i < count; i++) { idx_t base = (offset + i) * static_cast(sample_count); list_entries[offset + i].offset = base; @@ -1142,8 +1184,8 @@ static void PacSampleMCountFinalize(Vector &states, AggregateInputData &input, V } } -static unique_ptr PacSampleMCountBind(ClientContext &ctx, AggregateFunction &, - vector> &) { +static unique_ptr PacSampleMCountBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); return MakePacSampleMBindData(ctx); } diff --git a/src/aggregates/as_finalize.cpp b/src/aggregates/as_finalize.cpp index e8e88fb7..aa8566f8 100644 --- a/src/aggregates/as_finalize.cpp +++ b/src/aggregates/as_finalize.cpp @@ -1,3 +1,4 @@ +#include "duckdb/common/vector/list_vector.hpp" #include "aggregates/as_aggregate.hpp" #include "duckdb/common/types/vector.hpp" #include "duckdb/function/scalar_function.hpp" @@ -28,8 +29,8 @@ static void PacFinalizeFunction(DataChunk &args, ExpressionState &state, Vector UnifiedVectorFormat list_data; list_vec.ToUnifiedFormat(count, list_data); - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); auto &child_vec = ListVector::GetEntry(list_vec); UnifiedVectorFormat child_data; @@ -66,7 +67,7 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader) { auto list_type = LogicalType::LIST(PacFloatLogicalType()); auto scalar_type = PacFloatLogicalType(); ScalarFunction priv_finalize("priv_finalize", {list_type}, scalar_type, PacFinalizeFunction); - priv_finalize.stability = FunctionStability::VOLATILE; + priv_finalize.SetStability(FunctionStability::VOLATILE); loader.RegisterFunction(priv_finalize); // Register implicit casts LIST → FLOAT and LIST → DOUBLE so the binder @@ -89,7 +90,7 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader) { // Write to a PAC_FLOAT temp buffer, then cast to the result type if needed vector tmp(count); - auto &result_validity = FlatVector::Validity(result); + auto &result_validity = FlatVector::ValidityMutable(result); for (idx_t i = 0; i < count; i++) { auto list_idx = list_data.sel->get_index(i); @@ -112,12 +113,12 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader) { // Copy to result — handles both FLOAT and DOUBLE result types if (result.GetType().id() == LogicalTypeId::FLOAT) { - auto result_data = FlatVector::GetData(result); + auto result_data = FlatVector::GetDataMutable(result); for (idx_t i = 0; i < count; i++) { result_data[i] = static_cast(tmp[i]); } } else { - auto result_data = FlatVector::GetData(result); + auto result_data = FlatVector::GetDataMutable(result); for (idx_t i = 0; i < count; i++) { result_data[i] = static_cast(tmp[i]); } @@ -133,16 +134,16 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader) { // The post-optimizer rewrites these comparisons to priv_filter_ calls before // execution, so the cast is never actually run in comparison contexts. auto scalar_to_list_cast = [](Vector &source, Vector &result, idx_t count, CastParameters ¶meters) -> bool { - auto &child_vec = ListVector::GetEntry(result); - auto child_data = FlatVector::GetData(child_vec); - auto list_entries = FlatVector::GetData(result); + auto &child_vec = ListVector::GetChildMutable(result); + auto child_data = FlatVector::GetDataMutable(child_vec); + auto list_entries = FlatVector::GetDataMutable(result); UnifiedVectorFormat source_data; source.ToUnifiedFormat(count, source_data); idx_t offset = 0; for (idx_t i = 0; i < count; i++) { auto src_idx = source_data.sel->get_index(i); if (!source_data.validity.RowIsValid(src_idx)) { - FlatVector::Validity(result).SetInvalid(i); + FlatVector::ValidityMutable(result).SetInvalid(i); list_entries[i] = {offset, 0}; continue; } diff --git a/src/aggregates/as_min_max.cpp b/src/aggregates/as_min_max.cpp index a8c073a5..75218dd3 100644 --- a/src/aggregates/as_min_max.cpp +++ b/src/aggregates/as_min_max.cpp @@ -1,3 +1,4 @@ +#include "duckdb/common/vector/list_vector.hpp" #include "aggregates/as_min_max.hpp" #include "categorical/pac_categorical.hpp" #include "duckdb/parser/parsed_data/create_aggregate_function_info.hpp" @@ -23,25 +24,28 @@ using MinMaxState = PacMinMaxStateWrapper; // Non-grouped update template -static void PacMinMaxUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, idx_t count) { - auto &agg = *reinterpret_cast *>(state_p); - int sample_lanes = GetPrivSampleLanes(aggr); - - UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto values = UnifiedVectorFormat::GetData(value_data); - - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx) && value_data.validity.RowIsValid(v_idx)) { - // min/max do not use buffering for global aggregates (without GROUP BY) because it was slower - PacMinMaxUpdateOne(agg, TransformPacUpdateHash(hashes[h_idx], sample_lanes), values[v_idx], - aggr.allocator); +static void PacMinMaxUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, const ClusteredAggr &clusters, + idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto &agg = *reinterpret_cast *>(state_p); + int sample_lanes = GetPrivSampleLanes(aggr); + + UnifiedVectorFormat hash_data, value_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, value_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto values = UnifiedVectorFormat::GetData(value_data); + + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto v_idx = value_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx) && value_data.validity.RowIsValid(v_idx)) { + // min/max do not use buffering for global aggregates (without GROUP BY) because it was slower + PacMinMaxUpdateOne(agg, TransformPacUpdateHash(hashes[h_idx], sample_lanes), values[v_idx], + aggr.allocator); + } } - } + }); } // Grouped (scatter) update @@ -95,10 +99,11 @@ static void PacMinMaxCombine(Vector &src, Vector &dst, AggregateInputData &aggr, } template -static void PacMinMaxFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +static void PacMinMaxFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, + idx_t offset) { auto state_ptrs = FlatVector::GetData *>(states); - auto data = FlatVector::GetData(result); - auto &result_mask = FlatVector::Validity(result); + auto data = FlatVector::GetDataMutable(result); + auto &result_mask = FlatVector::ValidityMutable(result); double mi = input.bind_data ? input.bind_data->Cast().mi : 0.0; double correction = input.bind_data ? input.bind_data->Cast().correction : 1.0; @@ -146,13 +151,16 @@ static void PacMinMaxFinalize(Vector &states, AggregateInputData &input, Vector // ============================================================================ template -static idx_t PacMinMaxStateSize(const AggregateFunction &) { +static idx_t PacMinMaxStateSize(AggregateStateInput &) { return sizeof(MinMaxState); } template -static void PacMinMaxInitialize(const AggregateFunction &, data_ptr_t p) { - memset(p, 0, sizeof(MinMaxState)); +static void PacMinMaxInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto p = states[i]; + memset(p, 0, sizeof(MinMaxState)); + } } // ============================================================================ @@ -160,23 +168,25 @@ static void PacMinMaxInitialize(const AggregateFunction &, data_ptr_t p) { // ============================================================================ template -static unique_ptr PacMinMaxBind(ClientContext &ctx, AggregateFunction &function, - vector> &args) { - auto &value_type = args[1]->return_type; +static unique_ptr PacMinMaxBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &function = bind_input.GetBoundFunction(); + auto &args = bind_input.GetArguments(); + auto &value_type = args[1]->GetReturnType(); auto physical_type = value_type.InternalType(); - function.return_type = value_type; - function.arguments[1] = value_type; + function.SetReturnType(value_type); + function.GetArguments()[1] = value_type; // Select implementation based on physical type #define BIND_TYPE(PHYS_TYPE, CPP_TYPE) \ case PhysicalType::PHYS_TYPE: \ - function.state_size = PacMinMaxStateSize; \ - function.initialize = PacMinMaxInitialize; \ - function.update = PacMinMaxScatterUpdate; \ - function.simple_update = PacMinMaxUpdate; \ - function.combine = PacMinMaxCombine; \ - function.finalize = PacMinMaxFinalize; \ + function.SetStateSizeCallback(PacMinMaxStateSize); \ + function.SetStateInitCallback(PacMinMaxInitialize); \ + function.SetStateUpdateCallback(PacMinMaxScatterUpdate); \ + function.SetStateClusterUpdateCallback(PacMinMaxUpdate); \ + function.SetStateCombineCallback(PacMinMaxCombine); \ + function.SetStateFinalizeCallback(PacMinMaxFinalize); \ break switch (physical_type) { @@ -205,20 +215,20 @@ static unique_ptr PacMinMaxBind(ClientContext &ctx, AggregateFunct // PAC_MIN/MAX_COUNTERS: Returns all 64 counters as LIST for categorical queries // ============================================================================ template -static void PacMinMaxFinalizeCounters(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void PacMinMaxFinalizeCounters(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData *>(states); // Result is LIST - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); // Reserve space for all lists (64 elements each) idx_t total_elements = (offset + count) * 64; ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); for (idx_t i = 0; i < count; i++) { #ifndef PAC_NOBUFFERING @@ -264,23 +274,25 @@ static void PacMinMaxFinalizeCounters(Vector &states, AggregateInputData &input, // Bind function for as_min/max_counters - similar to PacMinMaxBind but returns LIST template -static unique_ptr PacMinMaxCountersBind(ClientContext &ctx, AggregateFunction &function, - vector> &args) { - auto &value_type = args[1]->return_type; +static unique_ptr PacMinMaxCountersBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &function = bind_input.GetBoundFunction(); + auto &args = bind_input.GetArguments(); + auto &value_type = args[1]->GetReturnType(); auto physical_type = value_type.InternalType(); - function.return_type = LogicalType::LIST(PacFloatLogicalType()); - function.arguments[1] = value_type; + function.SetReturnType(LogicalType::LIST(PacFloatLogicalType())); + function.GetArguments()[1] = value_type; // Select implementation based on physical type #define BIND_COUNTERS_TYPE(PHYS_TYPE, CPP_TYPE) \ case PhysicalType::PHYS_TYPE: \ - function.state_size = PacMinMaxStateSize; \ - function.initialize = PacMinMaxInitialize; \ - function.update = PacMinMaxScatterUpdate; \ - function.simple_update = PacMinMaxUpdate; \ - function.combine = PacMinMaxCombine; \ - function.finalize = PacMinMaxFinalizeCounters; \ + function.SetStateSizeCallback(PacMinMaxStateSize); \ + function.SetStateInitCallback(PacMinMaxInitialize); \ + function.SetStateUpdateCallback(PacMinMaxScatterUpdate); \ + function.SetStateClusterUpdateCallback(PacMinMaxUpdate); \ + function.SetStateCombineCallback(PacMinMaxCombine); \ + function.SetStateFinalizeCallback(PacMinMaxFinalizeCounters); \ break switch (physical_type) { @@ -309,18 +321,18 @@ static unique_ptr PacMinMaxCountersBind(ClientContext &ctx, Aggreg // SASS MIN/MAX counters: Returns all 64 sample values as LIST // ============================================================================ template -static void DpSampleMinMaxFinalizeCounters(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void DpSampleMinMaxFinalizeCounters(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData *>(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * 64; ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); - auto &child_validity = FlatVector::Validity(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); + auto &child_validity = FlatVector::ValidityMutable(child_vec); child_validity.SetAllValid(total_elements); PAC_FLOAT buf[64]; double scale_divisor = input.bind_data ? input.bind_data->Cast().scale_divisor : 1.0; @@ -357,25 +369,27 @@ static void DpSampleMinMaxFinalizeCounters(Vector &states, AggregateInputData &i } template -static unique_ptr DpSampleMinMaxBind(ClientContext &ctx, AggregateFunction &function, - vector> &args) { - auto &value_type = args[1]->return_type; +static unique_ptr DpSampleMinMaxBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &function = bind_input.GetBoundFunction(); + auto &args = bind_input.GetArguments(); + auto &value_type = args[1]->GetReturnType(); auto physical_type = value_type.InternalType(); double scale_divisor = value_type.id() == LogicalTypeId::DECIMAL ? std::pow(10.0, static_cast(DecimalType::GetScale(value_type))) : 1.0; - function.return_type = LogicalType::LIST(PacFloatLogicalType()); - function.arguments[1] = value_type; + function.SetReturnType(LogicalType::LIST(PacFloatLogicalType())); + function.GetArguments()[1] = value_type; #define BIND_DP_SAMPLE_TYPE(PHYS_TYPE, CPP_TYPE) \ case PhysicalType::PHYS_TYPE: \ - function.state_size = PacMinMaxStateSize; \ - function.initialize = PacMinMaxInitialize; \ - function.update = PacMinMaxScatterUpdate; \ - function.simple_update = PacMinMaxUpdate; \ - function.combine = PacMinMaxCombine; \ - function.finalize = DpSampleMinMaxFinalizeCounters; \ + function.SetStateSizeCallback(PacMinMaxStateSize); \ + function.SetStateInitCallback(PacMinMaxInitialize); \ + function.SetStateUpdateCallback(PacMinMaxScatterUpdate); \ + function.SetStateClusterUpdateCallback(PacMinMaxUpdate); \ + function.SetStateCombineCallback(PacMinMaxCombine); \ + function.SetStateFinalizeCallback(DpSampleMinMaxFinalizeCounters); \ break switch (physical_type) { @@ -412,13 +426,16 @@ struct DpSampleMMinMaxState { }; template -static idx_t DpSampleMMinMaxStateSize(const AggregateFunction &) { +static idx_t DpSampleMMinMaxStateSize(AggregateStateInput &) { return sizeof(DpSampleMMinMaxState); } template -static void DpSampleMMinMaxInitialize(const AggregateFunction &, data_ptr_t state_ptr) { - memset(state_ptr, 0, sizeof(DpSampleMMinMaxState)); +static void DpSampleMMinMaxInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i]; + memset(state_ptr, 0, sizeof(DpSampleMMinMaxState)); + } } template @@ -443,21 +460,24 @@ static void DpSampleMMinMaxUpdateOne(DpSampleMMinMaxState &state, Are } template -static void DpSampleMMinMaxUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - auto &state = *reinterpret_cast *>(state_ptr); - int sample_count = GetPrivSampleCount(aggr); - UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto values = UnifiedVectorFormat::GetData(value_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx) && value_data.validity.RowIsValid(v_idx)) { - DpSampleMMinMaxUpdateOne(state, aggr.allocator, hashes[h_idx], values[v_idx], sample_count); +static void DpSampleMMinMaxUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast *>(state_ptr); + int sample_count = GetPrivSampleCount(aggr); + UnifiedVectorFormat hash_data, value_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, value_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto values = UnifiedVectorFormat::GetData(value_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto v_idx = value_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx) && value_data.validity.RowIsValid(v_idx)) { + DpSampleMMinMaxUpdateOne(state, aggr.allocator, hashes[h_idx], values[v_idx], sample_count); + } } - } + }); } template @@ -503,19 +523,19 @@ static void DpSampleMMinMaxCombine(Vector &src, Vector &dst, AggregateInputData } template -static void DpSampleMMinMaxFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void DpSampleMMinMaxFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { int sample_count = GetPrivSampleCount(input); auto state_ptrs = FlatVector::GetData *>(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * static_cast(sample_count); ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); - auto &child_validity = FlatVector::Validity(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); + auto &child_validity = FlatVector::ValidityMutable(child_vec); child_validity.SetAllValid(total_elements); double scale_divisor = input.bind_data ? input.bind_data->Cast().scale_divisor : 1.0; @@ -536,30 +556,32 @@ static void DpSampleMMinMaxFinalize(Vector &states, AggregateInputData &input, V } template -static unique_ptr DpSampleMMinMaxBind(ClientContext &ctx, AggregateFunction &function, - vector> &args) { +static unique_ptr DpSampleMMinMaxBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &function = bind_input.GetBoundFunction(); + auto &args = bind_input.GetArguments(); int sample_lanes = GetDpSampleLanes(ctx); if (sample_lanes != 1) { throw InvalidInputException("as_sample_m_%s: dp_sample_lanes must be 1 when dp_sass_m > 64", IS_MAX ? "max" : "min"); } - auto &value_type = args[1]->return_type; + auto &value_type = args[1]->GetReturnType(); auto physical_type = value_type.InternalType(); double scale_divisor = value_type.id() == LogicalTypeId::DECIMAL ? std::pow(10.0, static_cast(DecimalType::GetScale(value_type))) : 1.0; - function.return_type = LogicalType::LIST(PacFloatLogicalType()); - function.arguments[1] = value_type; + function.SetReturnType(LogicalType::LIST(PacFloatLogicalType())); + function.GetArguments()[1] = value_type; #define BIND_DP_SAMPLE_M_TYPE(PHYS_TYPE, CPP_TYPE) \ case PhysicalType::PHYS_TYPE: \ - function.state_size = DpSampleMMinMaxStateSize; \ - function.initialize = DpSampleMMinMaxInitialize; \ - function.update = DpSampleMMinMaxScatterUpdate; \ - function.simple_update = DpSampleMMinMaxUpdate; \ - function.combine = DpSampleMMinMaxCombine; \ - function.finalize = DpSampleMMinMaxFinalize; \ + function.SetStateSizeCallback(DpSampleMMinMaxStateSize); \ + function.SetStateInitCallback(DpSampleMMinMaxInitialize); \ + function.SetStateUpdateCallback(DpSampleMMinMaxScatterUpdate); \ + function.SetStateClusterUpdateCallback(DpSampleMMinMaxUpdate); \ + function.SetStateCombineCallback(DpSampleMMinMaxCombine); \ + function.SetStateFinalizeCallback(DpSampleMMinMaxFinalize); \ break switch (physical_type) { @@ -722,8 +744,8 @@ void RegisterDpSampleMMaxFunctions(ExtensionLoader &loader) { // Explicit template instantiations #define INST_ALL(T) \ - template void PacMinMaxUpdate(Vector[], AggregateInputData &, idx_t, data_ptr_t, idx_t); \ - template void PacMinMaxUpdate(Vector[], AggregateInputData &, idx_t, data_ptr_t, idx_t); \ + template void PacMinMaxUpdate(Vector[], AggregateInputData &, idx_t, const ClusteredAggr &, idx_t); \ + template void PacMinMaxUpdate(Vector[], AggregateInputData &, idx_t, const ClusteredAggr &, idx_t); \ template void PacMinMaxScatterUpdate(Vector[], AggregateInputData &, idx_t, Vector &, idx_t); \ template void PacMinMaxScatterUpdate(Vector[], AggregateInputData &, idx_t, Vector &, idx_t); \ template void PacMinMaxCombine(Vector &, Vector &, AggregateInputData &, idx_t); \ diff --git a/src/aggregates/as_sum.cpp b/src/aggregates/as_sum.cpp index a0633cd4..07f08014 100644 --- a/src/aggregates/as_sum.cpp +++ b/src/aggregates/as_sum.cpp @@ -1,3 +1,4 @@ +#include "duckdb/common/vector/list_vector.hpp" #include "aggregates/as_sum.hpp" #include "categorical/pac_categorical.hpp" #include "duckdb/common/types/decimal.hpp" @@ -426,10 +427,10 @@ AUTOVECTORIZE static void PacSumCombineDouble(Vector &src, Vector &dst, idx_t co // Unified Finalize for both int and double states template -void PacSumFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +void PacSumFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData(states); - auto data = FlatVector::GetData(result); - auto &result_mask = FlatVector::Validity(result); + auto data = FlatVector::GetDataMutable(result); + auto &result_mask = FlatVector::ValidityMutable(result); double mi = input.bind_data ? input.bind_data->Cast().mi : 0.0; double correction = input.bind_data ? input.bind_data->Cast().correction : 1.0; uint64_t query_hash = input.bind_data ? input.bind_data->Cast().query_hash : 0; @@ -513,11 +514,13 @@ void PacSumFinalize(Vector &states, AggregateInputData &input, Vector &result, i // Generate exact as_sum Update/ScatterUpdate functions via X-macros // ============================================================================ #define X(NAME, VALUE_T, INPUT_T, SIGNED) \ - void PacSumUpdate##NAME(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, data_ptr_t state_p, \ - idx_t count) { \ - auto &state = *reinterpret_cast *>(state_p); \ - PacSumUpdate, SIGNED, VALUE_T, INPUT_T>( \ - inputs, state, count, aggr_input_data.allocator, GetPrivSampleLanes(aggr_input_data)); \ + void PacSumUpdate##NAME(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, \ + const ClusteredAggr &clusters, idx_t count) { \ + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { \ + auto &state = *reinterpret_cast *>(state_p); \ + PacSumUpdate, SIGNED, VALUE_T, INPUT_T>( \ + inputs, state, count, aggr_input_data.allocator, GetPrivSampleLanes(aggr_input_data)); \ + }); \ } \ void PacSumScatterUpdate##NAME(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, Vector &states, \ idx_t count) { \ @@ -529,27 +532,37 @@ PAC_INT_TYPES_UNSIGNED #undef X // Double/Float/[U]HugeInt use double state (not generated by X-macro) -void PacSumUpdateHugeIntDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, data_ptr_t state_p, - idx_t count) { - auto &state = *reinterpret_cast(state_p); - PacSumUpdate(inputs, state, count, aggr_input_data.allocator, - GetPrivSampleLanes(aggr_input_data)); -} -void PacSumUpdateUHugeInt(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, data_ptr_t state_p, - idx_t count) { - auto &state = *reinterpret_cast(state_p); - PacSumUpdate(inputs, state, count, aggr_input_data.allocator, - GetPrivSampleLanes(aggr_input_data)); -} -void PacSumUpdateFloat(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, data_ptr_t state_p, idx_t count) { - auto &state = *reinterpret_cast(state_p); - PacSumUpdate(inputs, state, count, aggr_input_data.allocator, - GetPrivSampleLanes(aggr_input_data)); -} -void PacSumUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, data_ptr_t state_p, idx_t count) { - auto &state = *reinterpret_cast(state_p); - PacSumUpdate(inputs, state, count, aggr_input_data.allocator, - GetPrivSampleLanes(aggr_input_data)); +void PacSumUpdateHugeIntDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto &state = *reinterpret_cast(state_p); + PacSumUpdate(inputs, state, count, aggr_input_data.allocator, + GetPrivSampleLanes(aggr_input_data)); + }); +} +void PacSumUpdateUHugeInt(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto &state = *reinterpret_cast(state_p); + PacSumUpdate(inputs, state, count, aggr_input_data.allocator, + GetPrivSampleLanes(aggr_input_data)); + }); +} +void PacSumUpdateFloat(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto &state = *reinterpret_cast(state_p); + PacSumUpdate(inputs, state, count, aggr_input_data.allocator, + GetPrivSampleLanes(aggr_input_data)); + }); +} +void PacSumUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto &state = *reinterpret_cast(state_p); + PacSumUpdate(inputs, state, count, aggr_input_data.allocator, + GetPrivSampleLanes(aggr_input_data)); + }); } void PacSumScatterUpdateHugeIntDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, Vector &states, idx_t count) { @@ -572,11 +585,13 @@ void PacSumScatterUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_d GetPrivSampleLanes(aggr_input_data)); } -static void DpSampleSumUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, data_ptr_t state_p, - idx_t count) { - auto &state = *reinterpret_cast(state_p); - PacSumUpdate(inputs, state, count, aggr_input_data.allocator, - GetPrivSampleLanes(aggr_input_data)); +static void DpSampleSumUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto &state = *reinterpret_cast(state_p); + PacSumUpdate(inputs, state, count, aggr_input_data.allocator, + GetPrivSampleLanes(aggr_input_data)); + }); } static void DpSampleSumScatterUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, Vector &states, @@ -597,16 +612,19 @@ void PacSumCombineDoubleWrapper(Vector &src, Vector &dst, AggregateInputData &ag } // instantiate Finalize methods for as_sum -void PacSumFinalizeSigned(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +void PacSumFinalizeSigned(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, + idx_t offset) { PacSumFinalize, hugeint_t, true>(states, input, result, count, offset); } -void PacSumFinalizeUnsigned(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +void PacSumFinalizeUnsigned(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, + idx_t offset) { PacSumFinalize, hugeint_t, false>(states, input, result, count, offset); } -void PacSumFinalizeDoubleWrapper(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +void PacSumFinalizeDoubleWrapper(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, + idx_t offset) { PacSumFinalize(states, input, result, count, offset); } -void PacSumFinalizeDoubleToHugeint(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +void PacSumFinalizeDoubleToHugeint(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { PacSumFinalize(states, input, result, count, offset); } @@ -618,41 +636,49 @@ static unique_ptr PacBindWithScaleDivisor(ClientContext &ctx, vect } unique_ptr // Bind function for as_sum with optional mi parameter (must be constant) -PacSumBind(ClientContext &ctx, AggregateFunction &, vector> &args) { +PacSumBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &args = bind_input.GetArguments(); return PacBindWithScaleDivisor(ctx, args, 1.0); // scale_divisor=1.0 for sum } -static unique_ptr DpSampleSumBind(ClientContext &ctx, AggregateFunction &, - vector> &) { +static unique_ptr DpSampleSumBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); return MakeDpSampleBindData(ctx); } -idx_t PacSumIntStateSize(const AggregateFunction &) { +idx_t PacSumIntStateSize(AggregateStateInput &) { return sizeof(ScatterIntState); // signed/unsigned have same wrapper size } -void PacSumIntInitialize(const AggregateFunction &, data_ptr_t state_p) { - memset(state_p, 0, sizeof(ScatterIntState)); +void PacSumIntInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_p = states[i]; + memset(state_p, 0, sizeof(ScatterIntState)); #if !defined(PAC_EXACTSUM) && defined(PAC_NOBUFFERING) - // In APPROX mode without buffering, the raw state needs max_level_used=-1 and inline_level_idx=-1 - // to indicate no levels are allocated yet (memset sets them to 0, which is incorrect) - auto *state = reinterpret_cast *>(state_p); - state->max_level_used = -1; - state->inline_level_idx = -1; + // In APPROX mode without buffering, the raw state needs max_level_used=-1 and inline_level_idx=-1 + // to indicate no levels are allocated yet (memset sets them to 0, which is incorrect) + auto *state = reinterpret_cast *>(state_p); + state->max_level_used = -1; + state->inline_level_idx = -1; #endif + } } -idx_t PacSumDoubleStateSize(const AggregateFunction &) { +idx_t PacSumDoubleStateSize(AggregateStateInput &) { return sizeof(ScatterDoubleState); } -void PacSumDoubleInitialize(const AggregateFunction &, data_ptr_t state_ptr) { - memset(state_ptr, 0, sizeof(ScatterDoubleState)); +void PacSumDoubleInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i]; + memset(state_ptr, 0, sizeof(ScatterDoubleState)); + } } // Helper to register both 2-param and 3-param (with optional mi) versions static void AddFcn(AggregateFunctionSet &set, const LogicalType &value_type, const LogicalType &result_type, aggregate_size_t state_size, aggregate_initialize_t init, aggregate_update_t scatter, - aggregate_combine_t combine, aggregate_finalize_t finalize, aggregate_simple_update_t update, + aggregate_combine_t combine, aggregate_finalize_t finalize, aggregate_cluster_update_t update, aggregate_destructor_t destructor = nullptr) { set.AddFunction(AggregateFunction("as_noised_sum", {LogicalType::UBIGINT, value_type}, result_type, state_size, init, scatter, combine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, @@ -699,13 +725,15 @@ static AggregateFunction GetPacSumAggregate(PhysicalType type) { } // Dynamic dispatch for DECIMAL as_sum: selects the right integer implementation based on decimal width -unique_ptr BindDecimalPacSum(ClientContext &ctx, AggregateFunction &function, - vector> &args) { - auto decimal_type = args[1]->return_type; // value is arg 1 (arg 0 is hash) - function = GetPacSumAggregate(decimal_type.InternalType()); - function.name = "as_noised_sum"; - function.arguments[1] = decimal_type; - function.return_type = LogicalType::DECIMAL(Decimal::MAX_WIDTH_DECIMAL, DecimalType::GetScale(decimal_type)); +unique_ptr BindDecimalPacSum(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &function = bind_input.GetBoundFunction(); + auto &args = bind_input.GetArguments(); + auto decimal_type = args[1]->GetReturnType(); // value is arg 1 (arg 0 is hash) + function.ReplaceImplementation(GetPacSumAggregate(decimal_type.InternalType())); + function.SetName("as_noised_sum"); + function.GetArguments()[1] = decimal_type; + function.SetReturnType(LogicalType::DECIMAL(Decimal::MAX_WIDTH_DECIMAL, DecimalType::GetScale(decimal_type))); return PacBindWithScaleDivisor(ctx, args, 1.0); } @@ -786,21 +814,21 @@ enum class PacSumCountersFinalizeMode : uint8_t { PAC, DP_SAMPLE, DP_SAMPLE_NULL // Returns LIST with exactly 64 elements (no NULLs). // Position j is 0 if key_hash bit j is 0, otherwise value * 2 (to compensate for 50% sampling). template -static void PacSumAvgFinalizeCountersInternal(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacSumAvgFinalizeCountersInternal(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData(states); // Result is LIST - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); // Reserve space for all lists (64 elements each) idx_t total_elements = (offset + count) * 64; ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); - auto &child_validity = FlatVector::Validity(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); + auto &child_validity = FlatVector::ValidityMutable(child_vec); // correction factor for value scaling auto *bind = input.bind_data ? &input.bind_data->Cast() : nullptr; @@ -867,31 +895,32 @@ static void PacSumAvgFinalizeCountersInternal(Vector &states, AggregateInputData } template -void PacSumAvgFinalizeCounters(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +void PacSumAvgFinalizeCounters(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, + idx_t offset) { PacSumAvgFinalizeCountersInternal(states, input, result, count, offset); } // Instantiate counter finalize methods for as_sum -static void PacSumFinalizeCountersSigned(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void PacSumFinalizeCountersSigned(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { PacSumAvgFinalizeCounters, true>(states, input, result, count, offset); } -static void PacSumFinalizeCountersUnsigned(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void PacSumFinalizeCountersUnsigned(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacSumAvgFinalizeCounters, false>(states, input, result, count, offset); } -static void PacSumFinalizeCountersDouble(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void PacSumFinalizeCountersDouble(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { PacSumAvgFinalizeCounters(states, input, result, count, offset); } -static void DpSampleSumFinalizeCountersDouble(Vector &states, AggregateInputData &input, Vector &result, idx_t count, - idx_t offset) { +static void DpSampleSumFinalizeCountersDouble(Vector &states, AggregateFinalizeInputData &input, Vector &result, + idx_t count, idx_t offset) { PacSumAvgFinalizeCountersInternal( states, input, result, count, offset); } -static void DpSampleNullableSumFinalizeCountersDouble(Vector &states, AggregateInputData &input, Vector &result, +static void DpSampleNullableSumFinalizeCountersDouble(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { PacSumAvgFinalizeCountersInternal( states, input, result, count, offset); @@ -900,7 +929,7 @@ static void DpSampleNullableSumFinalizeCountersDouble(Vector &states, AggregateI // Helper to register both 2-param and 3-param versions for as_sum static void AddCountersFcn(AggregateFunctionSet &set, const LogicalType &value_type, aggregate_size_t state_size, aggregate_initialize_t init, aggregate_update_t scatter, aggregate_combine_t combine, - aggregate_finalize_t finalize, aggregate_simple_update_t update) { + aggregate_finalize_t finalize, aggregate_cluster_update_t update) { auto list_double_type = LogicalType::LIST(PacFloatLogicalType()); set.AddFunction(AggregateFunction("as_sum", {LogicalType::UBIGINT, value_type}, list_double_type, state_size, init, scatter, combine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, update, @@ -1004,12 +1033,15 @@ struct DpSampleAvgState { PAC_FLOAT counts[64]; }; -static idx_t DpSampleAvgStateSize(const AggregateFunction &) { +static idx_t DpSampleAvgStateSize(AggregateStateInput &) { return sizeof(DpSampleAvgState); } -static void DpSampleAvgInitialize(const AggregateFunction &, data_ptr_t state_ptr) { - memset(state_ptr, 0, sizeof(DpSampleAvgState)); +static void DpSampleAvgInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i]; + memset(state_ptr, 0, sizeof(DpSampleAvgState)); + } } static void DpSampleAvgUpdateOne(DpSampleAvgState &state, uint64_t key_hash, double sum_value, double count_value) { @@ -1021,30 +1053,33 @@ static void DpSampleAvgUpdateOne(DpSampleAvgState &state, uint64_t key_hash, dou } } -static void DpSampleAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - int sample_lanes = GetPrivSampleLanes(aggr); +static void DpSampleAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + int sample_lanes = GetPrivSampleLanes(aggr); - UnifiedVectorFormat hash_data, sum_data, count_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, sum_data); - inputs[2].ToUnifiedFormat(count, count_data); + UnifiedVectorFormat hash_data, sum_data, count_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, sum_data); + inputs[2].ToUnifiedFormat(count, count_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto sums = UnifiedVectorFormat::GetData(sum_data); - auto counts = UnifiedVectorFormat::GetData(count_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto sums = UnifiedVectorFormat::GetData(sum_data); + auto counts = UnifiedVectorFormat::GetData(count_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto s_idx = sum_data.sel->get_index(i); - auto c_idx = count_data.sel->get_index(i); - if (!hash_data.validity.RowIsValid(h_idx) || !sum_data.validity.RowIsValid(s_idx) || - !count_data.validity.RowIsValid(c_idx)) { - continue; + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto s_idx = sum_data.sel->get_index(i); + auto c_idx = count_data.sel->get_index(i); + if (!hash_data.validity.RowIsValid(h_idx) || !sum_data.validity.RowIsValid(s_idx) || + !count_data.validity.RowIsValid(c_idx)) { + continue; + } + auto key_hash = TransformPacUpdateHash(hashes[h_idx], sample_lanes); + DpSampleAvgUpdateOne(state, key_hash, sums[s_idx], counts[c_idx]); } - auto key_hash = TransformPacUpdateHash(hashes[h_idx], sample_lanes); - DpSampleAvgUpdateOne(state, key_hash, sums[s_idx], counts[c_idx]); - } + }); } static void DpSampleAvgScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -1088,17 +1123,18 @@ static void DpSampleAvgCombine(Vector &src, Vector &dst, AggregateInputData &, i } } -static void DpSampleAvgFinalize(Vector &states, AggregateInputData &, Vector &result, idx_t count, idx_t offset) { +static void DpSampleAvgFinalize(Vector &states, AggregateFinalizeInputData &, Vector &result, idx_t count, + idx_t offset) { auto state_ptrs = FlatVector::GetData(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * 64; ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); - auto &child_validity = FlatVector::Validity(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); + auto &child_validity = FlatVector::ValidityMutable(child_vec); child_validity.SetAllValid(total_elements); for (idx_t i = 0; i < count; i++) { @@ -1118,8 +1154,8 @@ static void DpSampleAvgFinalize(Vector &states, AggregateInputData &, Vector &re } } -static unique_ptr DpSampleAvgBind(ClientContext &ctx, AggregateFunction &, - vector> &) { +static unique_ptr DpSampleAvgBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); return MakeDpSampleBindData(ctx); } @@ -1149,20 +1185,26 @@ struct DpSampleMNullableSumState { uint8_t *occupied; }; -static idx_t DpSampleMSumStateSize(const AggregateFunction &) { +static idx_t DpSampleMSumStateSize(AggregateStateInput &) { return sizeof(DpSampleMSumState); } -static void DpSampleMSumInitialize(const AggregateFunction &, data_ptr_t state_ptr) { - memset(state_ptr, 0, sizeof(DpSampleMSumState)); +static void DpSampleMSumInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i]; + memset(state_ptr, 0, sizeof(DpSampleMSumState)); + } } -static idx_t DpSampleMNullableSumStateSize(const AggregateFunction &) { +static idx_t DpSampleMNullableSumStateSize(AggregateStateInput &) { return sizeof(DpSampleMNullableSumState); } -static void DpSampleMNullableSumInitialize(const AggregateFunction &, data_ptr_t state_ptr) { - memset(state_ptr, 0, sizeof(DpSampleMNullableSumState)); +static void DpSampleMNullableSumInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i]; + memset(state_ptr, 0, sizeof(DpSampleMNullableSumState)); + } } static PAC_FLOAT *EnsureDpSampleMArray(PAC_FLOAT *&values, ArenaAllocator &allocator, int sample_count) { @@ -1187,21 +1229,24 @@ static void DpSampleMSumUpdateOne(DpSampleMSumState &state, ArenaAllocator &allo values[DpSassLaneIndex(hash, sample_count)] += static_cast(value); } -static void DpSampleMSumUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - int sample_count = GetPrivSampleCount(aggr); - UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto values = UnifiedVectorFormat::GetData(value_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx) && value_data.validity.RowIsValid(v_idx)) { - DpSampleMSumUpdateOne(state, aggr.allocator, hashes[h_idx], values[v_idx], sample_count); +static void DpSampleMSumUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + int sample_count = GetPrivSampleCount(aggr); + UnifiedVectorFormat hash_data, value_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, value_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto values = UnifiedVectorFormat::GetData(value_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto v_idx = value_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx) && value_data.validity.RowIsValid(v_idx)) { + DpSampleMSumUpdateOne(state, aggr.allocator, hashes[h_idx], values[v_idx], sample_count); + } } - } + }); } static void DpSampleMSumScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -1232,22 +1277,24 @@ static void DpSampleMNullableSumUpdateOne(DpSampleMNullableSumState &state, Aren occupied[lane] = 1; } -static void DpSampleMNullableSumUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, - idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - int sample_count = GetPrivSampleCount(aggr); - UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto values = UnifiedVectorFormat::GetData(value_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx) && value_data.validity.RowIsValid(v_idx)) { - DpSampleMNullableSumUpdateOne(state, aggr.allocator, hashes[h_idx], values[v_idx], sample_count); +static void DpSampleMNullableSumUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + int sample_count = GetPrivSampleCount(aggr); + UnifiedVectorFormat hash_data, value_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, value_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto values = UnifiedVectorFormat::GetData(value_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto v_idx = value_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx) && value_data.validity.RowIsValid(v_idx)) { + DpSampleMNullableSumUpdateOne(state, aggr.allocator, hashes[h_idx], values[v_idx], sample_count); + } } - } + }); } static void DpSampleMNullableSumScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, @@ -1270,32 +1317,35 @@ static void DpSampleMNullableSumScatterUpdate(Vector inputs[], AggregateInputDat } } -static void DpSampleMSumIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - int sample_count = GetPrivSampleCount(aggr); - UnifiedVectorFormat hash_data, value_data, predicate_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - inputs[2].ToUnifiedFormat(count, predicate_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto values = UnifiedVectorFormat::GetData(value_data); - auto &predicate_child = ListVector::GetEntry(inputs[2]); - UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[2]), predicate_child_data); - auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); - auto p_idx = predicate_data.sel->get_index(i); - if (!hash_data.validity.RowIsValid(h_idx) || !value_data.validity.RowIsValid(v_idx)) { - continue; - } - idx_t lane = DpSassLaneIndex(hashes[h_idx], sample_count); - if (DpSampleMPredicateLaneIsTrue(predicate_data, predicate_child_data, predicate_values, p_idx, lane)) { - auto values_ptr = EnsureDpSampleMArray(state.values, aggr.allocator, sample_count); - values_ptr[lane] += static_cast(values[v_idx]); +static void DpSampleMSumIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + int sample_count = GetPrivSampleCount(aggr); + UnifiedVectorFormat hash_data, value_data, predicate_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, value_data); + inputs[2].ToUnifiedFormat(count, predicate_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto values = UnifiedVectorFormat::GetData(value_data); + auto &predicate_child = ListVector::GetEntry(inputs[2]); + UnifiedVectorFormat predicate_child_data; + predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[2]), predicate_child_data); + auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto v_idx = value_data.sel->get_index(i); + auto p_idx = predicate_data.sel->get_index(i); + if (!hash_data.validity.RowIsValid(h_idx) || !value_data.validity.RowIsValid(v_idx)) { + continue; + } + idx_t lane = DpSassLaneIndex(hashes[h_idx], sample_count); + if (DpSampleMPredicateLaneIsTrue(predicate_data, predicate_child_data, predicate_values, p_idx, lane)) { + auto values_ptr = EnsureDpSampleMArray(state.values, aggr.allocator, sample_count); + values_ptr[lane] += static_cast(values[v_idx]); + } } - } + }); } static void DpSampleMSumIfScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -1366,17 +1416,18 @@ static void DpSampleMNullableSumCombine(Vector &src, Vector &dst, AggregateInput } } -static void DpSampleMSumFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +static void DpSampleMSumFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, + idx_t offset) { int sample_count = GetPrivSampleCount(input); auto state_ptrs = FlatVector::GetData(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * static_cast(sample_count); ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); bool sample_rescale = input.bind_data ? input.bind_data->Cast().sample_rescale : true; // SUM is extensive: multiply disjoint 1/m-sample totals by m only when the caller // wants a full-data estimator. AVG and MIN/MAX are not rescaled. @@ -1396,17 +1447,17 @@ static void DpSampleMSumFinalize(Vector &states, AggregateInputData &input, Vect } } -static void DpSampleMNullableSumFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void DpSampleMNullableSumFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { int sample_count = GetPrivSampleCount(input); auto state_ptrs = FlatVector::GetData(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * static_cast(sample_count); ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); - auto &child_validity = FlatVector::Validity(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); + auto &child_validity = FlatVector::ValidityMutable(child_vec); child_validity.SetAllValid(total_elements); bool sample_rescale = input.bind_data ? input.bind_data->Cast().sample_rescale : true; PAC_FLOAT rescale = sample_rescale ? static_cast(sample_count) : PAC_FLOAT(1.0); @@ -1426,8 +1477,8 @@ static void DpSampleMNullableSumFinalize(Vector &states, AggregateInputData &inp } } -static unique_ptr DpSampleMBind(ClientContext &ctx, AggregateFunction &, - vector> &) { +static unique_ptr DpSampleMBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); int sample_lanes = GetDpSampleLanes(ctx); if (sample_lanes != 1) { throw InvalidInputException("as_sample_m_*: dp_sample_lanes must be 1 when dp_sass_m > 64"); @@ -1482,12 +1533,15 @@ struct DpSampleMAvgState { PAC_FLOAT *counts; }; -static idx_t DpSampleMAvgStateSize(const AggregateFunction &) { +static idx_t DpSampleMAvgStateSize(AggregateStateInput &) { return sizeof(DpSampleMAvgState); } -static void DpSampleMAvgInitialize(const AggregateFunction &, data_ptr_t state_ptr) { - memset(state_ptr, 0, sizeof(DpSampleMAvgState)); +static void DpSampleMAvgInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i]; + memset(state_ptr, 0, sizeof(DpSampleMAvgState)); + } } static void DpSampleMAvgUpdateOne(DpSampleMAvgState &state, ArenaAllocator &allocator, uint64_t hash, double sum_value, @@ -1499,25 +1553,28 @@ static void DpSampleMAvgUpdateOne(DpSampleMAvgState &state, ArenaAllocator &allo counts[lane] += static_cast(count_value); } -static void DpSampleMAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - int sample_count = GetPrivSampleCount(aggr); - UnifiedVectorFormat hash_data, sum_data, count_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, sum_data); - inputs[2].ToUnifiedFormat(count, count_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto sums = UnifiedVectorFormat::GetData(sum_data); - auto counts = UnifiedVectorFormat::GetData(count_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto s_idx = sum_data.sel->get_index(i); - auto c_idx = count_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx) && sum_data.validity.RowIsValid(s_idx) && - count_data.validity.RowIsValid(c_idx)) { - DpSampleMAvgUpdateOne(state, aggr.allocator, hashes[h_idx], sums[s_idx], counts[c_idx], sample_count); +static void DpSampleMAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + int sample_count = GetPrivSampleCount(aggr); + UnifiedVectorFormat hash_data, sum_data, count_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, sum_data); + inputs[2].ToUnifiedFormat(count, count_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto sums = UnifiedVectorFormat::GetData(sum_data); + auto counts = UnifiedVectorFormat::GetData(count_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto s_idx = sum_data.sel->get_index(i); + auto c_idx = count_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx) && sum_data.validity.RowIsValid(s_idx) && + count_data.validity.RowIsValid(c_idx)) { + DpSampleMAvgUpdateOne(state, aggr.allocator, hashes[h_idx], sums[s_idx], counts[c_idx], sample_count); + } } - } + }); } static void DpSampleMAvgScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -1562,11 +1619,12 @@ static void DpSampleMAvgCombine(Vector &src, Vector &dst, AggregateInputData &ag } } -static void DpSampleMAvgFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +static void DpSampleMAvgFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, + idx_t offset) { int sample_count = GetPrivSampleCount(input); auto state_ptrs = FlatVector::GetData(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * static_cast(sample_count); ListVector::Reserve(result, total_elements); @@ -1574,8 +1632,8 @@ static void DpSampleMAvgFinalize(Vector &states, AggregateInputData &input, Vect // Form each lane's ratio before the SAA terminal combines lanes. The common m factor // would cancel between numerator and denominator, so AVG deliberately ignores rescaling. - auto child_data = FlatVector::GetData(child_vec); - auto &child_validity = FlatVector::Validity(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); + auto &child_validity = FlatVector::ValidityMutable(child_vec); child_validity.SetAllValid(total_elements); for (idx_t i = 0; i < count; i++) { auto *state = state_ptrs[i]; @@ -1614,21 +1672,24 @@ static void PacSampleMSumUpdateOne(DpSampleMSumState &state, ArenaAllocator &all ForEachPacMSelectedLane(hash, bind, [&](idx_t lane) { values[lane] += static_cast(value); }); } -static void PacSampleMSumUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - auto &bind = aggr.bind_data->Cast(); - UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto values = UnifiedVectorFormat::GetData(value_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx) && value_data.validity.RowIsValid(v_idx)) { - PacSampleMSumUpdateOne(state, aggr.allocator, hashes[h_idx], values[v_idx], bind); +static void PacSampleMSumUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + auto &bind = aggr.bind_data->Cast(); + UnifiedVectorFormat hash_data, value_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, value_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto values = UnifiedVectorFormat::GetData(value_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto v_idx = value_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx) && value_data.validity.RowIsValid(v_idx)) { + PacSampleMSumUpdateOne(state, aggr.allocator, hashes[h_idx], values[v_idx], bind); + } } - } + }); } static void PacSampleMSumScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -1650,33 +1711,36 @@ static void PacSampleMSumScatterUpdate(Vector inputs[], AggregateInputData &aggr } } -static void PacSampleMSumIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - auto &bind = aggr.bind_data->Cast(); - UnifiedVectorFormat hash_data, value_data, predicate_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - inputs[2].ToUnifiedFormat(count, predicate_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto values = UnifiedVectorFormat::GetData(value_data); - auto &predicate_child = ListVector::GetEntry(inputs[2]); - UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[2]), predicate_child_data); - auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); - auto p_idx = predicate_data.sel->get_index(i); - if (!hash_data.validity.RowIsValid(h_idx) || !value_data.validity.RowIsValid(v_idx)) { - continue; - } - auto values_ptr = EnsureDpSampleMArray(state.values, aggr.allocator, bind.sample_count); - ForEachPacMSelectedLane(hashes[h_idx], bind, [&](idx_t lane) { - if (DpSampleMPredicateLaneIsTrue(predicate_data, predicate_child_data, predicate_values, p_idx, lane)) { - values_ptr[lane] += static_cast(values[v_idx]); +static void PacSampleMSumIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + auto &bind = aggr.bind_data->Cast(); + UnifiedVectorFormat hash_data, value_data, predicate_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, value_data); + inputs[2].ToUnifiedFormat(count, predicate_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto values = UnifiedVectorFormat::GetData(value_data); + auto &predicate_child = ListVector::GetEntry(inputs[2]); + UnifiedVectorFormat predicate_child_data; + predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[2]), predicate_child_data); + auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto v_idx = value_data.sel->get_index(i); + auto p_idx = predicate_data.sel->get_index(i); + if (!hash_data.validity.RowIsValid(h_idx) || !value_data.validity.RowIsValid(v_idx)) { + continue; } - }); - } + auto values_ptr = EnsureDpSampleMArray(state.values, aggr.allocator, bind.sample_count); + ForEachPacMSelectedLane(hashes[h_idx], bind, [&](idx_t lane) { + if (DpSampleMPredicateLaneIsTrue(predicate_data, predicate_child_data, predicate_values, p_idx, lane)) { + values_ptr[lane] += static_cast(values[v_idx]); + } + }); + } + }); } static void PacSampleMSumIfScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, @@ -1711,18 +1775,18 @@ static void PacSampleMSumIfScatterUpdate(Vector inputs[], AggregateInputData &ag } } -static void PacSampleMSumFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void PacSampleMSumFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { int sample_count = GetPrivSampleCount(input); auto state_ptrs = FlatVector::GetData(states); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); idx_t total_elements = (offset + count) * static_cast(sample_count); ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); for (idx_t i = 0; i < count; i++) { idx_t base = (offset + i) * static_cast(sample_count); list_entries[offset + i].offset = base; @@ -1748,25 +1812,28 @@ static void PacSampleMAvgUpdateOne(DpSampleMAvgState &state, ArenaAllocator &all }); } -static void PacSampleMAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_ptr, idx_t count) { - auto &state = *reinterpret_cast(state_ptr); - auto &bind = aggr.bind_data->Cast(); - UnifiedVectorFormat hash_data, sum_data, count_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, sum_data); - inputs[2].ToUnifiedFormat(count, count_data); - auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto sums = UnifiedVectorFormat::GetData(sum_data); - auto counts = UnifiedVectorFormat::GetData(count_data); - for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto s_idx = sum_data.sel->get_index(i); - auto c_idx = count_data.sel->get_index(i); - if (hash_data.validity.RowIsValid(h_idx) && sum_data.validity.RowIsValid(s_idx) && - count_data.validity.RowIsValid(c_idx)) { - PacSampleMAvgUpdateOne(state, aggr.allocator, hashes[h_idx], sums[s_idx], counts[c_idx], bind); +static void PacSampleMAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { + auto &state = *reinterpret_cast(state_ptr); + auto &bind = aggr.bind_data->Cast(); + UnifiedVectorFormat hash_data, sum_data, count_data; + inputs[0].ToUnifiedFormat(count, hash_data); + inputs[1].ToUnifiedFormat(count, sum_data); + inputs[2].ToUnifiedFormat(count, count_data); + auto hashes = UnifiedVectorFormat::GetData(hash_data); + auto sums = UnifiedVectorFormat::GetData(sum_data); + auto counts = UnifiedVectorFormat::GetData(count_data); + for (idx_t i = 0; i < count; i++) { + auto h_idx = hash_data.sel->get_index(i); + auto s_idx = sum_data.sel->get_index(i); + auto c_idx = count_data.sel->get_index(i); + if (hash_data.validity.RowIsValid(h_idx) && sum_data.validity.RowIsValid(s_idx) && + count_data.validity.RowIsValid(c_idx)) { + PacSampleMAvgUpdateOne(state, aggr.allocator, hashes[h_idx], sums[s_idx], counts[c_idx], bind); + } } - } + }); } static void PacSampleMAvgScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -1792,8 +1859,8 @@ static void PacSampleMAvgScatterUpdate(Vector inputs[], AggregateInputData &aggr } } -static unique_ptr PacSampleMBind(ClientContext &ctx, AggregateFunction &, - vector> &) { +static unique_ptr PacSampleMBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); return MakePacSampleMBindData(ctx); } diff --git a/src/aggregates/dp_approx_bounds.cpp b/src/aggregates/dp_approx_bounds.cpp index c936a61f..52a9f2f7 100644 --- a/src/aggregates/dp_approx_bounds.cpp +++ b/src/aggregates/dp_approx_bounds.cpp @@ -1,3 +1,5 @@ +#include "duckdb/common/vector/struct_vector.hpp" +#include "duckdb/common/vector/list_vector.hpp" #include "aggregates/dp_approx_bounds.hpp" #include "aggregates/as_clip_aggr.hpp" @@ -111,7 +113,7 @@ DpApproxBoundsParameters GetDpApproxBoundsParameters(const LogicalType &type) { } static double EvaluateConfig(ClientContext &context, const Expression &expression, const string &function_name, - const string &name) { + const Identifier &name) { if (!expression.IsFoldable()) { throw InvalidInputException(function_name + ": " + name + " must be a constant"); } @@ -128,32 +130,41 @@ static idx_t EvaluateBinCount(ClientContext &context, const Expression &expressi return static_cast(value); } -static unique_ptr BindDpApproxBounds(ClientContext &context, AggregateFunction &function, - vector> &arguments) { +static unique_ptr BindDpApproxBounds(BindAggregateFunctionInput &bind_input) { + auto &context = bind_input.GetClientContext(); + auto &function = bind_input.GetBoundFunction(); + auto &arguments = bind_input.GetArguments(); if (arguments.size() != 5 && arguments.size() != 6) { - throw InternalException(function.name + ": invalid internal argument count"); + throw InternalException(function.GetName().GetIdentifierName() + ": invalid internal argument count"); } - double epsilon = EvaluateConfig(context, *arguments[1], function.name, "epsilon"); - double max_groups = EvaluateConfig(context, *arguments[2], function.name, "max groups"); + double epsilon = EvaluateConfig(context, *arguments[1], function.GetName().GetIdentifierName(), "epsilon"); + double max_groups = EvaluateConfig(context, *arguments[2], function.GetName().GetIdentifierName(), "max groups"); double max_contributions = - arguments.size() == 6 ? EvaluateConfig(context, *arguments[3], function.name, "max contributions") : 1.0; + arguments.size() == 6 + ? EvaluateConfig(context, *arguments[3], function.GetName().GetIdentifierName(), "max contributions") + : 1.0; idx_t profile_offset = arguments.size() == 6 ? 4 : 3; - double scale = EvaluateConfig(context, *arguments[profile_offset], function.name, "scale"); - idx_t num_bins = EvaluateBinCount(context, *arguments[profile_offset + 1], function.name); + double scale = EvaluateConfig(context, *arguments[profile_offset], function.GetName().GetIdentifierName(), "scale"); + idx_t num_bins = EvaluateBinCount(context, *arguments[profile_offset + 1], function.GetName().GetIdentifierName()); if (!std::isfinite(epsilon) || epsilon <= 0.0) { - throw InvalidInputException(function.name + ": epsilon must be a positive finite number"); + throw InvalidInputException(function.GetName().GetIdentifierName() + + ": epsilon must be a positive finite number"); } if (!std::isfinite(max_groups) || max_groups <= 0.0) { - throw InvalidInputException(function.name + ": max groups must be a positive finite number"); + throw InvalidInputException(function.GetName().GetIdentifierName() + + ": max groups must be a positive finite number"); } if (!std::isfinite(max_contributions) || max_contributions <= 0.0) { - throw InvalidInputException(function.name + ": max contributions must be a positive finite number"); + throw InvalidInputException(function.GetName().GetIdentifierName() + + ": max contributions must be a positive finite number"); } if (arguments.size() == 6 && std::round(max_contributions) != max_contributions) { - throw InvalidInputException(function.name + ": max contributions must be integer-valued"); + throw InvalidInputException(function.GetName().GetIdentifierName() + + ": max contributions must be integer-valued"); } if (!std::isfinite(scale) || scale <= 0.0) { - throw InvalidInputException(function.name + ": scale must be a positive finite number"); + throw InvalidInputException(function.GetName().GetIdentifierName() + + ": scale must be a positive finite number"); } auto result = make_uniq(); result->epsilon = epsilon; @@ -165,12 +176,15 @@ static unique_ptr BindDpApproxBounds(ClientContext &context, Aggre return std::move(result); } -static idx_t DpApproxBoundsStateSize(const AggregateFunction &) { +static idx_t DpApproxBoundsStateSize(AggregateStateInput &) { return sizeof(DpApproxBoundsState); } -static void DpApproxBoundsInitialize(const AggregateFunction &, data_ptr_t state_p) { - memset(state_p, 0, sizeof(DpApproxBoundsState)); +static void DpApproxBoundsInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_p = states[i]; + memset(state_p, 0, sizeof(DpApproxBoundsState)); + } } static DpApproxBoundsBin *EnsureBins(DpApproxBoundsBin *&bins, idx_t num_bins, ArenaAllocator &allocator) { @@ -249,9 +263,12 @@ static void UpdateRows(Vector inputs[], AggregateInputData &input, idx_t count, } } -static void DpApproxBoundsUpdate(Vector inputs[], AggregateInputData &input, idx_t, data_ptr_t state_p, idx_t count) { - auto state = reinterpret_cast(state_p); - UpdateRows(inputs, input, count, [state](idx_t) { return state; }); +static void DpApproxBoundsUpdate(Vector inputs[], AggregateInputData &input, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto state = reinterpret_cast(state_p); + UpdateRows(inputs, input, count, [state](idx_t) { return state; }); + }); } static void DpApproxBoundsScatterUpdate(Vector inputs[], AggregateInputData &input, idx_t, Vector &states, @@ -475,8 +492,9 @@ struct DpBoundedValueListState { idx_t size; }; -static unique_ptr BindDpBoundedValueList(ClientContext &context, AggregateFunction &, - vector> &arguments) { +static unique_ptr BindDpBoundedValueList(BindAggregateFunctionInput &bind_input) { + auto &context = bind_input.GetClientContext(); + auto &arguments = bind_input.GetArguments(); if (arguments.size() != 2 || !arguments[1]->IsFoldable()) { throw InvalidInputException("dp_bounded_value_list: max values must be a constant"); } @@ -493,12 +511,15 @@ static unique_ptr BindDpBoundedValueList(ClientContext &context, A return std::move(result); } -static idx_t DpBoundedValueListStateSize(const AggregateFunction &) { +static idx_t DpBoundedValueListStateSize(AggregateStateInput &) { return sizeof(DpBoundedValueListState); } -static void DpBoundedValueListInitialize(const AggregateFunction &, data_ptr_t state_p) { - memset(state_p, 0, sizeof(DpBoundedValueListState)); +static void DpBoundedValueListInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_p = states[i]; + memset(state_p, 0, sizeof(DpBoundedValueListState)); + } } static uint64_t BoundedValueScore(double value) { @@ -567,10 +588,12 @@ static void UpdateBoundedValueRows(Vector inputs[], AggregateInputData &input, i } } -static void DpBoundedValueListUpdate(Vector inputs[], AggregateInputData &input, idx_t, data_ptr_t state_p, - idx_t count) { - auto state = reinterpret_cast(state_p); - UpdateBoundedValueRows(inputs, input, count, [state](idx_t) { return state; }); +static void DpBoundedValueListUpdate(Vector inputs[], AggregateInputData &input, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto state = reinterpret_cast(state_p); + UpdateBoundedValueRows(inputs, input, count, [state](idx_t) { return state; }); + }); } static void DpBoundedValueListScatterUpdate(Vector inputs[], AggregateInputData &input, idx_t, Vector &states, @@ -592,18 +615,18 @@ static void DpBoundedValueListCombine(Vector &source, Vector &target, AggregateI } } -static void DpBoundedValueListFinalize(Vector &states, AggregateInputData &, Vector &result, idx_t count, +static void DpBoundedValueListFinalize(Vector &states, AggregateFinalizeInputData &, Vector &result, idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData(states); - auto entries = FlatVector::GetData(result); - auto &child = ListVector::GetEntry(result); + auto entries = FlatVector::GetDataMutable(result); + auto &child = ListVector::GetChildMutable(result); idx_t total = ListVector::GetListSize(result); for (idx_t i = 0; i < count; i++) { total += state_ptrs[i]->size; } ListVector::Reserve(result, total); ListVector::SetListSize(result, total); - auto output = FlatVector::GetData(child); + auto output = FlatVector::GetDataMutable(child); idx_t cursor = total; for (idx_t i = count; i > 0; i--) { auto *state = state_ptrs[i - 1]; @@ -730,14 +753,14 @@ static LogicalType DpApproxBoundsDebugType() { static void WriteDebug(Vector &result, idx_t row, const DpApproxBoundsResult &value) { auto &children = StructVector::GetEntries(result); - FlatVector::GetData(*children[0])[row] = value.success; - FlatVector::GetData(*children[1])[row] = value.lower_bound; - FlatVector::GetData(*children[2])[row] = value.upper_bound; - FlatVector::GetData(*children[3])[row] = value.clipped_value; - FlatVector::GetData(*children[4])[row] = value.noise_scale; - FlatVector::GetData(*children[5])[row] = value.selected_bin; - FlatVector::GetData(*children[6])[row] = value.attempts; - FlatVector::GetData(*children[7])[row] = value.contributions; + FlatVector::GetDataMutable(children[0])[row] = value.success; + FlatVector::GetDataMutable(children[1])[row] = value.lower_bound; + FlatVector::GetDataMutable(children[2])[row] = value.upper_bound; + FlatVector::GetDataMutable(children[3])[row] = value.clipped_value; + FlatVector::GetDataMutable(children[4])[row] = value.noise_scale; + FlatVector::GetDataMutable(children[5])[row] = value.selected_bin; + FlatVector::GetDataMutable(children[6])[row] = value.attempts; + FlatVector::GetDataMutable(children[7])[row] = value.contributions; } template @@ -771,10 +794,12 @@ static void UpdateMeanRows(Vector inputs[], AggregateInputData &input, idx_t cou } } -static void DpApproxBoundsMeanUpdate(Vector inputs[], AggregateInputData &input, idx_t, data_ptr_t state_p, - idx_t count) { - auto state = reinterpret_cast(state_p); - UpdateMeanRows(inputs, input, count, [state](idx_t) { return state; }); +static void DpApproxBoundsMeanUpdate(Vector inputs[], AggregateInputData &input, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto state = reinterpret_cast(state_p); + UpdateMeanRows(inputs, input, count, [state](idx_t) { return state; }); + }); } static void DpApproxBoundsMeanScatterUpdate(Vector inputs[], AggregateInputData &input, idx_t, Vector &states, @@ -789,15 +814,15 @@ static double ClampMean(double value, double lower, double upper) { return std::max(lower, std::min(upper, value)); } -static void DpApproxBoundsMeanFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void DpApproxBoundsMeanFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData(states); auto &bind = input.bind_data->Cast(); - auto result_data = FlatVector::GetData(result); + auto result_data = FlatVector::GetDataMutable(result); for (idx_t i = 0; i < count; i++) { auto bounds = FinalizeState(*state_ptrs[i], bind, false); if (!bounds.success) { - FlatVector::Validity(result).SetInvalid(offset + i); + FlatVector::ValidityMutable(result).SetInvalid(offset + i); continue; } // A point interval has zero sensitivity but cannot define the normalized mean. Widen it by one @@ -838,11 +863,11 @@ static void DpApproxBoundsMeanFinalize(Vector &states, AggregateInputData &input } template -static void DpApproxBoundsFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void DpApproxBoundsFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData(states); auto &bind = input.bind_data->Cast(); - auto result_data = DEBUG ? nullptr : FlatVector::GetData(result); + auto result_data = DEBUG ? nullptr : FlatVector::GetDataMutable(result); for (idx_t i = 0; i < count; i++) { auto value = FinalizeState(*state_ptrs[i], bind, true); if (DEBUG) { @@ -850,7 +875,7 @@ static void DpApproxBoundsFinalize(Vector &states, AggregateInputData &input, Ve continue; } if (!value.success) { - FlatVector::Validity(result).SetInvalid(offset + i); + FlatVector::ValidityMutable(result).SetInvalid(offset + i); continue; } result_data[offset + i] = @@ -858,10 +883,10 @@ static void DpApproxBoundsFinalize(Vector &states, AggregateInputData &input, Ve } } -static AggregateFunction MakeDpApproxBoundsFunction(const string &name, const LogicalType &return_type, +static AggregateFunction MakeDpApproxBoundsFunction(const Identifier &name, const LogicalType &return_type, aggregate_finalize_t finalize) { return AggregateFunction( - name, + Identifier(name), {LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::UBIGINT}, return_type, DpApproxBoundsStateSize, DpApproxBoundsInitialize, DpApproxBoundsScatterUpdate, DpApproxBoundsCombine, finalize, FunctionNullHandling::SPECIAL_HANDLING, DpApproxBoundsUpdate, diff --git a/src/aggregates/dp_laplace_noise.cpp b/src/aggregates/dp_laplace_noise.cpp index 590c4979..d33173bf 100644 --- a/src/aggregates/dp_laplace_noise.cpp +++ b/src/aggregates/dp_laplace_noise.cpp @@ -1,3 +1,5 @@ +#include "duckdb/common/vector/struct_vector.hpp" +#include "duckdb/common/vector/list_vector.hpp" #include "aggregates/dp_laplace_noise.hpp" #include "aggregates/as_aggregate.hpp" #include "privacy_debug.hpp" @@ -137,23 +139,23 @@ static void DpSassStabilityFunction(DataChunk &args, ExpressionState &, Vector & auto child_values = UnifiedVectorFormat::GetData(child_data); auto &entries = StructVector::GetEntries(result); - auto lane_count_data = FlatVector::GetData(*entries[0]); - auto valid_count_data = FlatVector::GetData(*entries[1]); - auto null_count_data = FlatVector::GetData(*entries[2]); - auto mean_data = FlatVector::GetData(*entries[3]); - auto stddev_data = FlatVector::GetData(*entries[4]); - auto cv_data = FlatVector::GetData(*entries[5]); - auto min_data = FlatVector::GetData(*entries[6]); - auto median_data = FlatVector::GetData(*entries[7]); - auto max_data = FlatVector::GetData(*entries[8]); - auto range_data = FlatVector::GetData(*entries[9]); - auto mad_data = FlatVector::GetData(*entries[10]); - auto max_abs_dev_data = FlatVector::GetData(*entries[11]); - auto stable_stddev_1_data = FlatVector::GetData(*entries[12]); - auto stable_mad_1_data = FlatVector::GetData(*entries[13]); - - auto &result_validity = FlatVector::Validity(result); - auto &cv_validity = FlatVector::Validity(*entries[5]); + auto lane_count_data = FlatVector::GetDataMutable(entries[0]); + auto valid_count_data = FlatVector::GetDataMutable(entries[1]); + auto null_count_data = FlatVector::GetDataMutable(entries[2]); + auto mean_data = FlatVector::GetDataMutable(entries[3]); + auto stddev_data = FlatVector::GetDataMutable(entries[4]); + auto cv_data = FlatVector::GetDataMutable(entries[5]); + auto min_data = FlatVector::GetDataMutable(entries[6]); + auto median_data = FlatVector::GetDataMutable(entries[7]); + auto max_data = FlatVector::GetDataMutable(entries[8]); + auto range_data = FlatVector::GetDataMutable(entries[9]); + auto mad_data = FlatVector::GetDataMutable(entries[10]); + auto max_abs_dev_data = FlatVector::GetDataMutable(entries[11]); + auto stable_stddev_1_data = FlatVector::GetDataMutable(entries[12]); + auto stable_mad_1_data = FlatVector::GetDataMutable(entries[13]); + + auto &result_validity = FlatVector::ValidityMutable(result); + auto &cv_validity = FlatVector::ValidityMutable(entries[5]); for (idx_t i = 0; i < count; i++) { auto list_idx = list_data.sel->get_index(i); if (!list_data.validity.RowIsValid(list_idx)) { @@ -282,8 +284,8 @@ static void DpSassRecordStabilityFunction(DataChunk &args, ExpressionState &, Ve child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); for (idx_t i = 0; i < count; i++) { auto list_idx = list_data.sel->get_index(i); auto aggregate_index_idx = aggregate_index_data.sel->get_index(i); @@ -501,8 +503,8 @@ static void DpLaplaceNoiseFunction(DataChunk &args, ExpressionState &, Vector &r // BinaryExecutor preserves a CONSTANT vector by invoking its callback once. Noise is volatile, so // materialize a FLAT result and sample once per output row even when value and scale are constants. result.SetVectorType(VectorType::FLAT_VECTOR); - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); result_validity.Reset(); unique_ptr shared_distribution; double shared_scale = 0.0; @@ -538,7 +540,8 @@ void RegisterDpLaplaceNoiseFunction(ExtensionLoader &loader) { DpLaplaceNoiseFunction); // Every released cell needs an independent draw. VOLATILE prevents constant folding and common- // subexpression elimination from reusing one draw across aggregate cells. - function.stability = FunctionStability::VOLATILE; + function.SetStability(FunctionStability::VOLATILE); + function.SetFallible(); set.AddFunction(std::move(function)); CreateScalarFunctionInfo info(set); FunctionDescription desc; @@ -758,7 +761,7 @@ static void DpSmoothMedianNoiseFunctionInternal(DataChunk &args, ExpressionState auto &context = state.GetContext(); bool noise_enabled = true; Value noise_val; - if (context.TryGetCurrentSetting("privacy_noise", noise_val) && !noise_val.IsNull()) { + if (context.TryGetCurrentSetting(Identifier("privacy_noise"), noise_val) && !noise_val.IsNull()) { noise_enabled = noise_val.GetValue(); } @@ -806,8 +809,8 @@ static void DpSmoothMedianNoiseFunctionInternal(DataChunk &args, ExpressionState (record_scale || has_cell_identity) ? UnifiedVectorFormat::GetData(aggregate_index_data) : nullptr; auto aggregate_counts = has_cell_identity ? UnifiedVectorFormat::GetData(aggregate_count_data) : nullptr; auto group_identities = has_cell_identity ? UnifiedVectorFormat::GetData(group_identity_data) : nullptr; - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); for (idx_t i = 0; i < count; i++) { auto list_idx = list_data.sel->get_index(i); @@ -887,7 +890,7 @@ static void DpAggregateFunction(DataChunk &args, ExpressionState &state, Vector auto &context = state.GetContext(); bool noise_enabled = true; Value noise_val; - if (context.TryGetCurrentSetting("privacy_noise", noise_val) && !noise_val.IsNull()) { + if (context.TryGetCurrentSetting(Identifier("privacy_noise"), noise_val) && !noise_val.IsNull()) { noise_enabled = noise_val.GetValue(); } @@ -911,8 +914,8 @@ static void DpAggregateFunction(DataChunk &args, ExpressionState &state, Vector auto sample_lanes_values = UnifiedVectorFormat::GetData(sample_lanes_data); auto lower_values = UnifiedVectorFormat::GetData(lower_data); auto upper_values = UnifiedVectorFormat::GetData(upper_data); - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); for (idx_t i = 0; i < count; i++) { auto list_idx = list_data.sel->get_index(i); @@ -988,7 +991,7 @@ static void DpGuptMeanNoiseFunctionInternal(DataChunk &args, ExpressionState &st auto &context = state.GetContext(); bool noise_enabled = true; Value noise_val; - if (context.TryGetCurrentSetting("privacy_noise", noise_val) && !noise_val.IsNull()) { + if (context.TryGetCurrentSetting(Identifier("privacy_noise"), noise_val) && !noise_val.IsNull()) { noise_enabled = noise_val.GetValue(); } @@ -1030,8 +1033,8 @@ static void DpGuptMeanNoiseFunctionInternal(DataChunk &args, ExpressionState &st (record_scale || has_cell_identity) ? UnifiedVectorFormat::GetData(aggregate_index_data) : nullptr; auto aggregate_counts = has_cell_identity ? UnifiedVectorFormat::GetData(aggregate_count_data) : nullptr; auto group_identities = has_cell_identity ? UnifiedVectorFormat::GetData(group_identity_data) : nullptr; - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); for (idx_t i = 0; i < count; i++) { auto list_idx = list_data.sel->get_index(i); @@ -1212,7 +1215,7 @@ static void DpNonEmptyMeanNoiseFunctionInternal(DataChunk &args, ExpressionState auto &context = state.GetContext(); bool noise_enabled = true; Value noise_val; - if (context.TryGetCurrentSetting("privacy_noise", noise_val) && !noise_val.IsNull()) { + if (context.TryGetCurrentSetting(Identifier("privacy_noise"), noise_val) && !noise_val.IsNull()) { noise_enabled = noise_val.GetValue(); } @@ -1248,8 +1251,8 @@ static void DpNonEmptyMeanNoiseFunctionInternal(DataChunk &args, ExpressionState auto aggregate_indexes = UnifiedVectorFormat::GetData(aggregate_index_data); auto aggregate_counts = record_scales ? nullptr : UnifiedVectorFormat::GetData(aggregate_count_data); auto group_identities = record_scales ? nullptr : UnifiedVectorFormat::GetData(group_identity_data); - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); for (idx_t i = 0; i < count; i++) { auto list_idx = list_data.sel->get_index(i); diff --git a/src/aggregates/filterless_aggregate.cpp b/src/aggregates/filterless_aggregate.cpp index 66af87f7..a0aef8b6 100644 --- a/src/aggregates/filterless_aggregate.cpp +++ b/src/aggregates/filterless_aggregate.cpp @@ -1,3 +1,4 @@ +#include "duckdb/common/vector/struct_vector.hpp" #include "aggregates/filterless_aggregate.hpp" #include "aggregates/as_clip_aggr.hpp" @@ -54,20 +55,20 @@ void ValidateFilterlessBoundsEpsilonFraction(double fraction) { FilterlessSettings GetFilterlessSettings(ClientContext &context) { Value value; int64_t sample_bits = 0; - if (context.TryGetCurrentSetting("dp_filterless_sample_bits", value) && !value.IsNull()) { + if (context.TryGetCurrentSetting(Identifier("dp_filterless_sample_bits"), value) && !value.IsNull()) { sample_bits = value.GetValue(); } ValidateFilterlessSampleBits(sample_bits); double clip_support = std::numeric_limits::quiet_NaN(); - if (!context.TryGetCurrentSetting("dp_filterless_clip_support", value) || value.IsNull()) { + if (!context.TryGetCurrentSetting(Identifier("dp_filterless_clip_support"), value) || value.IsNull()) { throw InvalidInputException("dp_filterless_clip_support must be set to a positive finite number"); } clip_support = value.GetValue(); ValidateFilterlessClipSupport(clip_support); double bounds_fraction = 0.25; - if (context.TryGetCurrentSetting("dp_filterless_bounds_epsilon_fraction", value) && !value.IsNull()) { + if (context.TryGetCurrentSetting(Identifier("dp_filterless_bounds_epsilon_fraction"), value) && !value.IsNull()) { bounds_fraction = value.GetValue(); } ValidateFilterlessBoundsEpsilonFraction(bounds_fraction); @@ -191,7 +192,7 @@ static double SaturatingNoiseScale(long double sensitivity, long double epsilon) : static_cast(scale); } -static double EvaluateConstantDouble(ClientContext &context, const Expression &expression, const string &name) { +static double EvaluateConstantDouble(ClientContext &context, const Expression &expression, const Identifier &name) { if (!expression.IsFoldable()) { throw InvalidInputException("filterless: %s must be a constant", name); } @@ -236,20 +237,24 @@ static unique_ptr BindFilterless(ClientContext &context, vector BindFilterlessSum(ClientContext &context, AggregateFunction &, - vector> &arguments) { - idx_t bin_count = arguments[2]->return_type.InternalType() == PhysicalType::INT64 ? FILTERLESS_COUNT_BIN_COUNT - : FILTERLESS_HUGEINT_BIN_COUNT; +static unique_ptr BindFilterlessSum(BindAggregateFunctionInput &bind_input) { + auto &context = bind_input.GetClientContext(); + auto &arguments = bind_input.GetArguments(); + idx_t bin_count = arguments[2]->GetReturnType().InternalType() == PhysicalType::INT64 + ? FILTERLESS_COUNT_BIN_COUNT + : FILTERLESS_HUGEINT_BIN_COUNT; return BindFilterless(context, arguments, 4, true, bin_count); } -static unique_ptr BindFilterlessCount(ClientContext &context, AggregateFunction &, - vector> &arguments) { +static unique_ptr BindFilterlessCount(BindAggregateFunctionInput &bind_input) { + auto &context = bind_input.GetClientContext(); + auto &arguments = bind_input.GetArguments(); return BindFilterless(context, arguments, 4, false, FILTERLESS_COUNT_BIN_COUNT); } -static unique_ptr BindFilterlessAvg(ClientContext &context, AggregateFunction &, - vector> &arguments) { +static unique_ptr BindFilterlessAvg(BindAggregateFunctionInput &bind_input) { + auto &context = bind_input.GetClientContext(); + auto &arguments = bind_input.GetArguments(); return BindFilterless(context, arguments, 6, true, FILTERLESS_HUGEINT_BIN_COUNT); } @@ -708,28 +713,37 @@ static FilterlessExactResult FinalizeExactComponent(const FilterlessExactCompone SaturatingNoiseScale(static_cast(bound) * bind.max_groups, value_epsilon)}; } -static idx_t FilterlessStateSize(const AggregateFunction &) { +static idx_t FilterlessStateSize(AggregateStateInput &) { return sizeof(FilterlessState); } -static idx_t FilterlessAvgStateSize(const AggregateFunction &) { +static idx_t FilterlessAvgStateSize(AggregateStateInput &) { return sizeof(FilterlessAvgState); } -static idx_t FilterlessExactStateSize(const AggregateFunction &) { +static idx_t FilterlessExactStateSize(AggregateStateInput &) { return sizeof(FilterlessExactState); } -static void FilterlessInitialize(const AggregateFunction &, data_ptr_t state_p) { - memset(state_p, 0, sizeof(FilterlessState)); +static void FilterlessInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_p = states[i]; + memset(state_p, 0, sizeof(FilterlessState)); + } } -static void FilterlessAvgInitialize(const AggregateFunction &, data_ptr_t state_p) { - memset(state_p, 0, sizeof(FilterlessAvgState)); +static void FilterlessAvgInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_p = states[i]; + memset(state_p, 0, sizeof(FilterlessAvgState)); + } } -static void FilterlessExactInitialize(const AggregateFunction &, data_ptr_t state_p) { - memset(state_p, 0, sizeof(FilterlessExactState)); +static void FilterlessExactInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_p = states[i]; + memset(state_p, 0, sizeof(FilterlessExactState)); + } } template @@ -818,9 +832,12 @@ static void FilterlessUpdateRows(Vector inputs[], AggregateInputData &aggr, idx_ } } -static void FilterlessUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, idx_t count) { - auto state = reinterpret_cast(state_p); - FilterlessUpdateRows<2, double>(inputs, aggr, count, [state](idx_t) { return state; }); +static void FilterlessUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto state = reinterpret_cast(state_p); + FilterlessUpdateRows<2, double>(inputs, aggr, count, [state](idx_t) { return state; }); + }); } static void FilterlessScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -831,9 +848,12 @@ static void FilterlessScatterUpdate(Vector inputs[], AggregateInputData &aggr, i [&](idx_t row) { return state_ptrs[state_data.sel->get_index(row)]; }); } -static void FilterlessAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, idx_t count) { - auto state = reinterpret_cast(state_p); - FilterlessUpdateRows<4, double>(inputs, aggr, count, [state](idx_t) { return state; }); +static void FilterlessAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto state = reinterpret_cast(state_p); + FilterlessUpdateRows<4, double>(inputs, aggr, count, [state](idx_t) { return state; }); + }); } static void FilterlessAvgScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { @@ -845,9 +865,12 @@ static void FilterlessAvgScatterUpdate(Vector inputs[], AggregateInputData &aggr } template -static void FilterlessExactUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, idx_t count) { - auto state = reinterpret_cast(state_p); - FilterlessUpdateRows<2, INPUT_TYPE>(inputs, aggr, count, [state](idx_t) { return state; }); +static void FilterlessExactUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto state = reinterpret_cast(state_p); + FilterlessUpdateRows<2, INPUT_TYPE>(inputs, aggr, count, [state](idx_t) { return state; }); + }); } template @@ -887,41 +910,42 @@ static void FilterlessExactCombine(Vector &source, Vector &target, AggregateInpu static void WriteDebugResult(Vector &result, idx_t row, const FilterlessResult &value) { auto &children = StructVector::GetEntries(result); - FlatVector::GetData(*children[0])[row] = value.lower_bound; - FlatVector::GetData(*children[1])[row] = value.upper_bound; - FlatVector::GetData(*children[2])[row] = value.clipped_value; - FlatVector::GetData(*children[3])[row] = value.noise_scale; - FlatVector::GetData(*children[4])[row] = value.negative_bin; - FlatVector::GetData(*children[5])[row] = value.positive_bin; - FlatVector::GetData(*children[6])[row] = value.negative_support; - FlatVector::GetData(*children[7])[row] = value.positive_support; - FlatVector::GetData(*children[8])[row] = value.active_contributions; - FlatVector::GetData(*children[9])[row] = value.sampled_contributions; + FlatVector::GetDataMutable(children[0])[row] = value.lower_bound; + FlatVector::GetDataMutable(children[1])[row] = value.upper_bound; + FlatVector::GetDataMutable(children[2])[row] = value.clipped_value; + FlatVector::GetDataMutable(children[3])[row] = value.noise_scale; + FlatVector::GetDataMutable(children[4])[row] = value.negative_bin; + FlatVector::GetDataMutable(children[5])[row] = value.positive_bin; + FlatVector::GetDataMutable(children[6])[row] = value.negative_support; + FlatVector::GetDataMutable(children[7])[row] = value.positive_support; + FlatVector::GetDataMutable(children[8])[row] = value.active_contributions; + FlatVector::GetDataMutable(children[9])[row] = value.sampled_contributions; } static void WriteAvgDebugResult(Vector &result, idx_t row, const FilterlessResult &sum, const FilterlessResult &denominator, double released_average) { auto &children = StructVector::GetEntries(result); - FlatVector::GetData(*children[0])[row] = sum.lower_bound; - FlatVector::GetData(*children[1])[row] = sum.upper_bound; - FlatVector::GetData(*children[2])[row] = sum.clipped_value; - FlatVector::GetData(*children[3])[row] = sum.noise_scale; - FlatVector::GetData(*children[4])[row] = denominator.upper_bound; - FlatVector::GetData(*children[5])[row] = denominator.clipped_value; - FlatVector::GetData(*children[6])[row] = denominator.noise_scale; - FlatVector::GetData(*children[7])[row] = released_average; - FlatVector::GetData(*children[8])[row] = sum.negative_bin; - FlatVector::GetData(*children[9])[row] = sum.positive_bin; - FlatVector::GetData(*children[10])[row] = denominator.positive_bin; - FlatVector::GetData(*children[11])[row] = sum.active_contributions; - FlatVector::GetData(*children[12])[row] = sum.sampled_contributions; + FlatVector::GetDataMutable(children[0])[row] = sum.lower_bound; + FlatVector::GetDataMutable(children[1])[row] = sum.upper_bound; + FlatVector::GetDataMutable(children[2])[row] = sum.clipped_value; + FlatVector::GetDataMutable(children[3])[row] = sum.noise_scale; + FlatVector::GetDataMutable(children[4])[row] = denominator.upper_bound; + FlatVector::GetDataMutable(children[5])[row] = denominator.clipped_value; + FlatVector::GetDataMutable(children[6])[row] = denominator.noise_scale; + FlatVector::GetDataMutable(children[7])[row] = released_average; + FlatVector::GetDataMutable(children[8])[row] = sum.negative_bin; + FlatVector::GetDataMutable(children[9])[row] = sum.positive_bin; + FlatVector::GetDataMutable(children[10])[row] = denominator.positive_bin; + FlatVector::GetDataMutable(children[11])[row] = sum.active_contributions; + FlatVector::GetDataMutable(children[12])[row] = sum.sampled_contributions; } template -static void FilterlessFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset) { +static void FilterlessFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, + idx_t offset) { auto state_ptrs = FlatVector::GetData(states); auto &bind = input.bind_data->Cast(); - auto result_data = DEBUG ? nullptr : FlatVector::GetData(result); + auto result_data = DEBUG ? nullptr : FlatVector::GetDataMutable(result); for (idx_t i = 0; i < count; i++) { auto value = FinalizeComponent(state_ptrs[i]->component, bind, bind.epsilon, COUNT); if (DEBUG) { @@ -934,11 +958,11 @@ static void FilterlessFinalize(Vector &states, AggregateInputData &input, Vector } template -static void FilterlessAvgFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void FilterlessAvgFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData(states); auto &bind = input.bind_data->Cast(); - auto result_data = DEBUG ? nullptr : FlatVector::GetData(result); + auto result_data = DEBUG ? nullptr : FlatVector::GetDataMutable(result); for (idx_t i = 0; i < count; i++) { double component_epsilon = bind.epsilon / 2.0; auto sum = FinalizeComponent(state_ptrs[i]->sum_component, bind, component_epsilon, false); @@ -1006,10 +1030,10 @@ static hugeint_t ReleaseExactComponent(const FilterlessExactComponentState &stat } template -static void FilterlessExactFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void FilterlessExactFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData(states); - auto result_data = FlatVector::GetData(result); + auto result_data = FlatVector::GetDataMutable(result); auto &bind = input.bind_data->Cast(); for (idx_t i = 0; i < count; i++) { auto released = @@ -1070,13 +1094,16 @@ struct FilterlessFusedAvgState { }; template -static idx_t FilterlessFusedAvgStateSize(const AggregateFunction &) { +static idx_t FilterlessFusedAvgStateSize(AggregateStateInput &) { return sizeof(FilterlessFusedAvgState); } template -static void FilterlessFusedAvgInitialize(const AggregateFunction &, data_ptr_t state_p) { - memset(state_p, 0, sizeof(FilterlessFusedAvgState)); +static void FilterlessFusedAvgInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_p = states[i]; + memset(state_p, 0, sizeof(FilterlessFusedAvgState)); + } } template @@ -1126,10 +1153,12 @@ static void FilterlessFusedAvgUpdateRows(Vector inputs[], AggregateInputData &ag } template -static void FilterlessFusedAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, data_ptr_t state_p, - idx_t count) { - auto state = reinterpret_cast *>(state_p); - FilterlessFusedAvgUpdateRows(inputs, aggr, count, [state](idx_t) { return state; }); +static void FilterlessFusedAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto state = reinterpret_cast *>(state_p); + FilterlessFusedAvgUpdateRows(inputs, aggr, count, [state](idx_t) { return state; }); + }); } template @@ -1153,11 +1182,11 @@ static void FilterlessFusedAvgCombine(Vector &source, Vector &target, AggregateI } template -static void FilterlessFusedAvgFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, +static void FilterlessFusedAvgFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset) { auto state_ptrs = FlatVector::GetData *>(states); - auto result_data = FlatVector::GetData(result); - auto &validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &validity = FlatVector::ValidityMutable(result); auto &bind = input.bind_data->Cast(); double component_epsilon = bind.epsilon / 2.0; for (idx_t i = 0; i < count; i++) { @@ -1348,13 +1377,16 @@ static LogicalType FilterlessLowerPairType(const LogicalType &value_type) { } template -static idx_t FilterlessLowerPairStateSize(const AggregateFunction &) { +static idx_t FilterlessLowerPairStateSize(AggregateStateInput &) { return sizeof(FilterlessLowerPairState); } template -static void FilterlessLowerPairInitialize(const AggregateFunction &, data_ptr_t state_p) { - memset(state_p, 0, sizeof(FilterlessLowerPairState)); +static void FilterlessLowerPairInitialize(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_p = states[i]; + memset(state_p, 0, sizeof(FilterlessLowerPairState)); + } } template @@ -1389,10 +1421,12 @@ static void FilterlessLowerPairUpdateRows(Vector inputs[], idx_t count, ArenaAll } template -static void FilterlessLowerPairUpdate(Vector inputs[], AggregateInputData &input, idx_t, data_ptr_t state_p, - idx_t count) { - auto state = reinterpret_cast *>(state_p); - FilterlessLowerPairUpdateRows(inputs, count, input.allocator, [state](idx_t) { return state; }); +static void FilterlessLowerPairUpdate(Vector inputs[], AggregateInputData &input, idx_t input_count, + const ClusteredAggr &clusters, idx_t count) { + ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { + auto state = reinterpret_cast *>(state_p); + FilterlessLowerPairUpdateRows(inputs, count, input.allocator, [state](idx_t) { return state; }); + }); } template @@ -1417,24 +1451,24 @@ static void FilterlessLowerPairCombine(Vector &source, Vector &target, Aggregate } template -static void FilterlessLowerPairFinalize(Vector &states, AggregateInputData &, Vector &result, idx_t count, +static void FilterlessLowerPairFinalize(Vector &states, AggregateFinalizeInputData &, Vector &result, idx_t count, idx_t offset) { using pair_state_t = FilterlessLowerPairState; auto state_ptrs = FlatVector::GetData(states); auto &children = StructVector::GetEntries(result); - auto answers = FlatVector::GetData(*children[0]); - auto histograms = FlatVector::GetData(*children[1]); + auto answers = FlatVector::GetDataMutable(children[0]); + auto histograms = FlatVector::GetDataMutable(children[1]); for (idx_t i = 0; i < count; i++) { auto row = offset + i; if (OPERATION::IsSet(state_ptrs[i]->answer)) { answers[row] = OPERATION::Finalize(state_ptrs[i]->answer); } else { - FlatVector::Validity(*children[0]).SetInvalid(row); + FlatVector::ValidityMutable(children[0]).SetInvalid(row); } if (OPERATION::IsSet(state_ptrs[i]->histogram)) { histograms[row] = OPERATION::Finalize(state_ptrs[i]->histogram); } else { - FlatVector::Validity(*children[1]).SetInvalid(row); + FlatVector::ValidityMutable(children[1]).SetInvalid(row); } } } @@ -1473,14 +1507,14 @@ static LogicalType FilterlessAvgDebugType() { } template -static AggregateFunction MakeFilterlessExactFunction(const string &name, const LogicalType &input_type, +static AggregateFunction MakeFilterlessExactFunction(const Identifier &name, const LogicalType &input_type, const LogicalType &return_type, bool explicit_config) { vector arguments = {LogicalType::UBIGINT, LogicalType::BOOLEAN, input_type, input_type}; if (explicit_config) { arguments.push_back(LogicalType::DOUBLE); arguments.push_back(LogicalType::DOUBLE); } - return AggregateFunction(name, std::move(arguments), return_type, FilterlessExactStateSize, + return AggregateFunction(Identifier(name), std::move(arguments), return_type, FilterlessExactStateSize, FilterlessExactInitialize, FilterlessExactScatterUpdate, FilterlessExactCombine, FilterlessExactFinalize, FunctionNullHandling::SPECIAL_HANDLING, FilterlessExactUpdate, @@ -1488,8 +1522,8 @@ static AggregateFunction MakeFilterlessExactFunction(const string &name, const L } template -static void AddFilterlessExactOverloads(AggregateFunctionSet &set, const string &name, const LogicalType &input_type, - const LogicalType &return_type) { +static void AddFilterlessExactOverloads(AggregateFunctionSet &set, const Identifier &name, + const LogicalType &input_type, const LogicalType &return_type) { set.AddFunction(MakeFilterlessExactFunction(name, input_type, return_type, false)); set.AddFunction(MakeFilterlessExactFunction(name, input_type, return_type, true)); } @@ -1520,29 +1554,31 @@ static void ConfigureFilterlessDecimalBind(FilterlessBindData &bind, const Logic bind.exact_output_min = Hugeint::Negate(bind.exact_output_max); } -static unique_ptr BindFilterlessDecimalSum(ClientContext &context, AggregateFunction &function, - vector> &arguments) { +static unique_ptr BindFilterlessDecimalSum(BindAggregateFunctionInput &bind_input) { + auto &context = bind_input.GetClientContext(); + auto &function = bind_input.GetBoundFunction(); + auto &arguments = bind_input.GetArguments(); if (arguments.size() != 4 && arguments.size() != 6) { throw InternalException("filterless_sum: unexpected DECIMAL argument count"); } - auto input_type = arguments[2]->return_type; - if (arguments[3]->return_type != input_type) { + auto input_type = arguments[2]->GetReturnType(); + if (arguments[3]->GetReturnType() != input_type) { throw InvalidInputException("filterless_sum: answer and histogram DECIMAL types must match"); } auto return_type = LogicalType::DECIMAL(Decimal::MAX_WIDTH_DECIMAL, DecimalType::GetScale(input_type)); - function = MakeFilterlessDecimalSumFunction(input_type, return_type, arguments.size() == 6); + function.ReplaceImplementation(MakeFilterlessDecimalSumFunction(input_type, return_type, arguments.size() == 6)); auto result = BindFilterless(context, arguments, 4, false, FILTERLESS_HUGEINT_BIN_COUNT); ConfigureFilterlessDecimalBind(result->Cast(), input_type); return result; } -static void AddSumCountOverloads(AggregateFunctionSet &set, const string &name, aggregate_finalize_t finalize, +static void AddSumCountOverloads(AggregateFunctionSet &set, const Identifier &name, aggregate_finalize_t finalize, const LogicalType &return_type, bind_aggregate_function_t bind) { - set.AddFunction( - AggregateFunction(name, {LogicalType::UBIGINT, LogicalType::BOOLEAN, LogicalType::DOUBLE, LogicalType::DOUBLE}, - return_type, FilterlessStateSize, FilterlessInitialize, FilterlessScatterUpdate, - FilterlessCombine, finalize, FunctionNullHandling::SPECIAL_HANDLING, FilterlessUpdate, bind)); - set.AddFunction(AggregateFunction(name, + set.AddFunction(AggregateFunction( + Identifier(name), {LogicalType::UBIGINT, LogicalType::BOOLEAN, LogicalType::DOUBLE, LogicalType::DOUBLE}, + return_type, FilterlessStateSize, FilterlessInitialize, FilterlessScatterUpdate, FilterlessCombine, finalize, + FunctionNullHandling::SPECIAL_HANDLING, FilterlessUpdate, bind)); + set.AddFunction(AggregateFunction(Identifier(name), {LogicalType::UBIGINT, LogicalType::BOOLEAN, LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::DOUBLE}, return_type, FilterlessStateSize, FilterlessInitialize, FilterlessScatterUpdate, @@ -1550,16 +1586,16 @@ static void AddSumCountOverloads(AggregateFunctionSet &set, const string &name, FilterlessUpdate, bind)); } -static void AddAvgOverloads(AggregateFunctionSet &set, const string &name, aggregate_finalize_t finalize, +static void AddAvgOverloads(AggregateFunctionSet &set, const Identifier &name, aggregate_finalize_t finalize, const LogicalType &return_type) { - set.AddFunction(AggregateFunction(name, + set.AddFunction(AggregateFunction(Identifier(name), {LogicalType::UBIGINT, LogicalType::BOOLEAN, LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::DOUBLE}, return_type, FilterlessAvgStateSize, FilterlessAvgInitialize, FilterlessAvgScatterUpdate, FilterlessAvgCombine, finalize, FunctionNullHandling::SPECIAL_HANDLING, FilterlessAvgUpdate, BindFilterlessAvg)); set.AddFunction(AggregateFunction( - name, + Identifier(name), {LogicalType::UBIGINT, LogicalType::BOOLEAN, LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::DOUBLE, LogicalType::DOUBLE}, return_type, FilterlessAvgStateSize, FilterlessAvgInitialize, FilterlessAvgScatterUpdate, FilterlessAvgCombine, @@ -1580,38 +1616,42 @@ static AggregateFunction MakeFilterlessFusedAvgFunction(const LogicalType &sum_t return function; } -static unique_ptr BindFilterlessFusedApproxAvg(ClientContext &context, AggregateFunction &, - vector> &arguments) { +static unique_ptr BindFilterlessFusedApproxAvg(BindAggregateFunctionInput &bind_input) { + auto &context = bind_input.GetClientContext(); + auto &arguments = bind_input.GetArguments(); return BindFilterless(context, arguments, 6, true, FILTERLESS_HUGEINT_BIN_COUNT); } -static unique_ptr BindFilterlessFusedExactAvg(ClientContext &context, AggregateFunction &, - vector> &arguments) { +static unique_ptr BindFilterlessFusedExactAvg(BindAggregateFunctionInput &bind_input) { + auto &context = bind_input.GetClientContext(); + auto &arguments = bind_input.GetArguments(); return BindFilterless(context, arguments, 6, false, FILTERLESS_HUGEINT_BIN_COUNT); } -static unique_ptr BindFilterlessFusedDecimalAvg(ClientContext &context, AggregateFunction &function, - vector> &arguments) { - auto sum_type = arguments[2]->return_type; - if (arguments[4]->return_type != sum_type) { +static unique_ptr BindFilterlessFusedDecimalAvg(BindAggregateFunctionInput &bind_input) { + auto &context = bind_input.GetClientContext(); + auto &function = bind_input.GetBoundFunction(); + auto &arguments = bind_input.GetArguments(); + auto sum_type = arguments[2]->GetReturnType(); + if (arguments[4]->GetReturnType() != sum_type) { throw InvalidInputException("priv_filterless_avg: answer and histogram DECIMAL types must match"); } switch (sum_type.InternalType()) { case PhysicalType::INT16: - function = MakeFilterlessFusedAvgFunction>( - sum_type, BindFilterlessFusedDecimalAvg); + function.ReplaceImplementation(MakeFilterlessFusedAvgFunction>( + sum_type, BindFilterlessFusedDecimalAvg)); break; case PhysicalType::INT32: - function = MakeFilterlessFusedAvgFunction>( - sum_type, BindFilterlessFusedDecimalAvg); + function.ReplaceImplementation(MakeFilterlessFusedAvgFunction>( + sum_type, BindFilterlessFusedDecimalAvg)); break; case PhysicalType::INT64: - function = MakeFilterlessFusedAvgFunction>( - sum_type, BindFilterlessFusedDecimalAvg); + function.ReplaceImplementation(MakeFilterlessFusedAvgFunction>( + sum_type, BindFilterlessFusedDecimalAvg)); break; case PhysicalType::INT128: - function = MakeFilterlessFusedAvgFunction>( - sum_type, BindFilterlessFusedDecimalAvg); + function.ReplaceImplementation(MakeFilterlessFusedAvgFunction>( + sum_type, BindFilterlessFusedDecimalAvg)); break; default: throw InternalException("priv_filterless_avg: unsupported DECIMAL physical type"); @@ -1622,10 +1662,10 @@ static unique_ptr BindFilterlessFusedDecimalAvg(ClientContext &con } template -static AggregateFunction MakeFilterlessLowerPairFunction(const string &name, const LogicalType &input_type, +static AggregateFunction MakeFilterlessLowerPairFunction(const Identifier &name, const LogicalType &input_type, const LogicalType &return_type) { auto function = - AggregateFunction(name, {input_type, LogicalType::BOOLEAN, LogicalType::BOOLEAN}, + AggregateFunction(Identifier(name), {input_type, LogicalType::BOOLEAN, LogicalType::BOOLEAN}, FilterlessLowerPairType(return_type), FilterlessLowerPairStateSize, FilterlessLowerPairInitialize, FilterlessLowerPairScatterUpdate, FilterlessLowerPairCombine, FilterlessLowerPairFinalize, @@ -1641,22 +1681,23 @@ static AggregateFunction MakeFilterlessExactSumPairFunction(const LogicalType &i "priv_filterless_sum_pair", input_type, return_type); } -static unique_ptr BindFilterlessDecimalSumPair(ClientContext &, AggregateFunction &function, - vector> &arguments) { - auto input_type = arguments[0]->return_type; +static unique_ptr BindFilterlessDecimalSumPair(BindAggregateFunctionInput &bind_input) { + auto &function = bind_input.GetBoundFunction(); + auto &arguments = bind_input.GetArguments(); + auto input_type = arguments[0]->GetReturnType(); auto return_type = LogicalType::DECIMAL(Decimal::MAX_WIDTH_DECIMAL, DecimalType::GetScale(input_type)); switch (input_type.InternalType()) { case PhysicalType::INT16: - function = MakeFilterlessExactSumPairFunction(input_type, return_type); + function.ReplaceImplementation(MakeFilterlessExactSumPairFunction(input_type, return_type)); break; case PhysicalType::INT32: - function = MakeFilterlessExactSumPairFunction(input_type, return_type); + function.ReplaceImplementation(MakeFilterlessExactSumPairFunction(input_type, return_type)); break; case PhysicalType::INT64: - function = MakeFilterlessExactSumPairFunction(input_type, return_type); + function.ReplaceImplementation(MakeFilterlessExactSumPairFunction(input_type, return_type)); break; case PhysicalType::INT128: - function = MakeFilterlessExactSumPairFunction(input_type, return_type); + function.ReplaceImplementation(MakeFilterlessExactSumPairFunction(input_type, return_type)); break; default: throw InternalException("priv_filterless_sum_pair: unsupported DECIMAL physical type"); diff --git a/src/categorical/pac_categorical.cpp b/src/categorical/pac_categorical.cpp index 0ae56f10..e3b720f7 100644 --- a/src/categorical/pac_categorical.cpp +++ b/src/categorical/pac_categorical.cpp @@ -6,6 +6,7 @@ // Created by ila on 1/22/26. // +#include "duckdb/common/vector/list_vector.hpp" #include "categorical/pac_categorical.hpp" #include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" #include "privacy_debug.hpp" @@ -63,7 +64,7 @@ struct PacCategoricalBindData : public FunctionData { random_state(GetPacRandomState(ctx)) { // Read utility threshold setting Value ut_val; - if (ctx.TryGetCurrentSetting("privacy_min_group_count", ut_val) && !ut_val.IsNull()) { + if (ctx.TryGetCurrentSetting(Identifier("privacy_min_group_count"), ut_val) && !ut_val.IsNull()) { utility_threshold = ut_val.GetValue(); } seed = GetPacSeed(ctx, mi); @@ -72,7 +73,7 @@ struct PacCategoricalBindData : public FunctionData { if (mi > 0.0) { Value pt_val; bool ptracking_enabled = true; - if (ctx.TryGetCurrentSetting("pac_ptracking", pt_val) && !pt_val.IsNull()) { + if (ctx.TryGetCurrentSetting(Identifier("pac_ptracking"), pt_val) && !pt_val.IsNull()) { ptracking_enabled = pt_val.GetValue(); } if (ptracking_enabled) { @@ -190,8 +191,8 @@ static void PacSelectFunction(DataChunk &args, ExpressionState &state, Vector &r child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -225,8 +226,8 @@ static PacFilterParams GetPacFilterParams(ExpressionState &state) { double mi = 0.0; double correction = 1.0; auto &function = state.expr.Cast(); - if (function.bind_info) { - auto &bind_data = function.bind_info->Cast(); + if (function.BindInfo()) { + auto &bind_data = function.BindInfo()->Cast(); mi = bind_data.mi; correction = bind_data.correction; } @@ -243,7 +244,7 @@ static void PacFilterFromBoolListFunction(DataChunk &args, ExpressionState &stat UnifiedVectorFormat list_data; list_vec.ToUnifiedFormat(count, list_data); - auto result_data = FlatVector::GetData(result); + auto result_data = FlatVector::GetDataMutable(result); auto &child_vec = ListVector::GetEntry(list_vec); UnifiedVectorFormat child_data; @@ -280,8 +281,7 @@ static void PacFilterFromBoolListFunction(DataChunk &args, ExpressionState &stat } // Forward declarations for functions used by priv_filter_ registration -static unique_ptr PacCategoricalBind(ClientContext &ctx, ScalarFunction &func, - vector> &args); +static unique_ptr PacCategoricalBind(BindScalarFunctionInput &bind_input); // ============================================================================ // PAC_FILTER_: Compare scalar against counter list and filter in one pass @@ -338,7 +338,7 @@ static void PacFilterCmpFunction(DataChunk &args, ExpressionState &state, Vector corrections = UnifiedVectorFormat::GetData(correction_data); } - auto result_data = FlatVector::GetData(result); + auto result_data = FlatVector::GetDataMutable(result); for (idx_t i = 0; i < count; i++) { auto val_idx = val_data.sel->get_index(i); @@ -379,14 +379,15 @@ static void PacFilterCmpFunction(DataChunk &args, ExpressionState &state, Vector } template -static void RegisterPacFilterCmp(ExtensionLoader &loader, const string &name) { +static void RegisterPacFilterCmp(ExtensionLoader &loader, const Identifier &name) { auto pf = PacFloatLogicalType(); auto ldt = LogicalType::LIST(pf); - ScalarFunctionSet fset(name); - fset.AddFunction(ScalarFunction(name, {pf, ldt}, LogicalType::BOOLEAN, PacFilterCmpFunction, - PacCategoricalBind, nullptr, nullptr, PacCategoricalInitLocal)); - fset.AddFunction(ScalarFunction(name, {pf, ldt, pf}, LogicalType::BOOLEAN, PacFilterCmpFunction, - PacCategoricalBind, nullptr, nullptr, PacCategoricalInitLocal)); + ScalarFunctionSet fset {Identifier(name)}; + fset.AddFunction(ScalarFunction(Identifier(name), {pf, ldt}, LogicalType::BOOLEAN, PacFilterCmpFunction, + PacCategoricalBind, nullptr, PacCategoricalInitLocal)); + fset.AddFunction(ScalarFunction(Identifier(name), {pf, ldt, pf}, LogicalType::BOOLEAN, + PacFilterCmpFunction, PacCategoricalBind, nullptr, + PacCategoricalInitLocal)); CreateScalarFunctionInfo info(fset); FunctionDescription desc; desc.description = "[INTERNAL] Categorical comparison + probabilistic filter for PAC queries."; @@ -420,8 +421,8 @@ static void PacSelectCmpFunction(DataChunk &args, ExpressionState &state, Vector child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -452,7 +453,7 @@ static void PacSelectCmpFunction(DataChunk &args, ExpressionState &state, Vector } template -static void RegisterPacSelectCmp(ExtensionLoader &loader, const string &name) { +static void RegisterPacSelectCmp(ExtensionLoader &loader, const Identifier &name) { auto pf = PacFloatLogicalType(); auto ldt = LogicalType::LIST(pf); ScalarFunction f(name, {LogicalType::UBIGINT, pf, ldt}, LogicalType::UBIGINT, PacSelectCmpFunction, @@ -486,8 +487,8 @@ static void PacCoalesceFunction(DataChunk &args, ExpressionState &state, Vector auto input_entries = UnifiedVectorFormat::GetData(input_data); result.SetVectorType(VectorType::FLAT_VECTOR); - auto list_entries = FlatVector::GetData(result); - auto &child_vec = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &child_vec = ListVector::GetChildMutable(result); // First pass: count total elements needed idx_t total_elements = 0; @@ -502,8 +503,8 @@ static void PacCoalesceFunction(DataChunk &args, ExpressionState &state, Vector ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto child_data = FlatVector::GetData(child_vec); - auto &child_validity = FlatVector::Validity(child_vec); + auto child_data = FlatVector::GetDataMutable(child_vec); + auto &child_validity = FlatVector::ValidityMutable(child_vec); child_validity.EnsureWritable(); // Get the source child vector for non-NULL lists @@ -579,14 +580,14 @@ static void PacNoisedFunction(DataChunk &args, ExpressionState &state, Vector &r idx_t count = args.size(); auto &function = state.expr.Cast(); - auto &bind_data = function.bind_info->Cast(); + auto &bind_data = function.BindInfo()->Cast(); auto gen = PacRandomStream(bind_data.seed, *bind_data.random_state); UnifiedVectorFormat list_data; list_vec.ToUnifiedFormat(count, list_data); - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); auto &child_vec = ListVector::GetEntry(list_vec); UnifiedVectorFormat child_data; @@ -670,14 +671,14 @@ static void PacDivFunction(DataChunk &args, ExpressionState &state, Vector &resu } result.SetVectorType(VectorType::FLAT_VECTOR); - auto list_entries = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); - auto &result_child = ListVector::GetEntry(result); + auto list_entries = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); + auto &result_child = ListVector::GetChildMutable(result); ListVector::Reserve(result, total_elements); ListVector::SetListSize(result, total_elements); - auto result_data = FlatVector::GetData(result_child); - auto &result_child_validity = FlatVector::Validity(result_child); + auto result_data = FlatVector::GetDataMutable(result_child); + auto &result_child_validity = FlatVector::ValidityMutable(result_child); result_child_validity.SetAllValid(total_elements); idx_t base = 0; @@ -725,15 +726,15 @@ static void PacNoisedDivFunction(DataChunk &args, ExpressionState &state, Vector idx_t count = args.size(); auto &function = state.expr.Cast(); - auto &bind_data = function.bind_info->Cast(); + auto &bind_data = function.BindInfo()->Cast(); auto gen = PacRandomStream(bind_data.seed, *bind_data.random_state); UnifiedVectorFormat sum_data, cnt_data; sum_vec.ToUnifiedFormat(count, sum_data); cnt_vec.ToUnifiedFormat(count, cnt_data); - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); auto &sum_child = ListVector::GetEntry(sum_vec); auto &cnt_child = ListVector::GetEntry(cnt_vec); @@ -787,8 +788,10 @@ static void PacNoisedDivFunction(DataChunk &args, ExpressionState &state, Vector // ============================================================================ // Bind function for priv_filter/priv_select (reads privacy_seed and pac_mi settings) // ============================================================================ -static unique_ptr PacCategoricalBind(ClientContext &ctx, ScalarFunction &func, - vector> &args) { +static unique_ptr PacCategoricalBind(BindScalarFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &func = bind_input.GetBoundFunction(); + auto &args = bind_input.GetArguments(); // Read mi from pac_mi setting double mi = GetPacMiFromSetting(ctx); // Default correction is 1.0 (can be overridden via explicit parameter in some functions) @@ -1003,13 +1006,16 @@ struct PacListOps { // ============================================================================ template -static void PacListAggregateInit(const AggregateFunction &, data_ptr_t state_ptr) { - auto &state = *reinterpret_cast(state_ptr); - state.key_hash = 0; - PAC_FLOAT init_val = PacListOps::InitValue(); - for (idx_t i = 0; i < 64; i++) { - state.values[i] = init_val; - state.counts[i] = 0; +static void PacListAggregateInit(AggregateStateInput &, data_ptr_t *states, idx_t count) { + for (idx_t i = 0; i < count; i++) { + auto state_ptr = states[i]; + auto &state = *reinterpret_cast(state_ptr); + state.key_hash = 0; + PAC_FLOAT init_val = PacListOps::InitValue(); + for (idx_t i = 0; i < 64; i++) { + state.values[i] = init_val; + state.counts[i] = 0; + } } } @@ -1062,7 +1068,7 @@ static void PacListAggregateUpdate(Vector inputs[], AggregateInputData &, idx_t UnifiedVectorFormat sdata; state_vector.ToUnifiedFormat(count, sdata); - auto states = reinterpret_cast(sdata.data); + auto states = UnifiedVectorFormat::GetData(sdata); // Check if child vector is flat with all valid entries (enables dense path) bool child_is_flat = child_vec.GetVectorType() == VectorType::FLAT_VECTOR && child_data.validity.AllValid(); @@ -1093,8 +1099,8 @@ static void PacListAggregateCombine(Vector &source_vec, Vector &target_vec, Aggr UnifiedVectorFormat sdata, tdata; source_vec.ToUnifiedFormat(count, sdata); target_vec.ToUnifiedFormat(count, tdata); - auto sources = reinterpret_cast(sdata.data); - auto targets = reinterpret_cast(tdata.data); + auto sources = UnifiedVectorFormat::GetData(sdata); + auto targets = UnifiedVectorFormat::GetData(tdata); for (idx_t i = 0; i < count; i++) { auto &source = *sources[sdata.sel->get_index(i)]; @@ -1109,14 +1115,14 @@ static void PacListAggregateCombine(Vector &source_vec, Vector &target_vec, Aggr } template -static void PacListAggregateFinalize(Vector &state_vector, AggregateInputData &, Vector &result, idx_t count, +static void PacListAggregateFinalize(Vector &state_vector, AggregateFinalizeInputData &, Vector &result, idx_t count, idx_t offset) { UnifiedVectorFormat sdata; state_vector.ToUnifiedFormat(count, sdata); - auto states = reinterpret_cast(sdata.data); + auto states = UnifiedVectorFormat::GetData(sdata); result.SetVectorType(VectorType::FLAT_VECTOR); - auto &result_validity = FlatVector::Validity(result); + auto &result_validity = FlatVector::ValidityMutable(result); for (idx_t i = 0; i < count; i++) { auto &state = *states[sdata.sel->get_index(i)]; @@ -1153,9 +1159,9 @@ static void PacListAggregateDestructor(Vector &, AggregateInputData &, idx_t) { } template -static AggregateFunction CreatePacListAggregate(const string &name) { +static AggregateFunction CreatePacListAggregate(const Identifier &name) { auto list_double_type = LogicalType::LIST(PacFloatLogicalType()); - return AggregateFunction(name, {list_double_type}, list_double_type, + return AggregateFunction(Identifier(name), {list_double_type}, list_double_type, AggregateFunction::StateSize, PacListAggregateInit, PacListAggregateUpdate, PacListAggregateCombine, PacListAggregateFinalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, nullptr, @@ -1196,20 +1202,20 @@ void RegisterPacCategoricalFunctions(ExtensionLoader &loader) { // priv_filter(list) -> BOOLEAN ScalarFunction pac_filter_list("priv_filter", {list_bool_type}, LogicalType::BOOLEAN, PacFilterFromBoolListFunction, - PacCategoricalBind, nullptr, nullptr, PacCategoricalInitLocal); + PacCategoricalBind, nullptr, PacCategoricalInitLocal); loader.RegisterFunction(make_scalar_info( pac_filter_list, "[INTERNAL] Probabilistic filter from boolean list for categorical queries.")); // priv_coalesce(list) -> list auto list_double_type = LogicalType::LIST(PacFloatLogicalType()); ScalarFunction priv_coalesce("priv_coalesce", {list_double_type}, list_double_type, PacCoalesceFunction); - priv_coalesce.null_handling = FunctionNullHandling::SPECIAL_HANDLING; + priv_coalesce.SetNullHandling(FunctionNullHandling::SPECIAL_HANDLING); loader.RegisterFunction(make_scalar_info(priv_coalesce, "[INTERNAL] Replaces NULL counter list with 64 zeros.")); // as_noised(list) -> PAC_FLOAT ScalarFunction as_noised("as_noised", {list_double_type}, PacFloatLogicalType(), PacNoisedFunction, - PacCategoricalBind, nullptr, nullptr, PacCategoricalInitLocal); - as_noised.stability = FunctionStability::VOLATILE; + PacCategoricalBind, nullptr, PacCategoricalInitLocal); + as_noised.SetStability(FunctionStability::VOLATILE); loader.RegisterFunction( make_scalar_info(as_noised, "[INTERNAL] Applies PAC noise to 64-element counter list, returns scalar.")); @@ -1219,8 +1225,8 @@ void RegisterPacCategoricalFunctions(ExtensionLoader &loader) { // as_noised_div(list sum, list cnt) -> DOUBLE ScalarFunction as_noised_div("as_noised_div", {list_double_type, list_double_type}, LogicalType::DOUBLE, - PacNoisedDivFunction, PacCategoricalBind, nullptr, nullptr, PacCategoricalInitLocal); - as_noised_div.stability = FunctionStability::VOLATILE; + PacNoisedDivFunction, PacCategoricalBind, nullptr, PacCategoricalInitLocal); + as_noised_div.SetStability(FunctionStability::VOLATILE); loader.RegisterFunction(make_scalar_info(as_noised_div, "[INTERNAL] Fused counter-list division + noise for AVG.")); // priv_filter_: optimized comparison + filter in a single pass diff --git a/src/categorical/pac_categorical_detection.cpp b/src/categorical/pac_categorical_detection.cpp index 9d22ddc0..ae2b51c5 100644 --- a/src/categorical/pac_categorical_detection.cpp +++ b/src/categorical/pac_categorical_detection.cpp @@ -5,130 +5,89 @@ // // Created by ila on 1/23/26. // +#include "privacy_debug.hpp" #include "categorical/pac_categorical_detection.hpp" +#include "duckdb/planner/operator/logical_cteref.hpp" +#include "duckdb/planner/operator/logical_materialized_cte.hpp" namespace duckdb { -static string FindPacAggregateInExpression(Expression *expr, LogicalOperator *plan_root, - ColumnBinding *out_source_binding = nullptr); // forward declaration - -// Trace a column binding through the plan to find if it comes from a PAC aggregate -// Returns the PAC aggregate name if found (base name without _counters), empty string otherwise -static string TracePacAggregateFromBinding(const ColumnBinding &binding, LogicalOperator *plan_root, - ColumnBinding *out_source_binding = nullptr) { - auto *source_op = FindOperatorByTableIndex(plan_root, binding.table_index); - if (!source_op) { - return ""; +// Follow every source, including both COUNT copies in DuckDB's nullable-count +// wrapper and aggregates over aggregates. A projection may depend on several +// stochastic outputs; retaining only its first source prematurely finalizes the rest. +static void CollectPacSources(const Expression &expr, LogicalOperator &root, vector &sources, + unordered_set &visited) { + if (IsAlreadyWrappedInPacNoised(&expr)) { + return; } - if (source_op->type == LogicalOperatorType::LOGICAL_PROJECTION) { - auto *unwrapped = RecognizeDuckDBScalarWrapper(source_op); - if (unwrapped) { // Scalar subquery wrapper: trace through inner projection's expression - auto &agg_child = source_op->Cast().children[0]->Cast(); - auto &inner_proj = agg_child.children[0]->Cast(); - if (!inner_proj.expressions.empty()) { - return FindPacAggregateInExpression(inner_proj.expressions[0].get(), plan_root, out_source_binding); - } - return ""; + if (expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { + auto binding = expr.Cast().Binding(); + if (!visited.insert(HashBinding(binding)).second) { + return; } - auto &proj = source_op->Cast(); - if (binding.column_index < proj.expressions.size()) { - if (IsAlreadyWrappedInPacNoised(proj.expressions[binding.column_index].get())) { - return ""; - } - return FindPacAggregateInExpression(proj.expressions[binding.column_index].get(), plan_root, - out_source_binding); + auto *source = FindOperatorByTableIndex(&root, binding.table_index.index); + if (!source) { + return; } - } else if (source_op->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { - auto &aggr = source_op->Cast(); - if (binding.table_index == aggr.aggregate_index) { - if (binding.column_index < aggr.expressions.size()) { - auto &agg_expr = aggr.expressions[binding.column_index]; - if (agg_expr->type == ExpressionType::BOUND_AGGREGATE) { - auto &bound_agg = agg_expr->Cast(); - if (IsAnyPacAggregate(bound_agg.function.name)) { - if (out_source_binding) { - *out_source_binding = binding; - } - return GetBasePacAggregateName(bound_agg.function.name); - } - } + if (source->type == LogicalOperatorType::LOGICAL_PROJECTION) { + auto &projection = source->Cast(); + if (binding.column_index < projection.expressions.size()) { + CollectPacSources(*projection.expressions[binding.column_index], root, sources, visited); } - } - } - return ""; -} - -// Helper to collect ALL distinct PAC aggregate bindings in an expression -// Returns the bindings in the order they were discovered -static void CollectPacBindingsInExpression(Expression *expr, LogicalOperator *root, vector &bindings, - unordered_map &binding_hash_to_index) { - // Check if this is a column reference that traces back to a PAC aggregate - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { - auto &col_ref = expr->Cast(); - ColumnBinding source_binding; - string pac_name = TracePacAggregateFromBinding(col_ref.binding, root, &source_binding); - if (!pac_name.empty()) { - uint64_t binding_hash = HashBinding(col_ref.binding); - if (binding_hash_to_index.find(binding_hash) == binding_hash_to_index.end()) { - PacBindingInfo info; - info.binding = col_ref.binding; - info.index = bindings.size(); - info.source_binding = source_binding; - binding_hash_to_index[binding_hash] = info.index; - bindings.push_back(info); + } else if (source->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + auto &aggregate = source->Cast(); + if (binding.table_index != aggregate.aggregate_index || + binding.column_index >= aggregate.expressions.size()) { + return; + } + auto &aggregate_expr = *aggregate.expressions[binding.column_index]; + if (aggregate_expr.GetExpressionType() == ExpressionType::BOUND_AGGREGATE && + IsAnyPacAggregate( + aggregate_expr.Cast().Function().GetName().GetIdentifierName())) { + sources.push_back(binding); + } else { + CollectPacSources(aggregate_expr, root, sources, visited); + } + } else if (source->type == LogicalOperatorType::LOGICAL_CTE_REF) { + auto &ref = source->Cast(); + PRIVACY_DEBUG_PRINT("Categorical detection: tracing CTE reader #" + std::to_string(ref.table_index.index)); + auto *cte = FindMaterializedCTE(&root, ref.cte_index.index); + if (cte) { + auto bindings = cte->children[0]->GetColumnBindings(); + if (binding.column_index < bindings.size()) { + BoundColumnRefExpression definition_ref(expr.GetReturnType(), bindings[binding.column_index]); + CollectPacSources(definition_ref, root, sources, visited); + } } } + return; } - ExpressionIterator::EnumerateChildren(*expr, [&](Expression &child) { - CollectPacBindingsInExpression(&child, root, bindings, binding_hash_to_index); - }); + ExpressionIterator::EnumerateChildren( + expr, [&](const Expression &child) { CollectPacSources(child, root, sources, visited); }); } -// Find all PAC aggregate bindings in an expression vector FindAllPacBindingsInExpression(Expression *expr, LogicalOperator *plan_root) { - vector bindings; - unordered_map binding_hash_to_index; - CollectPacBindingsInExpression(expr, plan_root, bindings, binding_hash_to_index); - return bindings; -} - -// Recursively search for PAC aggregate in an expression tree, with plan context for tracing column refs -// Returns the base aggregate name (without _counters suffix) -static string FindPacAggregateInExpression(Expression *expr, LogicalOperator *plan_root, - ColumnBinding *out_source_binding) { - // examine specific expression types where the PAC aggregate could be in - if (expr->type == ExpressionType::BOUND_AGGREGATE) { // Base case: direct PAC aggregate - auto &agg_expr = expr->Cast(); - if (IsAnyPacAggregate(agg_expr.function.name)) { - return GetBasePacAggregateName(agg_expr.function.name); - } - } else if (expr->type == ExpressionType::BOUND_COLUMN_REF) { - auto &col_ref = expr->Cast(); - return TracePacAggregateFromBinding(col_ref.binding, plan_root, out_source_binding); - } else if (expr->type == ExpressionType::SUBQUERY) { - auto &subquery_expr = expr->Cast(); - for (auto &child : subquery_expr.children) { - string result = FindPacAggregateInExpression(child.get(), plan_root); - if (!result.empty()) { - return result; // Found PAC aggregate in the subquery's children (for IN, ANY, ALL operators) - } - } - if (subquery_expr.subquery_type == SubqueryType::SCALAR) { - string result = FindPacAggregateInOperator(plan_root); - if (!result.empty()) { - return result; // found PAC aggregate in scalar subquery + vector result; + unordered_set seen; + std::function collect = [&](const Expression &expression) { + if (expression.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { + auto binding = expression.Cast().Binding(); + if (seen.insert(HashBinding(binding)).second) { + vector sources; + unordered_set visited; + CollectPacSources(expression, *plan_root, sources, visited); + for (auto &source : sources) { + PacBindingInfo info; + info.binding = binding; + info.source_binding = source; + info.index = result.size(); + result.push_back(info); + } } } - return ""; - } - // Generic traversal for all other expression types (comparisons, operators, casts, - // functions, constants, CASE, BETWEEN, conjunctions, window functions, etc.) - string result; - ExpressionIterator::EnumerateChildren(*expr, [&](Expression &child) { - if (result.empty()) { - result = FindPacAggregateInExpression(&child, plan_root, out_source_binding); - } - }); + ExpressionIterator::EnumerateChildren(expression, collect); + }; + collect(*expr); return result; } @@ -146,7 +105,7 @@ static bool IsHavingBinding(LogicalOperator *filter_op, const ColumnBinding &bin LogicalOperator *child = filter_op->children[0].get(); while (child && child->type == LogicalOperatorType::LOGICAL_PROJECTION) { auto &proj = child->Cast(); - if (binding.table_index == proj.table_index) { + if (binding.table_index.index == proj.table_index.index) { return true; } if (child->children.empty()) { @@ -156,7 +115,8 @@ static bool IsHavingBinding(LogicalOperator *filter_op, const ColumnBinding &bin } if (child && child->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { auto &agg = child->Cast(); - return binding.table_index == agg.aggregate_index || binding.table_index == agg.group_index; + return binding.table_index.index == agg.aggregate_index.index || + binding.table_index.index == agg.group_index.index; } return false; } @@ -178,16 +138,18 @@ void DetectCategoricalPatterns(LogicalOperator *op, LogicalOperator *plan_root, // Fallback: extract from children[0] of the first PAC aggregate expression. if (op->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { auto &agg = op->Cast(); - auto it = pac_agg_info.find(agg.aggregate_index); + auto it = pac_agg_info.find(agg.aggregate_index.index); if (it != pac_agg_info.end()) { hash_for_children = it->second.hash_binding; } else { for (auto &agg_expr : agg.expressions) { - if (agg_expr->type == ExpressionType::BOUND_AGGREGATE) { + if (agg_expr->GetExpressionType() == ExpressionType::BOUND_AGGREGATE) { auto &bound_agg = agg_expr->Cast(); - if (IsPacAggregate(bound_agg.function.name) && !bound_agg.children.empty() && - bound_agg.children[0]->type == ExpressionType::BOUND_COLUMN_REF) { - hash_for_children = bound_agg.children[0]->Cast().binding; + if (IsPacAggregate(bound_agg.FunctionMutable().GetName().GetIdentifierName()) && + !bound_agg.GetChildrenMutable().empty() && + bound_agg.GetChildrenMutable()[0]->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { + hash_for_children = + bound_agg.GetChildrenMutable()[0]->Cast().BindingMutable(); break; } } @@ -196,14 +158,14 @@ void DetectCategoricalPatterns(LogicalOperator *op, LogicalOperator *plan_root, } // Resolve hash binding through projections (follow col_ref chains) - if (hash_for_children.table_index != DConstants::INVALID_INDEX && + if (hash_for_children.table_index.index != DConstants::INVALID_INDEX && op->type == LogicalOperatorType::LOGICAL_PROJECTION) { auto &proj = op->Cast(); - if (hash_for_children.table_index == proj.table_index && + if (hash_for_children.table_index.index == proj.table_index.index && hash_for_children.column_index < proj.expressions.size()) { auto &expr = proj.expressions[hash_for_children.column_index]; - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { - hash_for_children = expr->Cast().binding; + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { + hash_for_children = expr->Cast().BindingMutable(); } } } @@ -238,7 +200,7 @@ void DetectCategoricalPatterns(LogicalOperator *op, LogicalOperator *plan_root, info.expr_index = i; info.pac_bindings = std::move(pac_bindings); info.outer_pac_hash = hash_for_children; - info.has_outer_pac_hash = (hash_for_children.table_index != DConstants::INVALID_INDEX); + info.has_outer_pac_hash = (hash_for_children.table_index.index != DConstants::INVALID_INDEX); patterns.push_back(info); } } @@ -247,20 +209,26 @@ void DetectCategoricalPatterns(LogicalOperator *op, LogicalOperator *plan_root, auto &join = op->Cast(); for (idx_t i = 0; i < join.conditions.size(); i++) { auto &cond = join.conditions[i]; - // Check both sides; any PAC aggregate in a join condition is categorical (not HAVING). - for (auto *side : {cond.left.get(), cond.right.get()}) { - string pac_name = FindPacAggregateInExpression(side, plan_root); - if (pac_name.empty()) { - continue; + vector bindings; + unordered_set seen; + for (auto *side : {cond.LeftReference().get(), cond.RightReference().get()}) { + for (auto &binding : FindAllPacBindingsInExpression(side, plan_root)) { + if (seen.insert(HashBinding(binding.source_binding)).second) { + binding.index = bindings.size(); + bindings.push_back(binding); + } } - CategoricalPatternInfo info; - info.parent_op = op; - info.expr_index = i; - info.outer_pac_hash = hash_for_children; - info.has_outer_pac_hash = (hash_for_children.table_index != DConstants::INVALID_INDEX); - patterns.push_back(info); - break; // one match per condition is enough } + if (bindings.empty()) { + continue; + } + CategoricalPatternInfo info; + info.parent_op = op; + info.expr_index = i; + info.pac_bindings = std::move(bindings); + info.outer_pac_hash = hash_for_children; + info.has_outer_pac_hash = (hash_for_children.table_index.index != DConstants::INVALID_INDEX); + patterns.push_back(std::move(info)); } } else if (!inside_aggregate && patterns.empty() && op->type == LogicalOperatorType::LOGICAL_PROJECTION) { // Check projection expressions for arithmetic involving multiple PAC aggregates @@ -271,7 +239,7 @@ void DetectCategoricalPatterns(LogicalOperator *op, LogicalOperator *plan_root, auto &proj = op->Cast(); for (idx_t i = 0; i < proj.expressions.size(); i++) { auto &expr = proj.expressions[i]; - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { continue; // Skip simple column references - they don't need rewriting } // Find ALL PAC aggregate bindings in this expression @@ -287,18 +255,19 @@ void DetectCategoricalPatterns(LogicalOperator *op, LogicalOperator *plan_root, is_arithmetic_with_pac = true; } else if (pac_bindings.size() == 1) { // Single aggregate - check if it's in an arithmetic expression (not just a column ref or simple cast) - if (expr->type != ExpressionType::BOUND_COLUMN_REF && expr->type != ExpressionType::OPERATOR_CAST) { + if (expr->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF && + expr->GetExpressionType() != ExpressionType::OPERATOR_CAST) { is_arithmetic_with_pac = true; - } else if (expr->type == ExpressionType::OPERATOR_CAST) { + } else if (expr->GetExpressionType() == ExpressionType::OPERATOR_CAST) { // Check if cast contains arithmetic - auto &cast_expr = expr->Cast(); - if (cast_expr.child->type != ExpressionType::BOUND_COLUMN_REF) { + auto &cast_expr = expr->Cast(); + if (cast_expr.GetChildrenMutable()[0]->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) { is_arithmetic_with_pac = true; } } } if (is_arithmetic_with_pac) { - if (!IsNumericalType(expr->return_type)) { + if (!IsNumericalType(expr->GetReturnType())) { continue; // Only create pattern if result is numerical (as_noised only works on numbers) } CategoricalPatternInfo info; diff --git a/src/categorical/pac_categorical_lambdas.cpp b/src/categorical/pac_categorical_lambdas.cpp index fc6d8f01..f55abc1b 100644 --- a/src/categorical/pac_categorical_lambdas.cpp +++ b/src/categorical/pac_categorical_lambdas.cpp @@ -19,55 +19,45 @@ namespace duckdb { // pac_binding_map: maps binding hash -> struct field index (single: one entry mapping to 0) // struct_type: nullptr = single binding (elem ref), non-null = multi binding (struct field extract) static unique_ptr -CloneForLambdaBody(Expression *expr, const unordered_map &pac_binding_map, +CloneForLambdaBody(ClientContext &context, Expression *expr, const unordered_map &pac_binding_map, vector> &captures, unordered_map &capture_map, const unordered_map &counter_bindings, const LogicalType *struct_type) { // Helper to build the replacement expression for a matched PAC binding auto make_pac_replacement = [&](const unordered_map::const_iterator &it, - const string &alias) -> unique_ptr { + const Identifier &alias) -> unique_ptr { if (struct_type) { idx_t field_idx = it->second; - string field_name = GetStructFieldName(field_idx); auto elem_ref = make_uniq("elem", *struct_type, idx_t(0)); - auto child_types = StructType::GetChildTypes(*struct_type); - LogicalType extract_return_type = PacFloatLogicalType(); - for (idx_t j = 0; j < child_types.size(); j++) { - if (child_types[j].first == field_name) { - extract_return_type = child_types[j].second; - break; - } - } auto extract_func = StructExtractAtFun::GetFunction(); - auto bind_data = StructExtractAtFun::GetBindData(field_idx); + vector> extract_children; extract_children.push_back(std::move(elem_ref)); extract_children.push_back( make_uniq(Value::BIGINT(static_cast(field_idx + 1)))); - return make_uniq(extract_return_type, extract_func, std::move(extract_children), - std::move(bind_data)); + return FunctionBinder(context).BindScalarFunction(extract_func, std::move(extract_children)); } else { return make_uniq(alias, PacFloatLogicalType(), idx_t(0)); } }; // rewrite all kinds of expressions: - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr->Cast(); - uint64_t binding_hash = HashBinding(col_ref.binding); + uint64_t binding_hash = HashBinding(col_ref.BindingMutable()); // Direct match auto it = pac_binding_map.find(binding_hash); if (it != pac_binding_map.end()) { return make_pac_replacement(it, "elem"); } // Trace through counter_bindings to find the aggregate-level PAC binding - auto cb_it = counter_bindings.find(HashBinding(col_ref.binding)); + auto cb_it = counter_bindings.find(HashBinding(col_ref.BindingMutable())); if (cb_it != counter_bindings.end()) { auto traced_it = pac_binding_map.find(HashBinding(cb_it->second)); if (traced_it != pac_binding_map.end()) { - return make_pac_replacement(traced_it, col_ref.alias); + return make_pac_replacement(traced_it, col_ref.GetAlias()); } } // Other column ref — capture for use in lambda - uint64_t hash = HashBinding(col_ref.binding); + uint64_t hash = HashBinding(col_ref.BindingMutable()); idx_t capture_idx; auto cap_it = capture_map.find(hash); if (cap_it != capture_map.end()) { @@ -77,128 +67,130 @@ CloneForLambdaBody(Expression *expr, const unordered_map &pac_b capture_map[hash] = capture_idx; captures.push_back(col_ref.Copy()); } - return make_uniq(col_ref.alias, col_ref.return_type, 1 + capture_idx); - } else if (expr->type == ExpressionType::VALUE_CONSTANT) { + return make_uniq(col_ref.GetAlias(), col_ref.GetReturnType(), 1 + capture_idx); + } else if (expr->GetExpressionType() == ExpressionType::VALUE_CONSTANT) { return expr->Copy(); - } else if (expr->type == ExpressionType::OPERATOR_CAST) { - auto &cast = expr->Cast(); - auto child_clone = - CloneForLambdaBody(cast.child.get(), pac_binding_map, captures, capture_map, counter_bindings, struct_type); - if (child_clone->return_type == cast.return_type) { + } else if (expr->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + auto &cast = expr->Cast(); + auto child_clone = CloneForLambdaBody(context, cast.GetChildrenMutable()[0].get(), pac_binding_map, captures, + capture_map, counter_bindings, struct_type); + if (child_clone->GetReturnType() == cast.GetReturnType()) { return child_clone; // If the child's type already matches the target type, skip the cast } // Otherwise, create a new cast with the correct function for the new child type - return BoundCastExpression::AddDefaultCastToType(std::move(child_clone), cast.return_type); - } else if (expr->GetExpressionClass() == ExpressionClass::BOUND_COMPARISON) { - auto &comp = expr->Cast(); - auto left_clone = - CloneForLambdaBody(comp.left.get(), pac_binding_map, captures, capture_map, counter_bindings, struct_type); - auto right_clone = - CloneForLambdaBody(comp.right.get(), pac_binding_map, captures, capture_map, counter_bindings, struct_type); + return BoundCastExpression::AddDefaultCastToType(std::move(child_clone), cast.GetReturnType()); + } else if (BoundComparisonExpression::IsComparison(*expr)) { + auto &comp = expr->Cast(); + auto left_clone = CloneForLambdaBody(context, comp.GetChildrenMutable()[0].get(), pac_binding_map, captures, + capture_map, counter_bindings, struct_type); + auto right_clone = CloneForLambdaBody(context, comp.GetChildrenMutable()[1].get(), pac_binding_map, captures, + capture_map, counter_bindings, struct_type); // Reconcile types if they differ (needed for multi where struct fields are DOUBLE // but original CASTs may introduce DECIMAL) - if (left_clone->return_type != right_clone->return_type) { - if (left_clone->return_type != PacFloatLogicalType()) { + if (left_clone->GetReturnType() != right_clone->GetReturnType()) { + if (left_clone->GetReturnType() != PacFloatLogicalType()) { left_clone = BoundCastExpression::AddDefaultCastToType(std::move(left_clone), PacFloatLogicalType()); } - if (right_clone->return_type != PacFloatLogicalType()) { + if (right_clone->GetReturnType() != PacFloatLogicalType()) { right_clone = BoundCastExpression::AddDefaultCastToType(std::move(right_clone), PacFloatLogicalType()); } } - return make_uniq(expr->type, std::move(left_clone), std::move(right_clone)); - } else if (expr->type == ExpressionType::BOUND_FUNCTION) { + return BoundComparisonExpression::Create(expr->GetExpressionType(), std::move(left_clone), + std::move(right_clone)); + } else if (expr->GetExpressionType() == ExpressionType::BOUND_FUNCTION) { auto &func = expr->Cast(); vector> new_children; bool any_cast_needed = false; - for (idx_t i = 0; i < func.children.size(); i++) { - auto child_clone = CloneForLambdaBody(func.children[i].get(), pac_binding_map, captures, capture_map, - counter_bindings, struct_type); + for (idx_t i = 0; i < func.GetChildrenMutable().size(); i++) { + auto child_clone = CloneForLambdaBody(context, func.GetChildrenMutable()[i].get(), pac_binding_map, + captures, capture_map, counter_bindings, struct_type); // If a child's type changed (e.g., DECIMAL->DOUBLE from PAC counter conversion), // cast it to the type the bound function expects, so the function binding stays valid. - if (i < func.function.arguments.size() && child_clone->return_type != func.function.arguments[i]) { - child_clone = - BoundCastExpression::AddDefaultCastToType(std::move(child_clone), func.function.arguments[i]); + if (i < func.FunctionMutable().GetArguments().size() && + child_clone->GetReturnType() != func.FunctionMutable().GetArguments()[i]) { + child_clone = BoundCastExpression::AddDefaultCastToType(std::move(child_clone), + func.FunctionMutable().GetArguments()[i]); any_cast_needed = true; } new_children.push_back(std::move(child_clone)); } unique_ptr result = - make_uniq(func.return_type, func.function, std::move(new_children), - func.bind_info ? func.bind_info->Copy() : nullptr); + make_uniq(func.FunctionMutable(), std::move(new_children), + func.BindInfoMutable() ? func.BindInfoMutable()->Copy() : nullptr); // Children were cast from PAC_FLOAT to the function's bound types (e.g., DECIMAL). // Cast the result back to PAC_FLOAT so the list_transform output stays LIST. - if (any_cast_needed && result->return_type != PacFloatLogicalType()) { + if (any_cast_needed && result->GetReturnType() != PacFloatLogicalType()) { result = BoundCastExpression::AddDefaultCastToType(std::move(result), PacFloatLogicalType()); } return result; } else if (expr->GetExpressionClass() == ExpressionClass::BOUND_OPERATOR) { // NOT, arithmetic, COALESCE, IS NULL.. auto &op = expr->Cast(); vector> new_children; - for (auto &child : op.children) { - new_children.push_back( - CloneForLambdaBody(child.get(), pac_binding_map, captures, capture_map, counter_bindings, struct_type)); + for (auto &child : op.GetChildrenMutable()) { + new_children.push_back(CloneForLambdaBody(context, child.get(), pac_binding_map, captures, capture_map, + counter_bindings, struct_type)); } // For COALESCE with mismatched child types, cast all to the first child's type - LogicalType result_type = op.return_type; - if (expr->type == ExpressionType::OPERATOR_COALESCE && new_children.size() > 1) { - LogicalType first_type = new_children[0]->return_type; + LogicalType result_type = op.GetReturnType(); + if (expr->GetExpressionType() == ExpressionType::OPERATOR_COALESCE && new_children.size() > 1) { + LogicalType first_type = new_children[0]->GetReturnType(); bool types_mismatch = false; for (idx_t ci = 1; ci < new_children.size(); ci++) { - if (new_children[ci]->return_type != first_type) { + if (new_children[ci]->GetReturnType() != first_type) { types_mismatch = true; break; } } if (types_mismatch) { for (auto &c : new_children) { - if (c->return_type != first_type) { + if (c->GetReturnType() != first_type) { c = BoundCastExpression::AddDefaultCastToType(std::move(c), first_type); } } result_type = first_type; } } - auto op_result = make_uniq(expr->type, result_type); + auto op_result = make_uniq(expr->GetExpressionType(), result_type); for (auto &c : new_children) { - op_result->children.push_back(std::move(c)); + op_result->GetChildrenMutable().push_back(std::move(c)); } return op_result; } else if (expr->GetExpressionClass() == ExpressionClass::BOUND_CONJUNCTION) { // AND, OR auto &conj = expr->Cast(); - auto result = make_uniq(expr->type); - for (auto &child : conj.children) { - result->children.push_back( - CloneForLambdaBody(child.get(), pac_binding_map, captures, capture_map, counter_bindings, struct_type)); + auto result = make_uniq(expr->GetExpressionType()); + for (auto &child : conj.GetChildrenMutable()) { + result->GetChildrenMutable().push_back(CloneForLambdaBody(context, child.get(), pac_binding_map, captures, + capture_map, counter_bindings, struct_type)); } return result; - } else if (expr->type == ExpressionType::CASE_EXPR) { + } else if (expr->GetExpressionType() == ExpressionType::CASE_EXPR) { auto &case_expr = expr->Cast(); if (IsScalarSubqueryWrapper(case_expr)) { - return CloneForLambdaBody(case_expr.else_expr.get(), pac_binding_map, captures, capture_map, + return CloneForLambdaBody(context, case_expr.ElseMutable().get(), pac_binding_map, captures, capture_map, counter_bindings, struct_type); } // Regular CASE - recurse into all branches // Start with original return type, will update based on cloned branches - auto result = make_uniq(case_expr.return_type); - for (auto &check : case_expr.case_checks) { + auto result = make_uniq(case_expr.GetReturnType()); + for (auto &check : case_expr.CaseChecksMutable()) { BoundCaseCheck new_check; - new_check.when_expr = CloneForLambdaBody(check.when_expr.get(), pac_binding_map, captures, capture_map, - counter_bindings, struct_type); - new_check.then_expr = CloneForLambdaBody(check.then_expr.get(), pac_binding_map, captures, capture_map, - counter_bindings, struct_type); - result->case_checks.push_back(std::move(new_check)); + new_check.when_expr = CloneForLambdaBody(context, check.when_expr.get(), pac_binding_map, captures, + capture_map, counter_bindings, struct_type); + new_check.then_expr = CloneForLambdaBody(context, check.then_expr.get(), pac_binding_map, captures, + capture_map, counter_bindings, struct_type); + result->CaseChecksMutable().push_back(std::move(new_check)); } - if (case_expr.else_expr) { - result->else_expr = CloneForLambdaBody(case_expr.else_expr.get(), pac_binding_map, captures, capture_map, - counter_bindings, struct_type); + if (case_expr.ElseMutable()) { + result->ElseMutable() = CloneForLambdaBody(context, case_expr.ElseMutable().get(), pac_binding_map, + captures, capture_map, counter_bindings, struct_type); // Update return type to match ELSE branch (the PAC element type) - result->return_type = result->else_expr->return_type; + result->SetReturnType(result->ElseMutable()->GetReturnType()); } // Cast THEN branches to match the return type if needed - for (auto &check : result->case_checks) { - if (check.then_expr && check.then_expr->return_type != result->return_type) { + for (auto &check : result->CaseChecksMutable()) { + if (check.then_expr && check.then_expr->GetReturnType() != result->GetReturnType()) { check.then_expr = - BoundCastExpression::AddDefaultCastToType(std::move(check.then_expr), result->return_type); + BoundCastExpression::AddDefaultCastToType(std::move(check.then_expr), result->GetReturnType()); } } return result; @@ -211,7 +203,7 @@ static unique_ptr BuildPacLambda(unique_ptr lambda_body, vector> captures) { auto lambda = make_uniq(ExpressionType::LAMBDA, LogicalType::LAMBDA, std::move(lambda_body), idx_t(1)); - lambda->captures = std::move(captures); + lambda->CapturesMutable() = std::move(captures); return lambda; } @@ -221,29 +213,21 @@ static unique_ptr BuildListTransformCall(OptimizerExtensionInput &in unique_ptr counters_list, unique_ptr lambda_expr, const LogicalType &element_return_type = LogicalType::BOOLEAN) { - // Get the lambda body for ListLambdaBindData - auto &bound_lambda = lambda_expr->Cast(); - - // Get list_transform function from catalog - auto &catalog = Catalog::GetSystemCatalog(input.context); - auto &func_entry = catalog.GetEntry(input.context, DEFAULT_SCHEMA, "list_transform"); - - // Get the first function overload (list_transform has only one signature pattern) - auto &scalar_func = func_entry.functions.functions[0]; - - // Create the ListLambdaBindData with the lambda body - auto list_return_type = LogicalType::LIST(element_return_type); - auto bind_data = make_uniq(list_return_type, std::move(bound_lambda.lambda_expr), false, false); - - // Build children: [list, captures...] Note: The lambda itself is NOT a child after binding - only its captures are - vector> children; - children.push_back(std::move(counters_list)); - - for (auto &capture : bound_lambda.captures) { - children.push_back(std::move(capture)); // Add captures as children + vector> arguments; + arguments.push_back(std::move(counters_list)); + arguments.push_back(std::move(lambda_expr)); + auto result = + FunctionBinder(input.context).BindScalarFunction(DEFAULT_SCHEMA, "list_transform", std::move(arguments)); + D_ASSERT(result->GetReturnType() == LogicalType::LIST(element_return_type)); + auto &function = result->Cast(); + auto &lambda = function.GetChildrenMutable()[1]->Cast(); + // Match DuckDB's lambda binder: the placeholder stays at argument 1 and + // captured row values are appended as physical children. + for (auto &capture : lambda.CapturesMutable()) { + function.GetChildrenMutable().push_back(std::move(capture)); } - // Create the bound function expression - return make_uniq(list_return_type, scalar_func, std::move(children), std::move(bind_data)); + lambda.CapturesMutable().clear(); + return result; } // Build a list_zip function call combining multiple counter lists @@ -251,41 +235,19 @@ static unique_ptr BuildListTransformCall(OptimizerExtensionInput &in static unique_ptr BuildListZipCall(OptimizerExtensionInput &input, vector> counter_lists, LogicalType &out_struct_type) { - // Build the struct type for list_zip result -- list_zip returns LIST> - child_list_t struct_children; - for (idx_t i = 0; i < counter_lists.size(); i++) { - string field_name = GetStructFieldName(i); - struct_children.push_back( - make_pair(field_name, PacFloatLogicalType())); // All counter lists use PAC_FLOAT element type - } - out_struct_type = LogicalType::STRUCT(struct_children); - auto list_struct_type = LogicalType::LIST(out_struct_type); - - // Get list_zip function from catalog - auto &catalog = Catalog::GetSystemCatalog(input.context); - auto &func_entry = catalog.GetEntry(input.context, DEFAULT_SCHEMA, "list_zip"); - - // Find the appropriate overload (list_zip is variadic) - vector arg_types; - for (auto &list : counter_lists) { - arg_types.push_back(list->return_type); - } - ErrorData error; - FunctionBinder function_binder(input.context); - auto best_function = function_binder.BindFunction(func_entry.name, func_entry.functions, arg_types, error); - if (best_function.IsValid()) { - auto scalar_func = func_entry.functions.GetFunctionByOffset(best_function.GetIndex()); - return make_uniq(list_struct_type, scalar_func, std::move(counter_lists), nullptr); - } - return nullptr; + auto result = + FunctionBinder(input.context).BindScalarFunction(DEFAULT_SCHEMA, "list_zip", std::move(counter_lists)); + out_struct_type = ListType::GetChildType(result->GetReturnType()); + return result; } // Check whether an expression tree contains any null-handling operators // (COALESCE, IS NULL, IS NOT NULL). When absent, priv_coalesce is unnecessary // because NULL propagates identically through arithmetic. static bool ExpressionContainsNullHandling(Expression *expr) { - if (expr->type == ExpressionType::OPERATOR_COALESCE || expr->type == ExpressionType::OPERATOR_IS_NULL || - expr->type == ExpressionType::OPERATOR_IS_NOT_NULL) { + if (expr->GetExpressionType() == ExpressionType::OPERATOR_COALESCE || + expr->GetExpressionType() == ExpressionType::OPERATOR_IS_NULL || + expr->GetExpressionType() == ExpressionType::OPERATOR_IS_NOT_NULL) { return true; } bool found = false; @@ -300,15 +262,15 @@ static bool ExpressionContainsNullHandling(Expression *expr) { // Check which child of a binary function contains a counter binding // Returns 0 if first child has counter, 1 if second child has counter, -1 if neither or both static int FindPacChildIndex(Expression *expr, const unordered_map &counter_bindings) { - if (expr->type != ExpressionType::BOUND_FUNCTION) { + if (expr->GetExpressionType() != ExpressionType::BOUND_FUNCTION) { return -1; } auto &func = expr->Cast(); - if (func.children.size() != 2) { + if (func.GetChildrenMutable().size() != 2) { return -1; } - bool left_has = ReferencesCounterBinding(func.children[0].get(), counter_bindings); - bool right_has = ReferencesCounterBinding(func.children[1].get(), counter_bindings); + bool left_has = ReferencesCounterBinding(func.GetChildrenMutable()[0].get(), counter_bindings); + bool right_has = ReferencesCounterBinding(func.GetChildrenMutable()[1].get(), counter_bindings); if (left_has && !right_has) { return 0; } @@ -339,10 +301,10 @@ unique_ptr BuildCategoricalLambdas(OptimizerExtensionInput &input, pac_binding_map[HashBinding(pac_binding)] = 0; vector> captures; unordered_map capture_map; - auto lambda_body = - CloneForLambdaBody(expr_to_transform, pac_binding_map, captures, capture_map, counter_bindings, nullptr); + auto lambda_body = CloneForLambdaBody(input.context, expr_to_transform, pac_binding_map, captures, capture_map, + counter_bindings, nullptr); // Ensure body returns result_element_type if needed - if (result_element_type == PacFloatLogicalType() && lambda_body->return_type != PacFloatLogicalType()) { + if (result_element_type == PacFloatLogicalType() && lambda_body->GetReturnType() != PacFloatLogicalType()) { lambda_body = BoundCastExpression::AddDefaultCastToType(std::move(lambda_body), PacFloatLogicalType()); } auto lambda = BuildPacLambda(std::move(lambda_body), std::move(captures)); @@ -366,9 +328,9 @@ unique_ptr BuildCategoricalLambdas(OptimizerExtensionInput &input, if (zipped_list) { vector> captures; unordered_map map; - auto lambda_body = - CloneForLambdaBody(expr_to_transform, binding_to_index, captures, map, counter_bindings, &struct_type); - if (result_element_type == PacFloatLogicalType() && lambda_body->return_type != PacFloatLogicalType()) { + auto lambda_body = CloneForLambdaBody(input.context, expr_to_transform, binding_to_index, captures, map, + counter_bindings, &struct_type); + if (result_element_type == PacFloatLogicalType() && lambda_body->GetReturnType() != PacFloatLogicalType()) { lambda_body = BoundCastExpression::AddDefaultCastToType(std::move(lambda_body), PacFloatLogicalType()); } auto lambda = BuildPacLambda(std::move(lambda_body), std::move(captures)); @@ -382,15 +344,15 @@ unique_ptr TryRewriteFilterComparison(OptimizerExtensionInput &input const vector &pac_bindings, Expression *expr, const unordered_map &counter_bindings, PacWrapKind wrap_kind, ColumnBinding priv_hash) { - if (pac_bindings.size() != 1 || expr->GetExpressionClass() != ExpressionClass::BOUND_COMPARISON) { + if (pac_bindings.size() != 1 || !BoundComparisonExpression::IsComparison(*expr)) { return nullptr; // Only works with a comparison against a single PAC binding } - auto &comp = expr->Cast(); - ExpressionType cmp_type = comp.type; + auto &comp = expr->Cast(); + ExpressionType cmp_type = comp.GetExpressionType(); // Determine which side has the PAC binding - auto left_bindings = FindCounterBindings(comp.left.get(), counter_bindings); - auto right_bindings = FindCounterBindings(comp.right.get(), counter_bindings); + auto left_bindings = FindCounterBindings(comp.GetChildrenMutable()[0].get(), counter_bindings); + auto right_bindings = FindCounterBindings(comp.GetChildrenMutable()[1].get(), counter_bindings); bool left_has_pac = !left_bindings.empty(); bool right_has_pac = !right_bindings.empty(); @@ -401,17 +363,17 @@ unique_ptr TryRewriteFilterComparison(OptimizerExtensionInput &input unique_ptr scalar_side; unique_ptr list_side; if (left_has_pac) { // PAC on left: flip comparison - scalar_side = comp.right->Copy(); - list_side = comp.left->Copy(); + scalar_side = comp.GetChildrenMutable()[1]->Copy(); + list_side = comp.GetChildrenMutable()[0]->Copy(); cmp_type = FlipComparison(cmp_type); } else { - scalar_side = comp.left->Copy(); - list_side = comp.right->Copy(); + scalar_side = comp.GetChildrenMutable()[0]->Copy(); + list_side = comp.GetChildrenMutable()[1]->Copy(); } // Iteratively simplify: move arithmetic from list_side to scalar_side - while (list_side->type == ExpressionType::BOUND_FUNCTION) { + while (list_side->GetExpressionType() == ExpressionType::BOUND_FUNCTION) { auto &func = list_side->Cast(); - if (func.children.size() != 2) { + if (func.GetChildrenMutable().size() != 2) { break; } int pac_child = FindPacChildIndex(list_side.get(), counter_bindings); @@ -419,11 +381,12 @@ unique_ptr TryRewriteFilterComparison(OptimizerExtensionInput &input break; } bool needs_positive_check = false; - const char *inverse_op = GetInverseArithmeticOp(func.function.name, pac_child, needs_positive_check); + const char *inverse_op = GetInverseArithmeticOp(func.FunctionMutable().GetName().GetIdentifierName(), pac_child, + needs_positive_check); if (!inverse_op) { break; } - auto &scalar_operand = func.children[1 - pac_child]; + auto &scalar_operand = func.GetChildrenMutable()[1 - pac_child]; if (needs_positive_check && !IsPositiveConstant(scalar_operand.get())) { break; } @@ -437,32 +400,32 @@ unique_ptr TryRewriteFilterComparison(OptimizerExtensionInput &input scalar_side = input.optimizer.BindScalarFunction(inverse_op, std::move(scalar_side), scalar_operand->Copy()); } - list_side = func.children[pac_child]->Copy(); + list_side = func.GetChildrenMutable()[pac_child]->Copy(); } // After simplification, list_side should be a bare column ref (possibly through casts) Expression *stripped = StripCasts(list_side.get()); - if (stripped->type != ExpressionType::BOUND_COLUMN_REF) { + if (stripped->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) { // Can't emit priv_filter_, but return the simplified comparison // so the lambda path benefits from the algebraic rewriting - return make_uniq(cmp_type, std::move(scalar_side), std::move(list_side)); + return BoundComparisonExpression::Create(cmp_type, std::move(scalar_side), std::move(list_side)); } // Verify the column ref is a counter binding auto &col_ref = stripped->Cast(); - if (!counter_bindings.count(HashBinding(col_ref.binding))) { - return make_uniq(cmp_type, std::move(scalar_side), std::move(list_side)); + if (!counter_bindings.count(HashBinding(col_ref.BindingMutable()))) { + return BoundComparisonExpression::Create(cmp_type, std::move(scalar_side), std::move(list_side)); } // Build priv_{filter,select}_(scalar_side, counters) or priv_select_(hash, scalar, counters) string privacy_mode = GetPrivacyMode(input.context); const char *func_name = (wrap_kind == PacWrapKind::PAC_SELECT) ? GetPacSelectCmpName(cmp_type, privacy_mode) : GetPacFilterCmpName(cmp_type, privacy_mode); if (!func_name) { - return make_uniq(cmp_type, std::move(scalar_side), std::move(list_side)); + return BoundComparisonExpression::Create(cmp_type, std::move(scalar_side), std::move(list_side)); } - if (scalar_side->return_type != PacFloatLogicalType()) { + if (scalar_side->GetReturnType() != PacFloatLogicalType()) { scalar_side = BoundCastExpression::AddDefaultCastToType(std::move(scalar_side), PacFloatLogicalType()); } - auto counters_ref = - make_uniq("pac_var", LogicalType::LIST(PacFloatLogicalType()), col_ref.binding); + auto counters_ref = make_uniq("pac_var", LogicalType::LIST(PacFloatLogicalType()), + col_ref.BindingMutable()); if (wrap_kind == PacWrapKind::PAC_SELECT) { // priv_select_(hash, scalar, counters) — 3 args, need manual binding @@ -473,21 +436,21 @@ unique_ptr TryRewriteFilterComparison(OptimizerExtensionInput &input args.push_back(std::move(counters_ref)); vector arg_types; for (auto &a : args) { - arg_types.push_back(a->return_type); + arg_types.push_back(a->GetReturnType()); } auto &catalog = Catalog::GetSystemCatalog(input.context); - auto &entry = catalog.GetEntry(input.context, DEFAULT_SCHEMA, func_name); + auto &entry = + catalog.GetEntry(input.context, DEFAULT_SCHEMA, Identifier(func_name)); ErrorData error; FunctionBinder func_binder(input.context); - auto best = func_binder.BindFunction(func_name, entry.functions, arg_types, error); + auto best = func_binder.BindFunction(Identifier(func_name), entry.functions, arg_types, error); if (best.IsValid()) { auto func = entry.functions.GetFunctionByOffset(best.GetIndex()); - auto bind_info = func.bind ? func.bind(input.context, func, args) : nullptr; - return make_uniq(func.return_type, func, std::move(args), std::move(bind_info)); + return func_binder.BindScalarFunction(func, std::move(args)); } return nullptr; } - return input.optimizer.BindScalarFunction(func_name, std::move(scalar_side), std::move(counters_ref)); + return input.optimizer.BindScalarFunction(Identifier(func_name), std::move(scalar_side), std::move(counters_ref)); } } // namespace duckdb diff --git a/src/categorical/pac_categorical_rewriter.cpp b/src/categorical/pac_categorical_rewriter.cpp index d2e7cfa2..63a41ba1 100644 --- a/src/categorical/pac_categorical_rewriter.cpp +++ b/src/categorical/pac_categorical_rewriter.cpp @@ -5,6 +5,7 @@ // // Created by ila on 1/23/26. // +#include "privacy_debug.hpp" #include "categorical/pac_categorical_lambdas.hpp" #include "aggregates/as_aggregate.hpp" #include "utils/privacy_helpers.hpp" @@ -13,6 +14,8 @@ #include "duckdb/planner/expression/bound_conjunction_expression.hpp" #include "duckdb/planner/operator/logical_order.hpp" #include "duckdb/planner/operator/logical_top_n.hpp" +#include "duckdb/planner/operator/logical_materialized_cte.hpp" +#include "duckdb/planner/operator/logical_cteref.hpp" namespace duckdb { @@ -23,12 +26,12 @@ vector FindCounterBindings(Expression *expr, vector result; unordered_map seen; std::function Collect = [&](Expression *e) { - if (e->type == ExpressionType::BOUND_COLUMN_REF) { + if (e->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e->Cast(); - uint64_t h = HashBinding(col_ref.binding); + uint64_t h = HashBinding(col_ref.BindingMutable()); if (counter_bindings.count(h) && !seen.count(h)) { PacBindingInfo info; - info.binding = col_ref.binding; + info.binding = col_ref.BindingMutable(); info.index = result.size(); seen[h] = info.index; result.push_back(info); @@ -43,7 +46,7 @@ vector FindCounterBindings(Expression *expr, // Register a binding as a counter binding, optionally tracing to its origin. static void AddCounterBinding(unordered_map &counter_bindings, const ColumnBinding &binding, const ColumnBinding &origin = ColumnBinding()) { - counter_bindings[HashBinding(binding)] = (origin.table_index != DConstants::INVALID_INDEX) ? origin : binding; + counter_bindings[HashBinding(binding)] = (origin.table_index.index != DConstants::INVALID_INDEX) ? origin : binding; } // For a projection expression, trace back through counter_bindings to find the origin binding. @@ -51,9 +54,9 @@ static void AddCounterBinding(unordered_map &counter_bi // itself. static ColumnBinding TraceCounterOrigin(const unique_ptr &expr, const unordered_map &counter_bindings) { - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr->Cast(); - auto it = counter_bindings.find(HashBinding(col_ref.binding)); + auto it = counter_bindings.find(HashBinding(col_ref.BindingMutable())); if (it != counter_bindings.end()) { return it->second; } @@ -63,8 +66,8 @@ static ColumnBinding TraceCounterOrigin(const unique_ptr &expr, // Check if an expression references any counter binding. bool ReferencesCounterBinding(Expression *expr, const unordered_map &counter_bindings) { - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { - return counter_bindings.count(HashBinding(expr->Cast().binding)) > 0; + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { + return counter_bindings.count(HashBinding(expr->Cast().BindingMutable())) > 0; } bool found = false; ExpressionIterator::EnumerateChildren(*expr, [&](Expression &child) { @@ -82,30 +85,33 @@ bool ReferencesCounterBinding(Expression *expr, const unordered_map, and the aggregate rebound to the _list variant. static bool TryRebindToListVariant(LogicalAggregate &agg, idx_t i, ClientContext &context) { auto &agg_expr = agg.expressions[i]; - if (agg_expr->type != ExpressionType::BOUND_AGGREGATE) { + if (agg_expr->GetExpressionType() != ExpressionType::BOUND_AGGREGATE) { return false; } auto &bound_agg = agg_expr->Cast(); - if (bound_agg.children.empty()) { + if (bound_agg.GetChildrenMutable().empty()) { return false; } // PAC/PAC_counters aggregates have (hash, value); standard aggregates have (value). // For _list variants we only need the value child. - bool is_pac_style = (IsPacAggregate(bound_agg.function.name) || IsPacCountersAggregate(bound_agg.function.name)) && - bound_agg.children.size() > 1; + bool is_pac_style = (IsPacAggregate(bound_agg.FunctionMutable().GetName().GetIdentifierName()) || + IsPacCountersAggregate(bound_agg.FunctionMutable().GetName().GetIdentifierName())) && + bound_agg.GetChildrenMutable().size() > 1; idx_t value_child_idx = is_pac_style ? 1 : 0; - string base_name = GetBasePacAggregateName(bound_agg.function.name); // strips _counters if present + string base_name = GetBasePacAggregateName( + bound_agg.FunctionMutable().GetName().GetIdentifierName()); // strips _counters if present string list_name = GetListAggregateVariant(base_name, GetPrivacyMode(context)); if (list_name.empty()) { // Generic aggregate (e.g., first()) over counter input — rebind with same name - if (bound_agg.children[value_child_idx]->return_type == LogicalType::LIST(PacFloatLogicalType())) { - list_name = bound_agg.function.name; + if (bound_agg.GetChildrenMutable()[value_child_idx]->GetReturnType() == + LogicalType::LIST(PacFloatLogicalType())) { + list_name = bound_agg.FunctionMutable().GetName().GetIdentifierName(); } else { return false; } } vector> children; - auto child_copy = bound_agg.children[value_child_idx]->Copy(); + auto child_copy = bound_agg.GetChildrenMutable()[value_child_idx]->Copy(); children.push_back(std::move(child_copy)); auto new_aggr = RebindAggregate(context, list_name, std::move(children), bound_agg.IsDistinct()); if (!new_aggr) { @@ -130,19 +136,23 @@ static void ReplaceAggregatesOverCounters(LogicalOperator *op, ClientContext &co auto &agg = op->Cast(); for (idx_t i = 0; i < agg.expressions.size(); i++) { auto &agg_expr = agg.expressions[i]; - if (agg_expr->type != ExpressionType::BOUND_AGGREGATE) { + if (agg_expr->GetExpressionType() != ExpressionType::BOUND_AGGREGATE) { continue; } auto &bound_agg = agg_expr->Cast(); - if (bound_agg.children.empty()) { + if (bound_agg.GetChildrenMutable().empty()) { continue; } - idx_t value_child_idx = IsPacAggregate(bound_agg.function.name) && bound_agg.children.size() > 1 ? 1 : 0; - if (!ReferencesCounterBinding(bound_agg.children[value_child_idx].get(), counter_bindings)) { + idx_t value_child_idx = IsPacAggregate(bound_agg.FunctionMutable().GetName().GetIdentifierName()) && + bound_agg.GetChildrenMutable().size() > 1 + ? 1 + : 0; + if (!ReferencesCounterBinding(bound_agg.GetChildrenMutable()[value_child_idx].get(), counter_bindings)) { continue; } if (TryRebindToListVariant(agg, i, context)) { - AddCounterBinding(counter_bindings, ColumnBinding(agg.aggregate_index, i)); + AddCounterBinding(counter_bindings, + ColumnBinding(TableIndex(agg.aggregate_index.index), ProjectionIndex(i))); } } } @@ -161,18 +171,18 @@ static void PrepareFilterPatternSlot(LogicalProjection &proj, idx_t i, // col_ref types are already fixed by the universal type fix above. // Update projection output type if (i < proj.types.size()) { - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { proj.types[i] = list_type; - } else if (expr->type == ExpressionType::BOUND_FUNCTION) { + } else if (expr->GetExpressionType() == ExpressionType::BOUND_FUNCTION) { proj.types[i] = expr->return_type; - } else if (expr->type == ExpressionType::CASE_EXPR && + } else if (expr->GetExpressionType() == ExpressionType::CASE_EXPR && IsScalarSubqueryWrapper(expr->Cast())) { // first(value) now returns a counter list, but DuckDB's scalar-subquery // cardinality-check CASE still has its original scalar result type. auto &wrapper = expr->Cast(); - if (wrapper.else_expr->return_type == list_type) { - wrapper.case_checks[0].then_expr = - BoundCastExpression::AddDefaultCastToType(std::move(wrapper.case_checks[0].then_expr), list_type); + if (wrapper.Else().return_type == list_type) { + wrapper.CaseChecksMutable()[0].then_expr = BoundCastExpression::AddDefaultCastToType( + std::move(wrapper.CaseChecksMutable()[0].then_expr), list_type); wrapper.return_type = list_type; proj.types[i] = list_type; PRIVACY_DEBUG_PRINT("Categorical rewrite: preserving counter lists through scalar-subquery wrapper"); @@ -205,8 +215,8 @@ RewriteExpressionWithCounters(OptimizerExtensionInput &input, const vectorGetExpressionClass() == ExpressionClass::BOUND_FUNCTION) { auto &func = cmp_result->Cast(); - if (func.function.name.rfind("priv_filter_", 0) == 0 || - func.function.name.rfind("priv_select_", 0) == 0) { + if (func.FunctionMutable().GetName().GetIdentifierName().rfind("priv_filter_", 0) == 0 || + func.FunctionMutable().GetName().GetIdentifierName().rfind("priv_select_", 0) == 0) { return cmp_result; // Full optimization to priv_{filter,select}_ } } @@ -236,83 +246,6 @@ RewriteExpressionWithCounters(OptimizerExtensionInput &input, const vector(); - if (col_ref.binding == old_binding) { - col_ref.binding = new_binding; - } - } else { - ExpressionIterator::EnumerateChildren( - expr, [&](Expression &child) { ReplaceBindingInExpression(child, old_binding, new_binding); }); - } -} - -// Expand projection references in a filter/join expression. -// For each col_ref to a projection slot with counter arithmetic: -// - create counter pass-through slots in the projection -// - substitute the col_ref with the arithmetic rebased to pass-through bindings -// This lets the filter fold arithmetic + comparison into one list_transform lambda. -static void ExpandProjectionRefsInExpression(unique_ptr &expr, LogicalOperator *plan_root, - unordered_map &counter_bindings) { - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { - auto &col_ref = expr->Cast(); - if (!counter_bindings.count(HashBinding(col_ref.binding))) { - return; // Not a counter binding — nothing to expand - } - auto *source_op = FindOperatorByTableIndex(plan_root, col_ref.binding.table_index); - if (!source_op || source_op->type != LogicalOperatorType::LOGICAL_PROJECTION) { - return; - } - auto &proj = source_op->Cast(); - idx_t slot = col_ref.binding.column_index; - if (slot >= proj.expressions.size()) { - return; - } - // Only expand slots with multi-binding counter arithmetic that needs decomposition. - // Skip: simple col_refs (already pass-throughs) and already-wrapped terminals. - auto &proj_expr = proj.expressions[slot]; - if (proj_expr->type == ExpressionType::BOUND_COLUMN_REF) { - return; - } - if (IsAlreadyWrappedInPacNoised(proj_expr.get())) { - return; - } - auto pac_bindings = FindCounterBindings(proj_expr.get(), counter_bindings); - if (pac_bindings.empty()) { - return; - } - auto list_type = LogicalType::LIST(PacFloatLogicalType()); - // Clone the projection expression for rebasing - auto expanded = proj_expr->Copy(); - // Create pass-through for first binding (reuse this slot) - proj.expressions[slot] = make_uniq("pac_var", list_type, pac_bindings[0].binding); - if (slot < proj.types.size()) { - proj.types[slot] = list_type; - } - ReplaceBindingInExpression(*expanded, pac_bindings[0].binding, ColumnBinding(proj.table_index, slot)); - // Extra pass-throughs for additional bindings - for (idx_t j = 1; j < pac_bindings.size(); j++) { - idx_t extra_slot = proj.expressions.size(); - proj.expressions.push_back( - make_uniq("pac_var", list_type, pac_bindings[j].binding)); - proj.types.push_back(list_type); - auto extra_cb = ColumnBinding(proj.table_index, extra_slot); - auto src_it = counter_bindings.find(HashBinding(pac_bindings[j].binding)); - AddCounterBinding(counter_bindings, extra_cb, - (src_it != counter_bindings.end()) ? src_it->second : pac_bindings[j].binding); - ReplaceBindingInExpression(*expanded, pac_bindings[j].binding, extra_cb); - } - expr = std::move(expanded); - return; - } - ExpressionIterator::EnumerateChildren(*expr, [&](unique_ptr &child) { - ExpandProjectionRefsInExpression(child, plan_root, counter_bindings); - }); -} - // Wrap counter column references with as_noised. // After converting as_noised_sum → as_sum (LIST output), // expressions may still reference the aggregate with the original type (e.g. DECIMAL). @@ -320,16 +253,16 @@ static void ExpandProjectionRefsInExpression(unique_ptr &expr, Logic static void WrapCounterRefsWithNoised(unique_ptr &expr, const unordered_map &counter_bindings, OptimizerExtensionInput &input) { - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr->Cast(); - if (counter_bindings.count(HashBinding(col_ref.binding))) { - auto original_type = col_ref.return_type; + if (counter_bindings.count(HashBinding(col_ref.BindingMutable()))) { + auto original_type = col_ref.GetReturnType(); auto list_type = LogicalType::LIST(PacFloatLogicalType()); // If the universal type fix already changed the col_ref to LIST, // we can't cast back to it — just use as_noised's natural scalar return type. bool needs_cast = (original_type != PacFloatLogicalType() && original_type != list_type); // Ensure col_ref type is LIST for correct as_noised binding. - col_ref.return_type = list_type; + col_ref.SetReturnType(list_type); unique_ptr noised = input.optimizer.BindScalarFunction("as_noised", expr->Copy()); if (needs_cast) { noised = BoundCastExpression::AddDefaultCastToType(std::move(noised), original_type); @@ -359,8 +292,8 @@ static bool RewriteProjectionExpression(OptimizerExtensionInput &input, LogicalP // Simple column reference to a counter list. // After the universal col_ref type fix, the col_ref type is LIST. // proj.types[i] still holds the original scalar type (e.g. HUGEINT). - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { - auto original_type = (i < proj.types.size()) ? proj.types[i] : expr->return_type; + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { + auto original_type = (i < proj.types.size()) ? proj.types[i] : expr->GetReturnType(); if (is_terminal) { unique_ptr result = input.optimizer.BindScalarFunction("as_noised", expr->Copy()); if (original_type != PacFloatLogicalType()) { @@ -374,21 +307,33 @@ static bool RewriteProjectionExpression(OptimizerExtensionInput &input, LogicalP } return true; // non-terminal pass-through — still a counter binding } - if (!IsNumericalType(expr->return_type)) { + if (!IsNumericalType(expr->GetReturnType())) { WrapCounterRefsWithNoised(expr, counter_bindings, input); return false; } // Arithmetic over PAC aggregates: build list_transform + as_noised terminal. Expression *expr_to_clone = expr.get(); - if (expr->type == ExpressionType::OPERATOR_CAST && - expr->Cast().return_type != PacFloatLogicalType()) { - expr_to_clone = expr->Cast().child.get(); + if (expr->GetExpressionType() == ExpressionType::OPERATOR_CAST && + expr->Cast().GetReturnType() != PacFloatLogicalType()) { + expr_to_clone = expr->Cast().GetChildrenMutable()[0].get(); + } + if (!is_terminal) { + auto result = + BuildCategoricalLambdas(input, pac_bindings, expr_to_clone, counter_bindings, PacFloatLogicalType()); + if (result) { + PRIVACY_DEBUG_PRINT("Categorical rewrite: preserving per-world projection #" + + std::to_string(proj.table_index.index) + "." + std::to_string(i)); + proj.expressions[i] = std::move(result); + proj.types[i] = proj.expressions[i]->GetReturnType(); + return true; + } + return false; } auto result = RewriteExpressionWithCounters(input, pac_bindings, expr_to_clone, counter_bindings, - PacWrapKind::PAC_NOISED, expr->return_type); + PacWrapKind::PAC_NOISED, expr->GetReturnType()); if (result) { proj.expressions[i] = std::move(result); - proj.types[i] = expr->return_type; + proj.types[i] = expr->GetReturnType(); } return false; } @@ -404,7 +349,7 @@ static ColumnBinding InsertPacSelectProjection(OptimizerExtensionInput &input, u auto old_bindings = filter_child_slot->GetColumnBindings(); idx_t num_cols = old_bindings.size(); - idx_t proj_table_index = input.optimizer.binder.GenerateTableIndex(); + idx_t proj_table_index = input.optimizer.binder.GenerateTableIndex().index; vector> proj_expressions; // Find which position the hash binding occupies @@ -430,7 +375,7 @@ static ColumnBinding InsertPacSelectProjection(OptimizerExtensionInput &input, u proj_expressions.push_back(std::move(pac_select_expr)); } - auto projection = make_uniq(proj_table_index, std::move(proj_expressions)); + auto projection = make_uniq(TableIndex(proj_table_index), std::move(proj_expressions)); projection->children.push_back(std::move(filter_child_slot)); projection->ResolveOperatorTypes(); @@ -440,12 +385,37 @@ static ColumnBinding InsertPacSelectProjection(OptimizerExtensionInput &input, u // Remap old bindings → new projection bindings ColumnBindingReplacer replacer; for (idx_t i = 0; i < num_cols; i++) { - replacer.replacement_bindings.emplace_back(old_bindings[i], ColumnBinding(proj_table_index, i)); + replacer.replacement_bindings.emplace_back(old_bindings[i], + ColumnBinding(TableIndex(proj_table_index), ProjectionIndex(i))); } replacer.stop_operator = proj_ptr; replacer.VisitOperator(*plan); - return ColumnBinding(proj_table_index, hash_position); + return ColumnBinding(TableIndex(proj_table_index), ProjectionIndex(hash_position)); +} + +// CTE readers have independent bindings and cache their input types. Preserve +// counter lineage across the materialization boundary before rewriting consumers. +static void SyncCTECounterBindings(LogicalOperator &op, LogicalMaterializedCTE &cte, + unordered_map &counter_bindings) { + if (op.type == LogicalOperatorType::LOGICAL_CTE_REF) { + auto &ref = op.Cast(); + if (ref.cte_index == cte.table_index) { + auto bindings = cte.children[0]->GetColumnBindings(); + for (idx_t i = 0; i < bindings.size() && i < ref.chunk_types.size(); i++) { + auto origin = counter_bindings.find(HashBinding(bindings[i])); + if (origin != counter_bindings.end()) { + ref.chunk_types[i] = cte.children[0]->types[i]; + AddCounterBinding(counter_bindings, ColumnBinding(ref.table_index, ProjectionIndex(i)), + origin->second); + } + } + ref.ResolveOperatorTypes(); + } + } + for (auto &child : op.children) { + SyncCTECounterBindings(*child, cte, counter_bindings); + } } // Single bottom-up rewrite pass. @@ -464,8 +434,15 @@ static void RewriteBottomUp(unique_ptr &op_ptr, OptimizerExtens for (idx_t ci = 0; ci < op->children.size(); ci++) { // Recurse into children first (bottom-up) // Child 0 of MATERIALIZED_CTE is the CTE definition — its output types must remain // stable (numeric, not LIST) because CTE_SCAN consumers may re-aggregate the results. - bool child_in_cte = - inside_cte_definition || (op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE && ci == 0); + bool is_cte = op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE; + bool generated_correlation_cte = + is_cte && + StringUtil::StartsWith(op->Cast().ctename.GetIdentifierName(), "__duckdb_delim_"); + bool child_in_cte = inside_cte_definition || (is_cte && ci == 0 && !generated_correlation_cte); + if (is_cte && ci == 1) { + op->children[0]->ResolveOperatorTypes(); + SyncCTECounterBindings(*op->children[1], op->Cast(), counter_bindings); + } RewriteBottomUp(op->children[ci], input, plan, pattern_lookup, patterns, replaced_mark_bindings, counter_bindings, child_in_cte); } @@ -477,9 +454,9 @@ static void RewriteBottomUp(unique_ptr &op_ptr, OptimizerExtens auto &filter = op->Cast(); bool has_dangling_mark = false; for (auto &fexpr : filter.expressions) { - if (fexpr->type == ExpressionType::BOUND_COLUMN_REF) { + if (fexpr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &ref = fexpr->Cast(); - if (replaced_mark_bindings.count(HashBinding(ref.binding))) { + if (replaced_mark_bindings.count(HashBinding(ref.BindingMutable()))) { has_dangling_mark = true; break; } @@ -517,10 +494,10 @@ static void RewriteBottomUp(unique_ptr &op_ptr, OptimizerExtens auto list_type = LogicalType::LIST(PacFloatLogicalType()); LogicalOperatorVisitor::EnumerateExpressions(*op, [&](unique_ptr *expr_ptr) { std::function &)> Fix = [&](unique_ptr &e) { - if (e->type == ExpressionType::BOUND_COLUMN_REF) { + if (e->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e->Cast(); - if (counter_bindings.count(HashBinding(col_ref.binding))) { - col_ref.return_type = list_type; + if (counter_bindings.count(HashBinding(col_ref.BindingMutable()))) { + col_ref.SetReturnType(list_type); } } ExpressionIterator::EnumerateChildren(*e, Fix); @@ -541,15 +518,15 @@ static void RewriteBottomUp(unique_ptr &op_ptr, OptimizerExtens // expressions that are sources of categorical patterns. for (idx_t i = 0; i < agg.expressions.size(); i++) { auto &agg_expr = agg.expressions[i]; - if (agg_expr->type != ExpressionType::BOUND_AGGREGATE) { + if (agg_expr->GetExpressionType() != ExpressionType::BOUND_AGGREGATE) { continue; } auto &bound_agg = agg_expr->Cast(); - if (!IsPacAggregate(bound_agg.function.name)) { + if (!IsPacAggregate(bound_agg.FunctionMutable().GetName().GetIdentifierName())) { continue; } // Check if this specific aggregate expression is a source of any pattern - ColumnBinding this_binding(agg.aggregate_index, i); + ColumnBinding this_binding {TableIndex(agg.aggregate_index.index), ProjectionIndex(i)}; bool convert = false; for (auto &p : patterns) { for (auto &bi : p.pac_bindings) { @@ -565,20 +542,21 @@ static void RewriteBottomUp(unique_ptr &op_ptr, OptimizerExtens if (!convert) { continue; } - string counters_name = GetCountersVariant(bound_agg.function.name); + string counters_name = GetCountersVariant(bound_agg.FunctionMutable().GetName().GetIdentifierName()); vector> children; - for (auto &child_expr : bound_agg.children) { + for (auto &child_expr : bound_agg.GetChildrenMutable()) { children.push_back(child_expr->Copy()); } auto new_aggr = RebindAggregate(input.context, counters_name, std::move(children), bound_agg.IsDistinct()); if (!new_aggr) { // Fallback: just rename the function in place - bound_agg.function.name = counters_name; - bound_agg.function.return_type = LogicalType::LIST(PacFloatLogicalType()); - agg_expr->return_type = LogicalType::LIST(PacFloatLogicalType()); + bound_agg.FunctionMutable().SetName(Identifier(counters_name)); + bound_agg.FunctionMutable().SetReturnType(LogicalType::LIST(PacFloatLogicalType())); + agg_expr->SetReturnType(LogicalType::LIST(PacFloatLogicalType())); } else { agg.expressions[i] = std::move(new_aggr); } - AddCounterBinding(counter_bindings, ColumnBinding(agg.aggregate_index, i)); + AddCounterBinding(counter_bindings, + ColumnBinding(TableIndex(agg.aggregate_index.index), ProjectionIndex(i))); idx_t types_index = agg.groups.size() + i; if (types_index < agg.types.size()) { agg.types[types_index] = LogicalType::LIST(PacFloatLogicalType()); @@ -587,55 +565,24 @@ static void RewriteBottomUp(unique_ptr &op_ptr, OptimizerExtens } else if (op->type == LogicalOperatorType::LOGICAL_PROJECTION && !inside_cte_definition) { // === PROJECTION: rewrite PAC expressions === auto &proj = op->Cast(); - // Check if this projection's output is consumed by a filter/join pattern. - // If so, skip terminal wrapping — the parent filter/join handles the rewrite. - bool is_filter_pattern = false; - for (auto &p : patterns) { - if (p.parent_op && pattern_lookup.count(p.parent_op)) { - for (auto &bi : p.pac_bindings) { - if (bi.binding.table_index == proj.table_index) { - is_filter_pattern = true; - break; - } - } - } - if (is_filter_pattern) { + // Intermediate projections compute valid per-world values; only the + // user-visible output projection finalizes them into scalar answers. + auto *output = plan_root; + while (output && !output->children.empty()) { + if (output->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE) { + output = output->children[1].get(); + } else if (output->type == LogicalOperatorType::LOGICAL_ORDER_BY || + output->type == LogicalOperatorType::LOGICAL_TOP_N || + output->type == LogicalOperatorType::LOGICAL_LIMIT) { + output = output->children[0].get(); + } else { break; } } - if (is_filter_pattern) { - // Filter-pattern projection: minimal processing only (type updates, no terminal wrapping). - // The parent filter/join will expand these expressions and generate list_transform + priv_filter. - for (idx_t i = 0; i < proj.expressions.size(); i++) { - PrepareFilterPatternSlot(proj, i, counter_bindings); - // All filter-pattern slots with counter content remain counter bindings - auto slot_bindings = FindCounterBindings(proj.expressions[i].get(), counter_bindings); - if (!slot_bindings.empty()) { - auto proj_cb = ColumnBinding(proj.table_index, i); - AddCounterBinding(counter_bindings, proj_cb, - TraceCounterOrigin(proj.expressions[i], counter_bindings)); - } - } - } else { - // Normal projection: full PAC rewriting with as_noised terminal. - bool is_terminal = (op == plan_root); - if (!is_terminal) { - auto *root = plan_root; - while (root && - (root->type == LogicalOperatorType::LOGICAL_ORDER_BY || - root->type == LogicalOperatorType::LOGICAL_TOP_N || - root->type == LogicalOperatorType::LOGICAL_LIMIT) && - !root->children.empty()) { - root = root->children[0].get(); - } - is_terminal = (root == op); - } - for (idx_t i = 0; i < proj.expressions.size(); i++) { - if (RewriteProjectionExpression(input, proj, i, plan_root, is_terminal, counter_bindings)) { - auto proj_cb = ColumnBinding(proj.table_index, i); - AddCounterBinding(counter_bindings, proj_cb, - TraceCounterOrigin(proj.expressions[i], counter_bindings)); - } + for (idx_t i = 0; i < proj.expressions.size(); i++) { + if (RewriteProjectionExpression(input, proj, i, plan_root, output == op, counter_bindings)) { + AddCounterBinding(counter_bindings, ColumnBinding(proj.table_index, ProjectionIndex(i)), + TraceCounterOrigin(proj.expressions[i], counter_bindings)); } } } else if (op->type == LogicalOperatorType::LOGICAL_FILTER && @@ -648,9 +595,7 @@ static void RewriteBottomUp(unique_ptr &op_ptr, OptimizerExtens continue; } auto &filter_expr = filter.expressions[expr_idx]; - // Expand projection arithmetic into the filter expression so arithmetic + comparison - // are folded into one list_transform lambda. Handles both single and multi-binding. - ExpandProjectionRefsInExpression(filter_expr, plan_root, counter_bindings); + // Projection arithmetic has already been lifted to per-world lists. auto pac_bindings = FindCounterBindings(filter_expr.get(), counter_bindings); if (pac_bindings.empty()) { continue; @@ -677,7 +622,7 @@ static void RewriteBottomUp(unique_ptr &op_ptr, OptimizerExtens auto new_hash = InsertPacSelectProjection(input, plan, op->children[0], pattern_hash, std::move(result)); // Replace filter condition with priv_hash <> 0 for majority-vote decision - filter.expressions[expr_idx] = make_uniq( + filter.expressions[expr_idx] = BoundComparisonExpression::Create( ExpressionType::COMPARE_NOTEQUAL, make_uniq("pac_pu", LogicalType::UBIGINT, new_hash), make_uniq(Value::UBIGINT(0))); @@ -723,75 +668,57 @@ static void RewriteBottomUp(unique_ptr &op_ptr, OptimizerExtens } } auto &join = op->Cast(); - for (auto expr_idx : it->second) { - if (expr_idx >= join.conditions.size()) { - continue; - } - auto &cond = join.conditions[expr_idx]; - // Expand projection arithmetic into join conditions - ExpandProjectionRefsInExpression(cond.left, plan_root, counter_bindings); - ExpandProjectionRefsInExpression(cond.right, plan_root, counter_bindings); - auto left_bindings = FindCounterBindings(cond.left.get(), counter_bindings); - auto right_bindings = FindCounterBindings(cond.right.get(), counter_bindings); - bool left_is_list = !left_bindings.empty(); - bool right_is_list = !right_bindings.empty(); - if (!left_is_list && !right_is_list) { - continue; - } - // Collect PAC bindings (deduplicated) - vector all_bindings; - unordered_set seen_hashes; - for (auto &b : left_bindings) { - uint64_t h = HashBinding(b.binding); - if (seen_hashes.insert(h).second) { - all_bindings.push_back(b); - } - } - for (auto &b : right_bindings) { - uint64_t h = HashBinding(b.binding); - if (seen_hashes.insert(h).second) { - all_bindings.push_back(b); - } - } - for (idx_t j = 0; j < all_bindings.size(); j++) { - all_bindings[j].index = j; + unique_ptr comparison; + vector> ordinary_conditions; + for (auto &condition : join.conditions) { + auto expression = BoundComparisonExpression::Create( + condition.GetComparisonType(), condition.LeftReference()->Copy(), condition.RightReference()->Copy()); + if (!ReferencesCounterBinding(expression.get(), counter_bindings)) { + ordinary_conditions.push_back(std::move(expression)); + } else if (comparison) { + comparison = make_uniq(ExpressionType::CONJUNCTION_AND, + std::move(comparison), std::move(expression)); + } else { + comparison = std::move(expression); } - auto comparison = - make_uniq(cond.comparison, cond.left->Copy(), cond.right->Copy()); - auto pac_expr = RewriteExpressionWithCounters(input, all_bindings, comparison.get(), counter_bindings, - join_wrap_kind, PacFloatLogicalType(), join_pattern_hash); - if (pac_expr) { - auto cross_product = - LogicalCrossProduct::Create(std::move(join.children[0]), std::move(join.children[1])); - auto filter_op = make_uniq(); - - if (join_wrap_kind == PacWrapKind::PAC_SELECT) { - // Insert projection below filter replacing hash with priv_select result - auto new_hash = - InsertPacSelectProjection(input, plan, cross_product, join_pattern_hash, std::move(pac_expr)); - filter_op->expressions.push_back(make_uniq( - ExpressionType::COMPARE_NOTEQUAL, - make_uniq("pac_pu", LogicalType::UBIGINT, new_hash), - make_uniq(Value::UBIGINT(0)))); - } else { - filter_op->expressions.push_back(std::move(pac_expr)); - } - filter_op->children.push_back(std::move(cross_product)); - - for (auto &p : patterns) { - if (p.parent_op == op) { - p.parent_op = nullptr; - } - } - // Record the mark column binding so the parent FILTER - // (which references it) can be cleaned up. - if (join.join_type == JoinType::MARK) { - replaced_mark_bindings.insert(HashBinding(ColumnBinding(join.mark_index, 0))); - } - op_ptr = std::move(filter_op); - break; + } + if (!comparison) { + return; + } + // Evaluate all world-dependent conditions together, then make one + // categorical decision. Keep ordinary join predicates outside the lambda + // so DuckDB can still recover its usual hash joins during optimization. + auto bindings = FindCounterBindings(comparison.get(), counter_bindings); + auto pac_expr = RewriteExpressionWithCounters(input, bindings, comparison.get(), counter_bindings, + join_wrap_kind, PacFloatLogicalType(), join_pattern_hash); + if (!pac_expr) { + return; + } + auto cross_product = LogicalCrossProduct::Create(std::move(join.children[0]), std::move(join.children[1])); + auto filter = make_uniq(); + if (join_wrap_kind == PacWrapKind::PAC_SELECT) { + auto new_hash = + InsertPacSelectProjection(input, plan, cross_product, join_pattern_hash, std::move(pac_expr)); + filter->expressions.push_back(BoundComparisonExpression::Create( + ExpressionType::COMPARE_NOTEQUAL, + make_uniq("pac_pu", LogicalType::UBIGINT, new_hash), + make_uniq(Value::UBIGINT(0)))); + } else { + filter->expressions.push_back(std::move(pac_expr)); + } + for (auto &condition : ordinary_conditions) { + filter->expressions.push_back(std::move(condition)); + } + filter->children.push_back(std::move(cross_product)); + for (auto &pattern : patterns) { + if (pattern.parent_op == op) { + pattern.parent_op = nullptr; } } + if (join.join_type == JoinType::MARK) { + replaced_mark_bindings.insert(HashBinding(ColumnBinding(join.mark_index, ProjectionIndex(0)))); + } + op_ptr = std::move(filter); } } @@ -808,26 +735,29 @@ static void ConvertPacAggregatesToListVariants(LogicalOperator &plan, ClientCont } auto &agg = plan.Cast(); for (idx_t i = 0; i < agg.expressions.size(); i++) { - if (agg.expressions[i]->type != ExpressionType::BOUND_AGGREGATE) { + if (agg.expressions[i]->GetExpressionType() != ExpressionType::BOUND_AGGREGATE) { continue; } - if (counter_bindings.count(HashBinding(ColumnBinding(agg.aggregate_index, i)))) { + if (counter_bindings.count( + HashBinding(ColumnBinding(TableIndex(agg.aggregate_index.index), ProjectionIndex(i))))) { continue; } auto &bound_agg = agg.expressions[i]->Cast(); - if (bound_agg.children.empty() || IsPacListAggregate(bound_agg.function.name)) { + if (bound_agg.GetChildrenMutable().empty() || + IsPacListAggregate(bound_agg.FunctionMutable().GetName().GetIdentifierName())) { continue; } - idx_t value_idx = - (IsPacAggregate(bound_agg.function.name) || IsPacCountersAggregate(bound_agg.function.name)) && - bound_agg.children.size() > 1 - ? 1 - : 0; - if (!ReferencesCounterBinding(bound_agg.children[value_idx].get(), counter_bindings)) { + idx_t value_idx = (IsPacAggregate(bound_agg.FunctionMutable().GetName().GetIdentifierName()) || + IsPacCountersAggregate(bound_agg.FunctionMutable().GetName().GetIdentifierName())) && + bound_agg.GetChildrenMutable().size() > 1 + ? 1 + : 0; + if (!ReferencesCounterBinding(bound_agg.GetChildrenMutable()[value_idx].get(), counter_bindings)) { continue; } if (TryRebindToListVariant(agg, i, context)) { - AddCounterBinding(counter_bindings, ColumnBinding(agg.aggregate_index, i)); + AddCounterBinding(counter_bindings, + ColumnBinding(TableIndex(agg.aggregate_index.index), ProjectionIndex(i))); } } } diff --git a/src/compiler/privacy_bitslice_add_fkjoins.cpp b/src/compiler/privacy_bitslice_add_fkjoins.cpp index 4ee68352..6b2dfe35 100644 --- a/src/compiler/privacy_bitslice_add_fkjoins.cpp +++ b/src/compiler/privacy_bitslice_add_fkjoins.cpp @@ -179,7 +179,7 @@ CreateGetsAndChainJoins(const PrivacyCompatibilityResult &check, OptimizerExtens throw InvalidInputException("Privacy compiler: missing table metadata for table: " + table); } auto required_cols = GetRequiredColumnsForTable(check, table, fk_path, privacy_units); - idx_t local_idx = binder.GenerateTableIndex(); + idx_t local_idx = binder.GenerateTableIndex().index; auto get = CreateLogicalGet(input.context, plan, table, local_idx, required_cols); local_get_map[table] = std::move(get); } @@ -209,15 +209,15 @@ static bool FindAccessibleAlternativeAndJoin(const PrivacyCompatibilityResult &c for (auto *table_node : table_nodes) { auto &table_get = table_node->get()->Cast(); - if (table_get.table_index == skip_table_index) { + if (table_get.table_index.index == skip_table_index) { continue; } - if (!AreTableColumnsAccessible(plan.get(), table_get.table_index)) { + if (!AreTableColumnsAccessible(plan.get(), table_get.table_index.index)) { continue; } #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("FindAccessibleAlternativeAndJoin: Found accessible table " + candidate + " #" + - std::to_string(table_get.table_index) + " - joining " + + std::to_string(table_get.table_index.index) + " - joining " + std::to_string(tables_to_join.size()) + " tables"); #endif unique_ptr current_node = (*table_node)->Copy(input.context); @@ -272,7 +272,7 @@ void AddMissingFKJoins(const PrivacyCompatibilityResult &check, OptimizerExtensi throw InvalidInputException("Privacy compiler: missing table metadata for missing GET: " + table); } auto required_cols = GetRequiredColumnsForTable(check, table, fk_path, privacy_units); - idx_t idx = binder.GenerateTableIndex(); + idx_t idx = binder.GenerateTableIndex().index; auto get = CreateLogicalGet(input.context, plan, table, idx, required_cols); get_map[table] = std::move(get); ordered_table_names.push_back(table); @@ -351,7 +351,7 @@ void AddMissingFKJoins(const PrivacyCompatibilityResult &check, OptimizerExtensi // Iterate over each instance of the connecting table and add joins for (auto *target_ref : all_connecting_nodes) { auto &target_op = (*target_ref)->Cast(); - idx_t connecting_table_idx = target_op.table_index; + idx_t connecting_table_idx = target_op.table_index.index; bool is_in_subquery = IsInDelimJoinSubqueryBranch(&plan, target_ref->get()); vector tables_to_join_for_instance = ordered_table_names; @@ -428,7 +428,7 @@ void AddMissingFKJoins(const PrivacyCompatibilityResult &check, OptimizerExtensi } for (auto *fk_node : fk_table_nodes) { auto &fk_table_get = fk_node->get()->Cast(); - idx_t fk_table_idx = fk_table_get.table_index; + idx_t fk_table_idx = fk_table_get.table_index.index; if (fk_table_idx == connecting_table_idx) { if (AreTableColumnsAccessible(plan.get(), fk_table_idx)) { @@ -468,7 +468,7 @@ void AddMissingFKJoins(const PrivacyCompatibilityResult &check, OptimizerExtensi string connecting_table_name; auto table_ptr = target_op.GetTable(); if (table_ptr) { - connecting_table_name = table_ptr->name; + connecting_table_name = table_ptr->name.GetIdentifierName(); } // Case 1: connecting table IS the FK table but blocked (SEMI/ANTI join) @@ -579,10 +579,10 @@ void AddMissingFKJoins(const PrivacyCompatibilityResult &check, OptimizerExtensi for (auto *table_node : table_nodes) { auto &table_get = table_node->get()->Cast(); - if (table_get.table_index == connecting_table_idx) { + if (table_get.table_index.index == connecting_table_idx) { continue; } - if (!AreTableColumnsAccessible(plan.get(), table_get.table_index)) { + if (!AreTableColumnsAccessible(plan.get(), table_get.table_index.index)) { continue; } @@ -609,7 +609,7 @@ void AddMissingFKJoins(const PrivacyCompatibilityResult &check, OptimizerExtensi } #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("AddMissingFKJoins: Found accessible table " + candidate + " #" + - std::to_string(table_get.table_index) + " - adding full join chain:"); + std::to_string(table_get.table_index.index) + " - adding full join chain:"); for (auto &t : full_tables_to_add) { PRIVACY_DEBUG_PRINT(" - " + t); } diff --git a/src/compiler/privacy_compiler.cpp b/src/compiler/privacy_compiler.cpp index 7b827eb0..6e1988c6 100644 --- a/src/compiler/privacy_compiler.cpp +++ b/src/compiler/privacy_compiler.cpp @@ -91,7 +91,7 @@ static bool IsDirectlyReachable(LogicalOperator *from, idx_t table_index, bool i } if (from->type == LogicalOperatorType::LOGICAL_GET) { auto &get = from->Cast(); - if (get.table_index == table_index) { + if (get.table_index.index == table_index) { return true; } } @@ -115,9 +115,9 @@ static unique_ptr HandlePassthroughDeepHash(OptimizerExtensionInput std::unordered_map &hash_cache) { auto inner_hash_binding = GetOrInsertHashProjection(input, plan, inner_get, key_cols, false, hash_cache); - auto propagated_to_inner = PropagateSingleBinding(*plan, inner_hash_binding.table_index, inner_hash_binding, + auto propagated_to_inner = PropagateSingleBinding(*plan, inner_hash_binding.table_index.index, inner_hash_binding, LogicalType::UBIGINT, inner_agg); - if (propagated_to_inner.table_index == DConstants::INVALID_INDEX) { + if (propagated_to_inner.table_index.index == DConstants::INVALID_INDEX) { return nullptr; } // Add hash as GROUP BY column in the inner aggregate @@ -125,16 +125,17 @@ static unique_ptr HandlePassthroughDeepHash(OptimizerExtensionInput inner_agg->groups.push_back(make_uniq(LogicalType::UBIGINT, propagated_to_inner)); // Also add to grouping_sets so the physical HASH_GROUP_BY includes this column for (auto &gs : inner_agg->grouping_sets) { - gs.insert(new_group_idx); + gs.insert(ProjectionIndex(new_group_idx)); } inner_agg->ResolveOperatorTypes(); - ColumnBinding inner_hash_output(inner_agg->group_index, inner_agg->groups.size() - 1); + ColumnBinding inner_hash_output {TableIndex(inner_agg->group_index.index), + ProjectionIndex(inner_agg->groups.size() - 1)}; // Propagate from inner aggregate output to outer aggregate - auto propagated_to_outer = PropagateSingleBinding(*plan, inner_hash_output.table_index, inner_hash_output, + auto propagated_to_outer = PropagateSingleBinding(*plan, inner_hash_output.table_index.index, inner_hash_output, LogicalType::UBIGINT, target_agg); - if (propagated_to_outer.table_index == DConstants::INVALID_INDEX) { + if (propagated_to_outer.table_index.index == DConstants::INVALID_INDEX) { return nullptr; } return make_uniq(LogicalType::UBIGINT, propagated_to_outer); @@ -230,7 +231,7 @@ static void CollectCTERefs(LogicalOperator *op, idx_t target_cte_index, vectortype == LogicalOperatorType::LOGICAL_CTE_REF) { auto &ref = op->Cast(); - if (ref.cte_index == target_cte_index) { + if (ref.cte_index.index == target_cte_index) { refs.push_back(&ref); } } @@ -278,7 +279,7 @@ static ColumnBinding PropagateColumnThroughSubtree(LogicalOperator *root, Logica auto &proj = op->Cast(); idx_t new_idx = proj.expressions.size(); proj.expressions.push_back(make_uniq(col_type, current)); - current = ColumnBinding(proj.table_index, new_idx); + current = ColumnBinding(TableIndex(proj.table_index.index), ProjectionIndex(new_idx)); proj.ResolveOperatorTypes(); } else if (op != &get) { // For non-projection operators, check that the binding passes through @@ -292,7 +293,7 @@ static ColumnBinding PropagateColumnThroughSubtree(LogicalOperator *root, Logica } if (!found) { // Binding is not in this operator's output — propagation is impossible - return ColumnBinding(DConstants::INVALID_INDEX, 0); + return ColumnBinding(TableIndex(DConstants::INVALID_INDEX), ProjectionIndex(0)); } } } @@ -305,7 +306,7 @@ static ColumnBinding PropagateColumnThroughSubtree(LogicalOperator *root, Logica static bool ExpandCTEWithColumns(LogicalOperator *plan_root, const LogicalCTERef &ref, const string &table_name, const vector &columns) { // Find the MATERIALIZED_CTE that defines this CTE - auto *mat_cte = FindMaterializedCTE(plan_root, ref.cte_index); + auto *mat_cte = FindMaterializedCTE(plan_root, ref.cte_index.index); if (!mat_cte || mat_cte->children.empty()) { return false; } @@ -347,7 +348,7 @@ static bool ExpandCTEWithColumns(LogicalOperator *plan_root, const LogicalCTERef LogicalType col_type = LogicalType::INVALID; auto &catalog_columns = entry->GetColumns(); for (auto &col : catalog_columns.Logical()) { - if (col.Name() == col_name) { + if (col.Name().GetIdentifierName() == col_name) { col_oid = col.Oid(); col_type = col.Type(); break; @@ -365,22 +366,22 @@ static bool ExpandCTEWithColumns(LogicalOperator *plan_root, const LogicalCTERef // Add to projection_ids if the GET uses them if (!get->projection_ids.empty()) { - get->projection_ids.push_back(new_col_idx); + get->projection_ids.push_back(ProjectionIndex(new_col_idx)); } get->ResolveOperatorTypes(); - // The GET now outputs this column at binding (get->table_index, new_output_idx) + // The GET now outputs this column at binding (get->table_index.index, new_output_idx) // If projection_ids is non-empty, the output index is projection_ids.size()-1 // Otherwise it's column_ids.size()-1 idx_t get_output_idx = get->projection_ids.empty() ? (get->GetColumnIds().size() - 1) : (get->projection_ids.size() - 1); - ColumnBinding get_binding(get->table_index, get_output_idx); + ColumnBinding get_binding {TableIndex(get->table_index.index), ProjectionIndex(get_output_idx)}; // Propagate through intermediate operators (projections, filters) in the CTE definition. // If propagation fails (binding blocked by aggregate, semi-join, etc.), bail out. auto top_binding = PropagateColumnThroughSubtree(mat_cte->children[0].get(), *get, get_binding, col_type); - if (top_binding.table_index == DConstants::INVALID_INDEX) { + if (top_binding.table_index.index == DConstants::INVALID_INDEX) { return false; } @@ -389,15 +390,15 @@ static bool ExpandCTEWithColumns(LogicalOperator *plan_root, const LogicalCTERef // Update ALL CTE_REF nodes that reference this CTE vector all_refs; - CollectCTERefs(plan_root, ref.cte_index, all_refs); + CollectCTERefs(plan_root, ref.cte_index.index, all_refs); for (auto *cte_ref : all_refs) { - cte_ref->bound_columns.push_back(col_name); + cte_ref->bound_columns.emplace_back(col_name); cte_ref->chunk_types.push_back(col_type); cte_ref->ResolveOperatorTypes(); } PRIVACY_DEBUG_PRINT("ExpandCTEWithColumns: added column '" + col_name + "' (oid=" + std::to_string(col_oid) + - ") to CTE index " + std::to_string(ref.cte_index)); + ") to CTE index " + std::to_string(ref.cte_index.index)); } return true; @@ -418,7 +419,7 @@ static CTEHashMatch FindCTEHashSource(LogicalOperator *op, const string &pu_tabl if (!inside_delim_join && op->type == LogicalOperatorType::LOGICAL_CTE_REF) { auto &ref = op->Cast(); - auto cte_it = cte_map.find(ref.cte_index); + auto cte_it = cte_map.find(ref.cte_index.index); if (cte_it != cte_map.end()) { auto &cte_tables = cte_it->second; @@ -509,17 +510,17 @@ class PrivateUnionRewriter { AppendMembership(cte.children[0], mask_column); cte.column_count = value_columns + 1; vector refs; - CollectCTERefs(plan.get(), cte.table_index, refs); + CollectCTERefs(plan.get(), cte.table_index.index, refs); for (auto *ref : refs) { if (ref->chunk_types.size() != value_columns) { throw InvalidInputException("Private UNION CTE has unresolved ownership output columns"); } ref->chunk_types.push_back(LogicalType::UBIGINT); - ref->bound_columns.push_back("private_union_membership"); + ref->bound_columns.emplace_back("private_union_membership"); ref->ResolveOperatorTypes(); - memberships.emplace(ref, ColumnBinding(ref->table_index, mask_column)); + memberships.emplace(ref, ColumnBinding(ref->table_index, ProjectionIndex(mask_column))); } - PRIVACY_DEBUG_PRINT("[PRIVATE UNION] Exported membership through CTE " + cte.ctename); + PRIVACY_DEBUG_PRINT("[PRIVATE UNION] Exported membership through CTE " + cte.ctename.GetIdentifierName()); } Rewrite(cte.children[1]); cte.ResolveOperatorTypes(); @@ -570,8 +571,8 @@ class PrivateUnionRewriter { } CheckBranch(*agg.children[0]); auto mask = - PropagateSingleBinding(*plan, found[0].second.table_index, found[0].second, LogicalType::UBIGINT, &agg); - if (mask.table_index == DConstants::INVALID_INDEX) { + PropagateSingleBinding(*plan, found[0].second.table_index.index, found[0].second, LogicalType::UBIGINT, &agg); + if (mask.table_index.index == DConstants::INVALID_INDEX) { throw InvalidInputException("Private UNION could not propagate membership to the consuming aggregate"); } return mask; @@ -584,7 +585,7 @@ class PrivateUnionRewriter { const vector &pus; std::unordered_map memberships; - void ProjectMembership(unique_ptr &op, idx_t count, idx_t output_index, bool distinct) { + void ProjectMembership(unique_ptr &op, idx_t count, TableIndex output_index, bool distinct) { auto types = op->types; auto bindings = op->GetColumnBindings(); vector> outputs; @@ -600,10 +601,10 @@ class PrivateUnionRewriter { input.optimizer.binder.GenerateTableIndex(), std::move(aggregates)); for (idx_t i = 0; i < count; i++) { dedup->groups.push_back(make_uniq(types[i], bindings[i])); - outputs.push_back(make_uniq(types[i], ColumnBinding(dedup->group_index, i))); + outputs.push_back(make_uniq(types[i], ColumnBinding(dedup->group_index, ProjectionIndex(i)))); } outputs.push_back( - make_uniq(LogicalType::UBIGINT, ColumnBinding(dedup->aggregate_index, 0))); + make_uniq(LogicalType::UBIGINT, ColumnBinding(dedup->aggregate_index, ProjectionIndex(0)))); dedup->children.push_back(std::move(op)); op = std::move(dedup); } else { @@ -616,7 +617,7 @@ class PrivateUnionRewriter { projection->children.push_back(std::move(op)); op = std::move(projection); op->ResolveOperatorTypes(); - memberships.emplace(op.get(), ColumnBinding(output_index, count)); + memberships.emplace(op.get(), ColumnBinding(output_index, ProjectionIndex(count))); } void RewriteDistinct(unique_ptr &op) { @@ -636,7 +637,7 @@ class PrivateUnionRewriter { auto old_bindings = op->children[0]->GetColumnBindings(); for (idx_t i = 0; i < count; i++) { auto &expr = distinct.distinct_targets[i]; - if (i >= old_bindings.size() || expr->type != ExpressionType::BOUND_COLUMN_REF || + if (i >= old_bindings.size() || expr->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF || expr->Cast().binding != old_bindings[i]) { throw InvalidInputException("Private UNION DISTINCT requires the visible output columns"); } @@ -646,7 +647,7 @@ class PrivateUnionRewriter { // DuckDB's own DISTINCT modifier above UNION DISTINCT is already satisfied. if (annotation != memberships.end() && child->type == LogicalOperatorType::LOGICAL_PROJECTION && child->children[0]->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY && - annotation->second.column_index == count) { + annotation->second.column_index.index == count) { op = std::move(op->children[0]); return; } @@ -656,7 +657,7 @@ class PrivateUnionRewriter { ProjectMembership(op, count, output_index, true); ColumnBindingReplacer replacer; for (idx_t i = 0; i < count; i++) { - replacer.replacement_bindings.emplace_back(old_bindings[i], ColumnBinding(output_index, i)); + replacer.replacement_bindings.emplace_back(old_bindings[i], ColumnBinding(output_index, ProjectionIndex(i))); } replacer.stop_operator = op.get(); replacer.VisitOperator(*plan); @@ -794,18 +795,18 @@ class PrivateUnionRewriter { if (!table) { continue; } - if (!AreTableColumnsAccessible(&target, get->table_index)) { + if (!AreTableColumnsAccessible(&target, get->table_index.index)) { if (ContainsUnaggregatedEntityInput(*get, check)) { throw InvalidInputException("Private UNION branch has inaccessible privacy-unit ownership"); } continue; } - auto meta = check.table_metadata.find(table->name); + auto meta = check.table_metadata.find(table->name.GetIdentifierName()); if (meta == check.table_metadata.end()) { continue; } vector candidate; - if (table->name == pus[0]) { + if (table->name.GetIdentifierName() == pus[0]) { candidate = meta->second.pks; } else { for (auto &fk : meta->second.fks) { @@ -838,8 +839,8 @@ class PrivateUnionRewriter { return; } auto propagated = - PropagateSingleBindingToOperator(*plan, source.table_index, source, LogicalType::UBIGINT, &target); - if (propagated.table_index == DConstants::INVALID_INDEX) { + PropagateSingleBindingToOperator(*plan, source.table_index.index, source, LogicalType::UBIGINT, &target); + if (propagated.table_index.index == DConstants::INVALID_INDEX) { throw InvalidInputException("Private UNION could not propagate branch membership"); } target.expressions.push_back(make_uniq(LogicalType::UBIGINT, propagated)); @@ -909,7 +910,7 @@ vector BuildPUHashExpressionsForAggregates(OptimizerExten // Cache for hash projections: get.table_index → hash column binding std::unordered_map hash_cache; - // Cache for CTE hash projections: cte_ref.table_index → hash column binding + // Cache for CTE hash projections: cte_ref.table_index.index → hash column binding // Prevents inserting duplicate projections above the same CTE_SCAN when // multiple aggregates try the CTE path for the same CTE_SCAN node. std::unordered_map cte_hash_cache; @@ -937,7 +938,7 @@ vector BuildPUHashExpressionsForAggregates(OptimizerExten if (pu_in_subtree) { get_ptr = FindTableScanInSubtree(target_agg, pu_table_name); if (get_ptr) { - pu_columns_accessible = AreTableColumnsAccessible(target_agg, get_ptr->table_index); + pu_columns_accessible = AreTableColumnsAccessible(target_agg, get_ptr->table_index.index); } } @@ -961,7 +962,7 @@ vector BuildPUHashExpressionsForAggregates(OptimizerExten // Check if the PU table is directly reachable (not behind a nested aggregate). // PU-key passthrough: the table is behind an inner aggregate that groups // by PU key — insert hash deep and propagate through the inner's GROUP BY. - if (!IsDirectlyReachable(target_agg, get.table_index)) { + if (!IsDirectlyReachable(target_agg, get.table_index.index)) { ColumnBinding pk_binding; auto *inner_agg = FindInnerAggregateWithPUKeyGroup(target_agg, check, pu_table_names, pk_binding); if (inner_agg) { @@ -979,17 +980,17 @@ vector BuildPUHashExpressionsForAggregates(OptimizerExten // Direct path: insert hash projection above get and propagate auto hash_binding = GetOrInsertHashProjection(input, plan, get, pks, false, hash_cache); - auto propagated = PropagateSingleBinding(*plan, hash_binding.table_index, hash_binding, + auto propagated = PropagateSingleBinding(*plan, hash_binding.table_index.index, hash_binding, LogicalType::UBIGINT, target_agg); - if (propagated.table_index == DConstants::INVALID_INDEX) { + if (propagated.table_index.index == DConstants::INVALID_INDEX) { // Try DELIM_JOIN fallback before giving up - auto delim_result = AddBindingToDelimJoin(plan, hash_binding.table_index, hash_binding, + auto delim_result = AddBindingToDelimJoin(plan, hash_binding.table_index.index, hash_binding, LogicalType::UBIGINT, target_agg); if (delim_result.IsValid()) { propagated = delim_result.binding; } } - if (propagated.table_index == DConstants::INVALID_INDEX) { + if (propagated.table_index.index == DConstants::INVALID_INDEX) { continue; } auto hash_ref = make_uniq(LogicalType::UBIGINT, propagated); @@ -1042,7 +1043,7 @@ vector BuildPUHashExpressionsForAggregates(OptimizerExten } // PU-key passthrough: if the FK table is behind an inner aggregate, // insert hash deep and propagate through the inner's GROUP BY. - if (!IsDirectlyReachable(target_agg, fk_get_ptr->table_index)) { + if (!IsDirectlyReachable(target_agg, fk_get_ptr->table_index.index)) { ColumnBinding pk_binding; auto *inner_agg = FindInnerAggregateWithPUKeyGroup(target_agg, check, pu_table_names, pk_binding); @@ -1063,18 +1064,19 @@ vector BuildPUHashExpressionsForAggregates(OptimizerExten // Direct path: insert hash projection and propagate auto fk_hash_binding = GetOrInsertHashProjection(input, plan, *fk_get_ptr, fk_cols, false, hash_cache); - auto fk_propagated = PropagateSingleBinding(*plan, fk_hash_binding.table_index, fk_hash_binding, - LogicalType::UBIGINT, target_agg); + auto fk_propagated = PropagateSingleBinding(*plan, fk_hash_binding.table_index.index, + fk_hash_binding, LogicalType::UBIGINT, target_agg); // If propagation failed, try DELIM_JOIN fallback - if (fk_propagated.table_index == DConstants::INVALID_INDEX) { - auto delim_result = AddBindingToDelimJoin( - plan, fk_hash_binding.table_index, fk_hash_binding, LogicalType::UBIGINT, target_agg); + if (fk_propagated.table_index.index == DConstants::INVALID_INDEX) { + auto delim_result = + AddBindingToDelimJoin(plan, fk_hash_binding.table_index.index, fk_hash_binding, + LogicalType::UBIGINT, target_agg); if (delim_result.IsValid()) { fk_propagated = delim_result.binding; } } - if (fk_propagated.table_index == DConstants::INVALID_INDEX) { + if (fk_propagated.table_index.index == DConstants::INVALID_INDEX) { continue; } auto fk_hash_ref = make_uniq(LogicalType::UBIGINT, fk_propagated); @@ -1108,29 +1110,29 @@ vector BuildPUHashExpressionsForAggregates(OptimizerExten continue; } ColumnBinding hash_binding; - auto cte_cache_it = cte_hash_cache.find(match.cte_ref->table_index); + auto cte_cache_it = cte_hash_cache.find(match.cte_ref->table_index.index); if (cte_cache_it != cte_hash_cache.end()) { hash_binding = cte_cache_it->second; } else { hash_binding = InsertHashProjectionAboveCTERef(input, plan, *match.cte_ref, match.hash_columns); - if (hash_binding.table_index == DConstants::INVALID_INDEX) { + if (hash_binding.table_index.index == DConstants::INVALID_INDEX) { continue; } - cte_hash_cache[match.cte_ref->table_index] = hash_binding; + cte_hash_cache[match.cte_ref->table_index.index] = hash_binding; } - auto propagated = PropagateSingleBinding(*plan, hash_binding.table_index, hash_binding, + auto propagated = PropagateSingleBinding(*plan, hash_binding.table_index.index, hash_binding, LogicalType::UBIGINT, target_agg); // If propagation failed (e.g., blocked by DELIM_JOIN), try AddBindingToDelimJoin - if (propagated.table_index == DConstants::INVALID_INDEX) { - auto delim_result = AddBindingToDelimJoin(plan, hash_binding.table_index, hash_binding, + if (propagated.table_index.index == DConstants::INVALID_INDEX) { + auto delim_result = AddBindingToDelimJoin(plan, hash_binding.table_index.index, hash_binding, LogicalType::UBIGINT, target_agg); if (delim_result.IsValid()) { propagated = delim_result.binding; } } - if (propagated.table_index == DConstants::INVALID_INDEX) { + if (propagated.table_index.index == DConstants::INVALID_INDEX) { continue; } auto hash_ref = make_uniq(LogicalType::UBIGINT, propagated); @@ -1151,8 +1153,8 @@ vector BuildPUHashExpressionsForAggregates(OptimizerExten // Record hash binding for categorical rewriter before modifying ColumnBinding hash_binding; - if (combined_hash_expr->type == ExpressionType::BOUND_COLUMN_REF) { - hash_binding = combined_hash_expr->Cast().binding; + if (combined_hash_expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { + hash_binding = combined_hash_expr->Cast().BindingMutable(); } result.push_back({target_agg, std::move(combined_hash_expr), hash_binding, correction}); @@ -1169,12 +1171,12 @@ void ModifyPlanWithPU(OptimizerExtensionInput &input, unique_ptraggregate_index; - info.group_index = hash_info.aggregate->group_index; + info.aggregate_index = hash_info.aggregate->aggregate_index.index; + info.group_index = hash_info.aggregate->group_index.index; info.hash_binding = hash_info.hash_binding; - pac_agg_info[hash_info.aggregate->aggregate_index] = info; + pac_agg_info[hash_info.aggregate->aggregate_index.index] = info; } } } @@ -1309,7 +1311,7 @@ static void CompilePacMechanism(const PrivacyCompatibilityResult &check, Optimiz // clipping variants with per-PU pre-aggregation below. { Value clip_val; - if (input.context.TryGetCurrentSetting("priv_clip_support", clip_val) && !clip_val.IsNull()) { + if (input.context.TryGetCurrentSetting(Identifier("priv_clip_support"), clip_val) && !clip_val.IsNull()) { RewriteClipAggregates(input, plan, check, pu_setup.pu_names); } } diff --git a/src/compiler/privacy_compiler_helpers.cpp b/src/compiler/privacy_compiler_helpers.cpp index 591a0253..c57f2528 100644 --- a/src/compiler/privacy_compiler_helpers.cpp +++ b/src/compiler/privacy_compiler_helpers.cpp @@ -2,6 +2,7 @@ // Created by ila on 12/23/25. // +#include "duckdb/main/database_manager.hpp" #include "compiler/privacy_compiler_helpers.hpp" #include "privacy_debug.hpp" #include "core/privacy_optimizer.hpp" @@ -44,14 +45,16 @@ void BuildJoinConditions(LogicalGet *left_get, LogicalGet *right_get, const vect if (lproj == DConstants::INVALID_INDEX || rproj == DConstants::INVALID_INDEX) { throw InternalException("Privacy compiler: failed to project FK/PK columns for join"); } - JoinCondition cond; - cond.comparison = ExpressionType::COMPARE_EQUAL; + JoinCondition cond(nullptr, nullptr, ExpressionType::COMPARE_EQUAL); + auto left_col_index = left_get->GetColumnIds()[lproj]; auto right_col_index = right_get->GetColumnIds()[rproj]; auto &left_type = left_get->GetColumnType(left_col_index); auto &right_type = right_get->GetColumnType(right_col_index); - cond.left = make_uniq(left_type, ColumnBinding(left_get->table_index, lproj)); - cond.right = make_uniq(right_type, ColumnBinding(right_get->table_index, rproj)); + cond.LeftReference() = make_uniq( + left_type, ColumnBinding(TableIndex(left_get->table_index.index), ProjectionIndex(lproj))); + cond.RightReference() = make_uniq( + right_type, ColumnBinding(TableIndex(right_get->table_index.index), ProjectionIndex(rproj))); conditions.push_back(std::move(cond)); } } @@ -83,8 +86,8 @@ unique_ptr CreateLogicalJoin(const PrivacyCompatibilityResult & throw InternalException( "Privacy compiler: expected both LogicalGet nodes to be bound to tables for join creation"); } - string left_table_name = left_table_ptr->name; - string right_table_name = right_table_ptr->name; + string left_table_name = left_table_ptr->name.GetIdentifierName(); + string right_table_name = right_table_ptr->name.GetIdentifierName(); // Require metadata to be present; compatibility check is responsible for populating it. auto lit = check.table_metadata.find(left_table_name); @@ -160,9 +163,8 @@ unique_ptr CreateLogicalJoin(const PrivacyCompatibilityResult & right_table_name); } - vector> extra; return LogicalComparisonJoin::CreateJoin(JoinType::INNER, JoinRefType::REGULAR, std::move(left_operator), - std::move(right), std::move(conditions), std::move(extra)); + std::move(right), std::move(conditions)); } // Create a LogicalGet operator for a table by name, projecting only the specified columns @@ -173,8 +175,8 @@ unique_ptr CreateLogicalGet(ClientContext &context, unique_ptr CreateLogicalGet(ClientContext &context, unique_ptr bind_data; auto scan_function = table_entry.GetScanFunction(context, bind_data); vector return_types = {}; - vector return_names = {}; + vector return_names = {}; vector column_ids = {}; - vector projection_ids = {}; + vector projection_ids = {}; // Build a set of required column names for fast lookup (case-insensitive) std::unordered_set required_set; @@ -198,15 +200,15 @@ unique_ptr CreateLogicalGet(ClientContext &context, unique_ptr found_columns; for (auto &col : table_entry.GetColumns().Logical()) { - string col_name_lower = StringUtil::Lower(col.Name()); + string col_name_lower = StringUtil::Lower(col.Name().GetIdentifierName()); // If required_columns is empty, include all columns (backward compatible) // Otherwise, only include columns that are in the required set if (required_columns.empty() || required_set.count(col_name_lower) > 0) { return_types.push_back(col.Type()); - return_names.push_back(col.Name()); + return_names.emplace_back(col.Name().GetIdentifierName()); column_ids.push_back(ColumnIndex(col.Oid())); - projection_ids.push_back(column_ids.size() - 1); + projection_ids.push_back(ProjectionIndex(column_ids.size() - 1)); if (!required_columns.empty()) { found_columns.insert(col_name_lower); @@ -238,20 +240,21 @@ unique_ptr CreateLogicalGet(ClientContext &context, unique_ptr get = make_uniq(idx, scan_function, std::move(bind_data), - std::move(return_types), std::move(return_names)); + unique_ptr get = + make_uniq(TableIndex(idx), BoundTableFunction(scan_function), std::move(bind_data), + std::move(return_types), std::move(return_names)); get->SetColumnIds(std::move(column_ids)); get->projection_ids = projection_ids; get->ResolveOperatorTypes(); get->Verify(context); - return get; + return std::move(get); } throw ParserException("PAC: missing internal sample table " + table); @@ -354,7 +357,7 @@ LogicalGet *FindAccessibleGetInSubtree(unique_ptr &plan, Logica FindAllNodesByTable(&plan, table_name, nodes); for (auto *node : nodes) { auto &node_get = node->get()->Cast(); - if (HasTableIndexInSubtree(subtree_root, node_get.table_index)) { + if (HasTableIndexInSubtree(subtree_root, node_get.table_index.index)) { return &node_get; } } diff --git a/src/compiler/privacy_mechanisms.cpp b/src/compiler/privacy_mechanisms.cpp index 9bfa2948..f9b6ab65 100644 --- a/src/compiler/privacy_mechanisms.cpp +++ b/src/compiler/privacy_mechanisms.cpp @@ -89,7 +89,7 @@ static unique_ptr BindScalarLocal(OptimizerExtensionInput &input, co vector> children) { FunctionBinder function_binder(input.context); ErrorData error; - auto expr = function_binder.BindScalarFunction(DEFAULT_SCHEMA, func_name, std::move(children), error); + auto expr = function_binder.BindScalarFunction(DEFAULT_SCHEMA, Identifier(func_name), std::move(children), error); if (error.HasError()) { throw InternalException("dp: failed to bind scalar function '" + func_name + "': " + error.Message()); } @@ -103,12 +103,12 @@ static unique_ptr BindAggregateLocal(OptimizerExtensionInput &input, vector arg_types; arg_types.reserve(children.size()); for (auto &child : children) { - arg_types.push_back(child->return_type); + arg_types.push_back(child->GetReturnType()); } auto &entry = Catalog::GetSystemCatalog(input.context) - .GetEntry(input.context, DEFAULT_SCHEMA, func_name); - auto best = function_binder.BindFunction(entry.name, entry.functions, arg_types, error); + .GetEntry(input.context, DEFAULT_SCHEMA, Identifier(func_name)); + auto best = function_binder.BindFunction(Identifier(entry.name), entry.functions, arg_types, error); if (!best.IsValid()) { throw InternalException("dp: failed to bind aggregate function '" + func_name + "': " + error.Message()); } @@ -130,15 +130,15 @@ static vector RewriteAvgAggregates(OptimizerExtensionInput &input, Logi idx_t avg_idx = 0; for (idx_t i = 0; i < n_original; i++) { auto &aggr = agg->expressions[i]->Cast(); - if (aggr.function.name != "avg") { + if (aggr.FunctionMutable().GetName().GetIdentifierName() != "avg") { continue; } - auto output_type = aggr.return_type; + auto output_type = aggr.GetReturnType(); // Preserve `FILTER (WHERE ...)` from the original AVG. For fixed-bound means, // both the rewritten SUM and appended COUNT(*) receive the predicate. - auto count_arg = count_non_null_value ? aggr.children[0]->Copy() : nullptr; - auto arg = std::move(aggr.children[0]); - auto filter_for_sum = std::move(aggr.filter); + auto count_arg = count_non_null_value ? aggr.GetChildrenMutable()[0]->Copy() : nullptr; + auto arg = std::move(aggr.GetChildrenMutable()[0]); + auto filter_for_sum = std::move(aggr.GetFilterMutable()); auto filter_for_count = !automatic_bounds_mean && filter_for_sum ? filter_for_sum->Copy() : nullptr; AvgDomain domain {0.0, 0.0}; if (avg_bounds) { @@ -159,14 +159,14 @@ static vector RewriteAvgAggregates(OptimizerExtensionInput &input, Logi } agg->expressions[i] = BindPlainAggregate(input, "sum", std::move(arg)); - agg->expressions[i]->Cast().filter = std::move(filter_for_sum); + agg->expressions[i]->Cast().GetFilterMutable() = std::move(filter_for_sum); idx_t count_pos = DConstants::INVALID_INDEX; if (!automatic_bounds_mean) { count_pos = agg->expressions.size(); agg->expressions.push_back(count_non_null_value ? BindPlainAggregate(input, "count", std::move(count_arg)) : BindPlainAggregate(input, "count_star", nullptr)); - agg->expressions.back()->Cast().filter = std::move(filter_for_count); + agg->expressions.back()->Cast().GetFilterMutable() = std::move(filter_for_count); } AvgInfo info; @@ -202,7 +202,7 @@ static DPFKChain ExtractFKChain(const PrivacyCompatibilityResult &check, const v if (!table) { continue; } - string name = StringUtil::Lower(table->name); + string name = StringUtil::Lower(table->name.GetIdentifierName()); name_count[name]++; if (name_count[name] > 1) { throw InvalidInputException("dp: self-joins are not supported (table '" + table->name + @@ -363,11 +363,11 @@ static void CheckDPAggregateNode(LogicalAggregate *agg, const string &mechanism_ throw InvalidInputException(mechanism_name + ": unsupported non-aggregate expression in aggregate node"); } auto &aggr = expr->Cast(); - string name = StringUtil::Lower(aggr.function.name); + string name = StringUtil::Lower(aggr.FunctionMutable().GetName().GetIdentifierName()); bool is_count = name == "count" || name == "count_star"; if (aggr.IsDistinct()) { if ((mechanism_name != "dp_sass" && mechanism_name != "dp_elastic" && mechanism_name != "dp_standard") || - !is_count || aggr.children.size() != 1 || aggr.filter) { + !is_count || aggr.GetChildrenMutable().size() != 1 || aggr.GetFilterMutable()) { throw InvalidInputException(mechanism_name + ": only COUNT(DISTINCT x) is supported for DISTINCT"); } continue; @@ -383,14 +383,14 @@ static void CheckDPAggregateNode(LogicalAggregate *agg, const string &mechanism_ if (!is_count && !is_sum_avg && !(allow_min_max && is_min_max)) { string supported = allow_min_max ? "COUNT, SUM, AVG, MIN, and MAX" : "COUNT, SUM, and AVG"; throw InvalidInputException(mechanism_name + ": only " + supported + " are supported (got '" + - aggr.function.name + "')"); + aggr.FunctionMutable().GetName().GetIdentifierName() + "')"); } - if ((is_sum_avg || is_min_max) && aggr.children.size() != 1) { + if ((is_sum_avg || is_min_max) && aggr.GetChildrenMutable().size() != 1) { throw InvalidInputException(mechanism_name + ": " + name + " must have exactly one argument"); } - if ((is_sum_avg || is_min_max) && !aggr.children[0]->return_type.IsNumeric()) { + if ((is_sum_avg || is_min_max) && !aggr.GetChildrenMutable()[0]->GetReturnType().IsNumeric()) { throw InvalidInputException(mechanism_name + ": " + name + " argument must be a numeric type (got '" + - aggr.children[0]->return_type.ToString() + "')"); + aggr.GetChildrenMutable()[0]->GetReturnType().ToString() + "')"); } } } @@ -419,7 +419,7 @@ static void CountAccessibleTableScans(LogicalOperator *op, std::unordered_mapCast(); auto table = get.GetTable(); if (table) { - counts[StringUtil::Lower(table->name)]++; + counts[StringUtil::Lower(table->name.GetIdentifierName())]++; } return; } @@ -453,7 +453,7 @@ static double ValidateDPSelfJoins(unique_ptr &plan, const strin for (auto *get : gets) { auto table = get->GetTable(); if (table) { - all_counts[StringUtil::Lower(table->name)]++; + all_counts[StringUtil::Lower(table->name.GetIdentifierName())]++; } } @@ -493,7 +493,8 @@ static double ComputeMfK(ClientContext &context, const string &table_name, const // Table is empty: no rows contribute → max frequency is 0 return 0.0; } - auto val = result->GetValue(0, 0); + auto chunk = result->Fetch(); + auto val = chunk->GetValue(0, 0); return val.IsNull() ? 0.0 : val.GetValue(); } @@ -608,7 +609,8 @@ enum class DPSensitivityKind { GLOBAL, ELASTIC_SMOOTH }; static bool AggregateContainsSum(const LogicalAggregate *agg) { for (auto &expr : agg->expressions) { auto &a = expr->Cast(); - if (a.function.name == "sum" || a.function.name == "avg") { + if (a.FunctionMutable().GetName().GetIdentifierName() == "sum" || + a.FunctionMutable().GetName().GetIdentifierName() == "avg") { return true; } } @@ -618,7 +620,7 @@ static bool AggregateContainsSum(const LogicalAggregate *agg) { static bool AggregateContainsAvg(const LogicalAggregate *agg) { for (auto &expr : agg->expressions) { auto &a = expr->Cast(); - if (a.function.name == "avg") { + if (a.FunctionMutable().GetName().GetIdentifierName() == "avg") { return true; } } @@ -629,7 +631,7 @@ static idx_t CountAvgAggregates(const LogicalAggregate *agg) { idx_t count = 0; for (auto &expr : agg->expressions) { auto &a = expr->Cast(); - if (a.function.name == "avg") { + if (a.FunctionMutable().GetName().GetIdentifierName() == "avg") { count++; } } @@ -640,7 +642,7 @@ static idx_t CountExplicitSumAggregates(const LogicalAggregate *agg) { idx_t count = 0; for (auto &expr : agg->expressions) { auto &a = expr->Cast(); - if (a.function.name == "sum") { + if (a.FunctionMutable().GetName().GetIdentifierName() == "sum") { count++; } } @@ -683,7 +685,7 @@ static vector GetDpSumContributionBounds(ClientContext &context, const s idx_t explicit_sum_count = 0; for (idx_t i = 0; i < agg->expressions.size(); i++) { auto &aggr = agg->expressions[i]->Cast(); - if (aggr.function.name == "sum" && !FindAvgInfoForSumPos(avg_infos, i)) { + if (aggr.FunctionMutable().GetName().GetIdentifierName() == "sum" && !FindAvgInfoForSumPos(avg_infos, i)) { explicit_sum_count++; } } @@ -709,7 +711,7 @@ static vector GetDpSumContributionBounds(ClientContext &context, const s bool needs_scalar_sum_bound = false; for (idx_t i = 0; i < agg->expressions.size(); i++) { auto &aggr = agg->expressions[i]->Cast(); - if (aggr.function.name != "sum") { + if (aggr.FunctionMutable().GetName().GetIdentifierName() != "sum") { continue; } if (FindAvgInfoForSumPos(avg_infos, i)) { @@ -727,7 +729,7 @@ static vector GetDpSumContributionBounds(ClientContext &context, const s idx_t sum_list_idx = 0; for (idx_t i = 0; i < agg->expressions.size(); i++) { auto &aggr = agg->expressions[i]->Cast(); - if (aggr.function.name != "sum") { + if (aggr.FunctionMutable().GetName().GetIdentifierName() != "sum") { continue; } if (FindAvgInfoForSumPos(avg_infos, i)) { @@ -745,7 +747,7 @@ static unique_ptr ClipToBounds(OptimizerExtensionInput &input, uniqu double hi, const LogicalType &restore_type) { auto as_double = BoundCastExpression::AddCastToType(input.context, std::move(expr), LogicalType::DOUBLE); auto is_null = make_uniq(ExpressionType::OPERATOR_IS_NULL, LogicalType::BOOLEAN); - is_null->children.push_back(as_double->Copy()); + is_null->GetChildrenMutable().push_back(as_double->Copy()); auto hi_const = make_uniq(Value::DOUBLE(hi)); auto lo_const = make_uniq(Value::DOUBLE(lo)); auto upper = input.optimizer.BindScalarFunction("least", std::move(as_double), std::move(hi_const)); @@ -754,8 +756,8 @@ static unique_ptr ClipToBounds(OptimizerExtensionInput &input, uniqu BoundCaseCheck null_check; null_check.when_expr = std::move(is_null); null_check.then_expr = make_uniq(Value(LogicalType::DOUBLE)); - preserve_null->case_checks.push_back(std::move(null_check)); - preserve_null->else_expr = std::move(clipped); + preserve_null->CaseChecksMutable().push_back(std::move(null_check)); + preserve_null->ElseMutable() = std::move(clipped); return BoundCastExpression::AddCastToType(input.context, std::move(preserve_null), restore_type); } @@ -763,13 +765,14 @@ static void ClipSumInputs(OptimizerExtensionInput &input, LogicalAggregate *agg, for (idx_t i = 0; i < agg->expressions.size(); i++) { auto &expr = agg->expressions[i]; auto &aggr = expr->Cast(); - if (aggr.function.name != "sum" || aggr.children.empty()) { + if (aggr.FunctionMutable().GetName().GetIdentifierName() != "sum" || aggr.GetChildrenMutable().empty()) { continue; } - auto original_type = aggr.children[0]->return_type; + auto original_type = aggr.GetChildrenMutable()[0]->GetReturnType(); D_ASSERT(i < sum_bounds.size()); double sum_bound = sum_bounds[i]; - aggr.children[0] = ClipToBounds(input, std::move(aggr.children[0]), -sum_bound, sum_bound, original_type); + aggr.GetChildrenMutable()[0] = + ClipToBounds(input, std::move(aggr.GetChildrenMutable()[0]), -sum_bound, sum_bound, original_type); } agg->ResolveOperatorTypes(); } @@ -786,7 +789,7 @@ static double GetRequiredDpBound(ClientContext &context, const string &setting_n } else if (setting_name == "dp_sass_count_output_bound" || setting_name == "dp_sass_sum_output_bound" || setting_name == "dp_sass_avg_lower_bound" || setting_name == "dp_sass_avg_upper_bound") { Value val; - found = context.TryGetCurrentSetting(setting_name, val) && !val.IsNull(); + found = context.TryGetCurrentSetting(Identifier(setting_name), val) && !val.IsNull(); if (found) { bound = val.GetValue(); } @@ -818,7 +821,7 @@ static int64_t GetRequiredDpIntegerBound(ClientContext &context, const string &s static double GetRequiredFiniteSetting(ClientContext &context, const string &setting_name, const string &mechanism_name) { Value val; - if (!context.TryGetCurrentSetting(setting_name, val) || val.IsNull()) { + if (!context.TryGetCurrentSetting(Identifier(setting_name), val) || val.IsNull()) { throw InvalidInputException(mechanism_name + ": " + setting_name + " must be set"); } double bound = val.GetValue(); @@ -831,7 +834,7 @@ static double GetRequiredFiniteSetting(ClientContext &context, const string &set static bool TryGetFiniteSetting(ClientContext &context, const string &setting_name, double &bound) { Value val; - if (!context.TryGetCurrentSetting(setting_name, val) || val.IsNull()) { + if (!context.TryGetCurrentSetting(Identifier(setting_name), val) || val.IsNull()) { return false; } bound = val.GetValue(); @@ -844,7 +847,7 @@ static bool TryGetFiniteSetting(ClientContext &context, const string &setting_na static vector ParseFiniteSettingList(ClientContext &context, const string &setting_name, bool &found) { found = false; Value val; - if (!context.TryGetCurrentSetting(setting_name, val) || val.IsNull()) { + if (!context.TryGetCurrentSetting(Identifier(setting_name), val) || val.IsNull()) { return {}; } string text = val.ToString(); @@ -1029,7 +1032,7 @@ static std::pair GetDpMinMaxBounds(ClientContext &context, const // Elastic Laplace scale = sensitivity / epsilon; sensitivity = es for COUNT, es * value_bound for SUM. static double ElasticSensitivity(idx_t aggregate_pos, const BoundAggregateExpression &aggr, const vector &sum_bounds, double es) { - const string &name = aggr.function.name; + const string &name = aggr.Function().GetName().GetIdentifierName(); if (name == "count" || name == "count_star") { return es; } @@ -1041,7 +1044,8 @@ static double ElasticSensitivity(idx_t aggregate_pos, const BoundAggregateExpres } static bool IsCountAggregate(const BoundAggregateExpression &aggr) { - return aggr.function.name == "count" || aggr.function.name == "count_star"; + return aggr.Function().GetName().GetIdentifierName() == "count" || + aggr.Function().GetName().GetIdentifierName() == "count_star"; } // Does the aggregate contain any COUNT/COUNT(*) (after AVG→SUM+COUNT rewrite)? @@ -1055,7 +1059,8 @@ static bool AggregateContainsCount(const LogicalAggregate *agg) { } static bool IsMinMaxAggregate(const BoundAggregateExpression &aggr) { - return aggr.function.name == "min" || aggr.function.name == "max"; + return aggr.Function().GetName().GetIdentifierName() == "min" || + aggr.Function().GetName().GetIdentifierName() == "max"; } static bool AggregateContainsMinMax(const LogicalAggregate *agg) { @@ -1108,7 +1113,7 @@ static void EnsurePuAdjacentTableAvailable(OptimizerExtensionInput &input, uniqu auto &binder = input.optimizer.binder; unique_ptr current = (*start_ref)->Copy(input.context); for (auto &table : tables_to_join) { - idx_t table_index = binder.GenerateTableIndex(); + idx_t table_index = binder.GenerateTableIndex().index; auto get = CreateLogicalGet(input.context, plan, table, table_index); current = CreateLogicalJoin(check, input.context, std::move(current), std::move(get)); } @@ -1155,10 +1160,10 @@ static unique_ptr BuildPerPuExpressionAtOperator(OptimizerExtensionI } auto col_index = key_get->GetColumnIds()[proj_idx]; auto col_type = key_get->GetColumnType(col_index); - ColumnBinding source_binding(key_get->table_index, proj_idx); + ColumnBinding source_binding {TableIndex(key_get->table_index.index), ProjectionIndex(proj_idx)}; auto propagated = - PropagateSingleBindingToOperator(*plan, key_get->table_index, source_binding, col_type, target); - if (propagated.table_index == DConstants::INVALID_INDEX) { + PropagateSingleBindingToOperator(*plan, key_get->table_index.index, source_binding, col_type, target); + if (propagated.table_index.index == DConstants::INVALID_INDEX) { throw InternalException(mech + ": could not propagate PU key column '" + key_column + "' to aggregate input"); } @@ -1167,18 +1172,19 @@ static unique_ptr BuildPerPuExpressionAtOperator(OptimizerExtensionI auto pu_group_expr = BuildXorHash(input, std::move(key_exprs)); pu_group_expr = BoundCastExpression::AddCastToType(input.context, std::move(pu_group_expr), LogicalType::UBIGINT); - pu_group_expr->alias = mech + "_pu_key"; + pu_group_expr->SetAlias(Identifier(mech + "_pu_key")); return pu_group_expr; } static unique_ptr BuildRankSourceRef(const vector &source_types, idx_t source_table_index, idx_t column) { - return make_uniq(source_types[column], ColumnBinding(source_table_index, column)); + return make_uniq(source_types[column], + ColumnBinding(TableIndex(source_table_index), ProjectionIndex(column))); } static unique_ptr BuildIsNullPredicate(unique_ptr expr) { auto is_null = make_uniq(ExpressionType::OPERATOR_IS_NULL, LogicalType::BOOLEAN); - is_null->children.push_back(std::move(expr)); + is_null->GetChildrenMutable().push_back(std::move(expr)); return is_null; } @@ -1205,22 +1211,23 @@ static unique_ptr BuildRankCapFilter(OptimizerExtensionInput &i return child; } auto &binder = input.optimizer.binder; - idx_t window_index = binder.GenerateTableIndex(); - auto window = make_uniq(window_index); + idx_t window_index = binder.GenerateTableIndex().index; + auto window = make_uniq(TableIndex(window_index)); for (auto &spec : specs) { - auto rank = make_uniq(spec.rank_type, LogicalType::BIGINT, nullptr, nullptr); + auto rank = BindRankingWindow(input.context, spec.rank_type); for (auto col : spec.partition_cols) { - rank->partitions.push_back(BuildRankSourceRef(source_types, source_table_index, col)); + rank->PartitionsMutable().push_back(BuildRankSourceRef(source_types, source_table_index, col)); } for (auto col : spec.keep_null_cols) { - rank->orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, - BuildIsNullPredicate(BuildRankSourceRef(source_types, source_table_index, col))); + rank->OrderByMutable().emplace_back( + OrderType::ASCENDING, OrderByNullType::NULLS_LAST, + BuildIsNullPredicate(BuildRankSourceRef(source_types, source_table_index, col))); } if (spec.rank_type == ExpressionType::WINDOW_RANK_DENSE) { for (auto col : spec.order_cols) { - rank->orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, - BuildRankSourceRef(source_types, source_table_index, col)); + rank->OrderByMutable().emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, + BuildRankSourceRef(source_types, source_table_index, col)); } } else { vector> hash_children; @@ -1229,19 +1236,21 @@ static unique_ptr BuildRankCapFilter(OptimizerExtensionInput &i hash_children.push_back(BuildRankSourceRef(source_types, source_table_index, col)); } auto hash_expr = BindScalarLocal(input, "hash", std::move(hash_children)); - rank->orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, std::move(hash_expr)); + rank->OrderByMutable().emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, + std::move(hash_expr)); } - rank->start = WindowBoundary::UNBOUNDED_PRECEDING; - rank->end = WindowBoundary::CURRENT_ROW_ROWS; + rank->WindowStartMutable() = WindowBoundary::UNBOUNDED_PRECEDING; + rank->WindowEndMutable() = WindowBoundary::CURRENT_ROW_ROWS; window->expressions.push_back(std::move(rank)); } window->children.push_back(std::move(child)); vector> keep_conditions; for (idx_t i = 0; i < specs.size(); i++) { - auto rank_ref = make_uniq(LogicalType::BIGINT, ColumnBinding(window_index, i)); + auto rank_ref = make_uniq( + LogicalType::BIGINT, ColumnBinding(TableIndex(window_index), ProjectionIndex(i))); auto cap_const = make_uniq(Value::BIGINT(specs[i].cap)); - unique_ptr keep_condition = make_uniq( + unique_ptr keep_condition = BoundComparisonExpression::Create( ExpressionType::COMPARE_LESSTHANOREQUALTO, std::move(rank_ref), std::move(cap_const)); for (auto col : specs[i].keep_null_cols) { keep_condition = make_uniq( @@ -1255,7 +1264,7 @@ static unique_ptr BuildRankCapFilter(OptimizerExtensionInput &i keep_expr = std::move(keep_conditions[0]); } else { auto conj = make_uniq(ExpressionType::CONJUNCTION_AND); - conj->children = std::move(keep_conditions); + conj->GetChildrenMutable() = std::move(keep_conditions); keep_expr = std::move(conj); } auto filter = make_uniq(std::move(keep_expr)); @@ -1265,7 +1274,7 @@ static unique_ptr BuildRankCapFilter(OptimizerExtensionInput &i static unique_ptr BuildIsNotNullPredicate(unique_ptr expr) { auto not_null = make_uniq(ExpressionType::OPERATOR_IS_NOT_NULL, LogicalType::BOOLEAN); - not_null->children.push_back(std::move(expr)); + not_null->GetChildrenMutable().push_back(std::move(expr)); return not_null; } @@ -1283,7 +1292,7 @@ static unique_ptr ConjoinPredicates(vector> p return std::move(present[0]); } auto conjunction = make_uniq(ExpressionType::CONJUNCTION_AND); - conjunction->children = std::move(present); + conjunction->GetChildrenMutable() = std::move(present); return std::move(conjunction); } @@ -1297,35 +1306,36 @@ static vector> PrepareVariableMDistinctListFilters(Optimi } auto &binder = input.optimizer.binder; - idx_t window_index = binder.GenerateTableIndex(); - auto window = make_uniq(window_index); + idx_t window_index = binder.GenerateTableIndex().index; + auto window = make_uniq(TableIndex(window_index)); vector distinct_filter_indices; for (idx_t ai = 0; ai < agg->expressions.size(); ai++) { auto &aggr = agg->expressions[ai]->Cast(); - string name = StringUtil::Lower(aggr.function.name); - if (name != "count" || !aggr.IsDistinct() || aggr.children.empty()) { + string name = StringUtil::Lower(aggr.FunctionMutable().GetName().GetIdentifierName()); + if (name != "count" || !aggr.IsDistinct() || aggr.GetChildrenMutable().empty()) { continue; } - auto rank = - make_uniq(ExpressionType::WINDOW_RANK_DENSE, LogicalType::BIGINT, nullptr, nullptr); + auto rank = BindRankingWindow(input.context, ExpressionType::WINDOW_RANK_DENSE); for (auto &group : agg->groups) { - rank->partitions.push_back(group->Copy()); + rank->PartitionsMutable().push_back(group->Copy()); } - rank->partitions.push_back(pu_hash_expr.Copy()); - rank->orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, aggr.children[0]->Copy()); - rank->start = WindowBoundary::UNBOUNDED_PRECEDING; - rank->end = WindowBoundary::CURRENT_ROW_ROWS; + rank->PartitionsMutable().push_back(pu_hash_expr.Copy()); + rank->OrderByMutable().emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, + aggr.GetChildrenMutable()[0]->Copy()); + rank->WindowStartMutable() = WindowBoundary::UNBOUNDED_PRECEDING; + rank->WindowEndMutable() = WindowBoundary::CURRENT_ROW_ROWS; idx_t rank_pos = window->expressions.size(); window->expressions.push_back(std::move(rank)); - auto rank_ref = make_uniq(LogicalType::BIGINT, ColumnBinding(window_index, rank_pos)); + auto rank_ref = make_uniq( + LogicalType::BIGINT, ColumnBinding(TableIndex(window_index), ProjectionIndex(rank_pos))); auto cap_const = make_uniq(Value::BIGINT(count_bound)); - auto in_cap = make_uniq(ExpressionType::COMPARE_LESSTHANOREQUALTO, - std::move(rank_ref), std::move(cap_const)); + auto in_cap = BoundComparisonExpression::Create(ExpressionType::COMPARE_LESSTHANOREQUALTO, std::move(rank_ref), + std::move(cap_const)); vector> predicates; - predicates.push_back(BuildIsNotNullPredicate(aggr.children[0]->Copy())); + predicates.push_back(BuildIsNotNullPredicate(aggr.GetChildrenMutable()[0]->Copy())); predicates.push_back(std::move(in_cap)); filters[ai] = ConjoinPredicates(std::move(predicates)); distinct_filter_indices.push_back(ai); @@ -1358,8 +1368,8 @@ static void ApplyMaxGroupsContributed(OptimizerExtensionInput &input, LogicalAgg spec.cap = max_groups; vector specs; specs.push_back(std::move(spec)); - agg->children[0] = - BuildRankCapFilter(input, std::move(agg->children[0]), pre.lower_agg->types, pre.lower_agg->group_index, specs); + agg->children[0] = BuildRankCapFilter(input, std::move(agg->children[0]), pre.lower_agg->types, + pre.lower_agg->group_index.index, specs); } // Standard (global-sensitivity) clipping. Bounds each privacy unit's total contribution to a @@ -1386,17 +1396,17 @@ static void ApplyMaxGroupsContributed(OptimizerExtensionInput &input, LogicalAgg // Returns the value unchanged when there is no filter. static unique_ptr FoldFilterIntoValue(const BoundAggregateExpression &aggr, unique_ptr value, bool else_zero) { - if (!aggr.filter) { + if (!aggr.GetFilter()) { return value; } - auto value_type = value->return_type; + auto value_type = value->GetReturnType(); auto case_expr = make_uniq(value_type); BoundCaseCheck check; - check.when_expr = aggr.filter->Copy(); + check.when_expr = aggr.GetFilter()->Copy(); check.then_expr = std::move(value); - case_expr->case_checks.push_back(std::move(check)); - case_expr->else_expr = make_uniq(else_zero ? Value::Numeric(value_type, 0) - : Value(value_type)); // 0, or NULL of the type + case_expr->CaseChecksMutable().push_back(std::move(check)); + case_expr->ElseMutable() = make_uniq( + else_zero ? Value::Numeric(value_type, 0) : Value(value_type)); // 0, or NULL of the type return std::move(case_expr); } @@ -1453,33 +1463,36 @@ static vector ApplyPerPuClipping(OptimizerExtensionInput &input, unique_ if (aggr.IsDistinct()) { lower_exprs.push_back(agg->expressions[i]->Copy()); // filtered DISTINCT is rejected upstream } else if (IsCountAggregate(aggr)) { - if (aggr.function.name == "count" && !aggr.children.empty()) { + if (aggr.FunctionMutable().GetName().GetIdentifierName() == "count" && !aggr.GetChildrenMutable().empty()) { lower_exprs.push_back(BindPlainAggregate( - input, "count", FoldFilterIntoValue(aggr, aggr.children[0]->Copy(), /*else_zero=*/false))); + input, "count", + FoldFilterIntoValue(aggr, aggr.GetChildrenMutable()[0]->Copy(), /*else_zero=*/false))); } else { // COUNT(*): no value child, so a FILTER clause binds correctly — attach it directly. lower_exprs.push_back(BindPlainAggregate(input, "count_star", nullptr)); - if (aggr.filter) { - lower_exprs.back()->Cast().filter = aggr.filter->Copy(); + if (aggr.GetFilterMutable()) { + lower_exprs.back()->Cast().GetFilterMutable() = + aggr.GetFilterMutable()->Copy(); } } } else if (IsMinMaxAggregate(aggr)) { // Per-PU MIN/MAX partial. else_zero=false → filtered-out rows become NULL, which MIN/MAX skip. lower_exprs.push_back(BindPlainAggregate( - input, aggr.function.name, FoldFilterIntoValue(aggr, aggr.children[0]->Copy(), /*else_zero=*/false))); + input, aggr.FunctionMutable().GetName().GetIdentifierName(), + FoldFilterIntoValue(aggr, aggr.GetChildrenMutable()[0]->Copy(), /*else_zero=*/false))); } else if (avg_sum_info && avg_sum_info->IsAutomaticBoundsMean()) { - auto value = FoldFilterIntoValue(aggr, aggr.children[0]->Copy(), /*else_zero=*/false); - bounds_parameters[i] = GetDpApproxBoundsParameters(value->return_type); + auto value = FoldFilterIntoValue(aggr, aggr.GetChildrenMutable()[0]->Copy(), /*else_zero=*/false); + bounds_parameters[i] = GetDpApproxBoundsParameters(value->GetReturnType()); vector> children; children.push_back( BoundCastExpression::AddCastToType(input.context, std::move(value), LogicalType::DOUBLE)); children.push_back(make_uniq(Value::BIGINT(automatic_mean_count_bound))); lower_exprs.push_back(BindAggregateLocal(input, "dp_bounded_value_list", std::move(children))); } else { // sum - auto value = FoldFilterIntoValue(aggr, aggr.children[0]->Copy(), /*else_zero=*/true); - bounds_parameters[i] = GetDpApproxBoundsParameters(value->return_type); - if (auto_bounds && (value->return_type.InternalType() == PhysicalType::FLOAT || - value->return_type.InternalType() == PhysicalType::DOUBLE)) { + auto value = FoldFilterIntoValue(aggr, aggr.GetChildrenMutable()[0]->Copy(), /*else_zero=*/true); + bounds_parameters[i] = GetDpApproxBoundsParameters(value->GetReturnType()); + if (auto_bounds && (value->GetReturnType().InternalType() == PhysicalType::FLOAT || + value->GetReturnType().InternalType() == PhysicalType::DOUBLE)) { lower_exprs.push_back(BindPlainAggregate(input, "priv_approx_sum", std::move(value))); } else { lower_exprs.push_back(BindPlainAggregate(input, "sum", std::move(value))); @@ -1496,10 +1509,10 @@ static vector ApplyPerPuClipping(OptimizerExtensionInput &input, unique_ auto &orig = agg->expressions[i]->Cast(); bool pos_min_max = IsMinMaxAggregate(orig); auto *avg_sum_info = FindAvgInfoForSumPos(avg_infos, i); - string min_max_name = pos_min_max ? orig.function.name : string(); + string min_max_name = pos_min_max ? orig.FunctionMutable().GetName().GetIdentifierName() : string(); auto lower_type = pre.lower_agg->types[pre.num_original_groups + 1 + i]; - unique_ptr lower_ref = - make_uniq(lower_type, ColumnBinding(pre.lower_agg_index, i)); + unique_ptr lower_ref = make_uniq( + lower_type, ColumnBinding(TableIndex(pre.lower_agg_index), ProjectionIndex(i))); if (pos_min_max) { // Clip the per-PU partial to [mm_lower, mm_upper], then take the overall MIN/MAX. Removing one // PU shifts the extremum by at most mm_range per group → L1 sensitivity mm_range · group_bound. @@ -1572,7 +1585,8 @@ static unique_ptr BuildSupportKeyExpression(unique_ptrGetColumnIds()[proj_idx]; auto col_type = get->GetColumnType(col_index); - return make_uniq(col_type, ColumnBinding(get->table_index, proj_idx)); + return make_uniq( + col_type, ColumnBinding(TableIndex(get->table_index.index), ProjectionIndex(proj_idx))); } static optional_idx AddSupportAggregate(OptimizerExtensionInput &input, unique_ptr &plan, @@ -1602,8 +1616,9 @@ static optional_idx AddSupportAggregate(OptimizerExtensionInput &input, unique_p static LogicalFilter *ApplySupportFilter(OptimizerExtensionInput &input, unique_ptr &plan, LogicalAggregate *agg, idx_t support_pos, double threshold, double noise_scale = 0.0) { - auto support_ref = make_uniq(agg->types[agg->groups.size() + support_pos], - ColumnBinding(agg->aggregate_index, support_pos)); + auto support_ref = make_uniq( + agg->types[agg->groups.size() + support_pos], + ColumnBinding(TableIndex(agg->aggregate_index.index), ProjectionIndex(support_pos))); unique_ptr support_expr = BoundCastExpression::AddCastToType(input.context, std::move(support_ref), LogicalType::DOUBLE); if (noise_scale > 0.0 && std::isfinite(noise_scale)) { @@ -1613,8 +1628,8 @@ static LogicalFilter *ApplySupportFilter(OptimizerExtensionInput &input, unique_ support_expr = BindScalarLocal(input, "dp_noise", std::move(noise_children)); } auto threshold_const = make_uniq(Value::DOUBLE(threshold)); - auto condition = make_uniq(ExpressionType::COMPARE_GREATERTHANOREQUALTO, - std::move(support_expr), std::move(threshold_const)); + auto condition = BoundComparisonExpression::Create(ExpressionType::COMPARE_GREATERTHANOREQUALTO, + std::move(support_expr), std::move(threshold_const)); auto filter = make_uniq(); filter->expressions.push_back(std::move(condition)); @@ -1675,12 +1690,14 @@ static NoiseProjection InsertProjectionAndRemap(unique_ptr &pla ColumnBindingReplacer replacer; for (idx_t gi = 0; gi < n_groups; gi++) { - replacer.replacement_bindings.emplace_back(ColumnBinding(source_group_idx, gi), ColumnBinding(proj_idx, gi), + replacer.replacement_bindings.emplace_back(ColumnBinding(TableIndex(source_group_idx), ProjectionIndex(gi)), + ColumnBinding(TableIndex(proj_idx), ProjectionIndex(gi)), proj_types[gi]); } for (idx_t ai = 0; ai < n_aggs; ai++) { - replacer.replacement_bindings.emplace_back(ColumnBinding(source_agg_idx, source_agg_offset + ai), - ColumnBinding(proj_idx, n_groups + ai), proj_types[n_groups + ai]); + replacer.replacement_bindings.emplace_back( + ColumnBinding(TableIndex(source_agg_idx), ProjectionIndex(source_agg_offset + ai)), + ColumnBinding(TableIndex(proj_idx), ProjectionIndex(n_groups + ai)), proj_types[n_groups + ai]); } replacer.stop_operator = proj_ptr; replacer.VisitOperator(*plan); @@ -1698,8 +1715,8 @@ static NoiseProjection WrapAggregateWithLaplace(OptimizerExtensionInput &input, const vector &output_types) { idx_t n_groups = agg->groups.size(); idx_t n_aggs = agg_scales.size(); - idx_t group_idx = agg->group_index; - idx_t agg_idx = agg->aggregate_index; + idx_t group_idx = agg->group_index.index; + idx_t agg_idx = agg->aggregate_index.index; auto agg_types = agg->types; D_ASSERT(agg_types.size() >= n_groups + n_aggs); @@ -1721,13 +1738,15 @@ static NoiseProjection WrapAggregateWithLaplace(OptimizerExtensionInput &input, } for (idx_t gi = 0; gi < n_groups; gi++) { - proj_exprs.push_back(make_uniq(agg_types[gi], ColumnBinding(group_idx, gi))); + proj_exprs.push_back(make_uniq( + agg_types[gi], ColumnBinding(TableIndex(group_idx), ProjectionIndex(gi)))); } for (idx_t ai = 0; ai < n_aggs; ai++) { auto agg_col_type = agg_types[n_groups + ai]; const auto &out_type = output_types[ai]; - auto col_ref = make_uniq(agg_col_type, ColumnBinding(agg_idx, ai)); + auto col_ref = + make_uniq(agg_col_type, ColumnBinding(TableIndex(agg_idx), ProjectionIndex(ai))); double scale = agg_scales[ai]; if (!std::isfinite(scale)) { // A non-finite scale means the sensitivity or ε overflowed (e.g. an elastic-sensitivity @@ -1765,14 +1784,15 @@ static NoiseProjection WrapAggregateWithLaplace(OptimizerExtensionInput &input, proj_exprs.push_back(std::move(noised)); } - idx_t proj_idx = input.optimizer.binder.GenerateTableIndex(); - auto projection = make_uniq(proj_idx, std::move(proj_exprs)); + idx_t proj_idx = input.optimizer.binder.GenerateTableIndex().index; + auto projection = make_uniq(TableIndex(proj_idx), std::move(proj_exprs)); return InsertProjectionAndRemap(plan, std::move(projection), insert_above, proj_idx, group_idx, agg_idx, n_groups, n_aggs, 0, "dp: could not locate noise insertion slot in plan"); } static bool IsPlainCountStarAggregate(const BoundAggregateExpression &aggr) { - return aggr.function.name == "count_star" && !aggr.IsDistinct() && aggr.children.empty() && !aggr.filter; + return aggr.Function().GetName().GetIdentifierName() == "count_star" && !aggr.IsDistinct() && + aggr.GetChildren().empty() && !aggr.GetFilter(); } static bool IsDPSassFastCountStarEligible(const LogicalAggregate *agg, const DPFKChain &chain) { @@ -1847,8 +1867,8 @@ WrapAvgRatioProjection(OptimizerExtensionInput &input, unique_ptr(noise_proj.proj_types[gi], ColumnBinding(noise_proj.proj_idx, gi))); + proj_exprs.push_back(make_uniq( + noise_proj.proj_types[gi], ColumnBinding(TableIndex(noise_proj.proj_idx), ProjectionIndex(gi)))); } // For each original agg position: pass-through or compute AVG ratio @@ -1857,15 +1877,15 @@ WrapAvgRatioProjection(OptimizerExtensionInput &input, unique_ptr(col_type, ColumnBinding(noise_proj.proj_idx, n_groups + ai))); + proj_exprs.push_back(make_uniq( + col_type, ColumnBinding(TableIndex(noise_proj.proj_idx), ProjectionIndex(n_groups + ai)))); } else { // AVG: compute CAST(noised_sum AS DOUBLE) / CAST(noised_count AS DOUBLE) const AvgInfo &info = *avg_info; const auto &sum_type = noise_proj.proj_types[n_groups + info.sum_pos]; if (info.IsAutomaticBoundsMean()) { auto mean_ref = make_uniq( - sum_type, ColumnBinding(noise_proj.proj_idx, n_groups + info.sum_pos)); + sum_type, ColumnBinding(TableIndex(noise_proj.proj_idx), ProjectionIndex(n_groups + info.sum_pos))); proj_exprs.push_back( BoundCastExpression::AddCastToType(input.context, std::move(mean_ref), LogicalType::DOUBLE)); continue; @@ -1874,12 +1894,13 @@ WrapAvgRatioProjection(OptimizerExtensionInput &input, unique_ptr( - sum_type, ColumnBinding(noise_proj.proj_idx, n_groups + info.sum_pos)); + sum_type, ColumnBinding(TableIndex(noise_proj.proj_idx), ProjectionIndex(n_groups + info.sum_pos))); return BoundCastExpression::AddCastToType(input.context, std::move(sum_ref), LogicalType::DOUBLE); }; auto make_cnt_dbl = [&]() { auto cnt_ref = make_uniq( - cnt_type, ColumnBinding(noise_proj.proj_idx, n_groups + info.count_pos)); + cnt_type, + ColumnBinding(TableIndex(noise_proj.proj_idx), ProjectionIndex(n_groups + info.count_pos))); return BoundCastExpression::AddCastToType(input.context, std::move(cnt_ref), LogicalType::DOUBLE); }; auto add_midpoint = [&](unique_ptr value) -> unique_ptr { @@ -1902,18 +1923,18 @@ WrapAvgRatioProjection(OptimizerExtensionInput &input, unique_ptr(LogicalType::DOUBLE); BoundCaseCheck check; check.when_expr = - make_uniq(ExpressionType::COMPARE_LESSTHANOREQUALTO, make_cnt_dbl(), - make_uniq(Value::DOUBLE(0.0))); + BoundComparisonExpression::Create(ExpressionType::COMPARE_LESSTHANOREQUALTO, make_cnt_dbl(), + make_uniq(Value::DOUBLE(0.0))); check.then_expr = make_uniq(Value(LogicalType::DOUBLE)); - case_expr->case_checks.push_back(std::move(check)); - case_expr->else_expr = maybe_clamp(add_midpoint(std::move(ratio)), info); + case_expr->CaseChecksMutable().push_back(std::move(check)); + case_expr->ElseMutable() = maybe_clamp(add_midpoint(std::move(ratio)), info); proj_exprs.push_back(std::move(case_expr)); } } } - idx_t ratio_idx = input.optimizer.binder.GenerateTableIndex(); - auto ratio_proj = make_uniq(ratio_idx, std::move(proj_exprs)); + idx_t ratio_idx = input.optimizer.binder.GenerateTableIndex().index; + auto ratio_proj = make_uniq(TableIndex(ratio_idx), std::move(proj_exprs)); auto inserted = InsertProjectionAndRemap(plan, std::move(ratio_proj), insert_above, ratio_idx, noise_proj.proj_idx, noise_proj.proj_idx, n_groups, n_original_aggs, n_groups, "dp: could not locate insert point for AVG ratio projection"); @@ -1922,7 +1943,7 @@ WrapAvgRatioProjection(OptimizerExtensionInput &input, unique_ptr(window_index); - auto row_number = - make_uniq(ExpressionType::WINDOW_ROW_NUMBER, LogicalType::BIGINT, nullptr, nullptr); - row_number->start = WindowBoundary::UNBOUNDED_PRECEDING; - row_number->end = WindowBoundary::CURRENT_ROW_ROWS; + idx_t window_index = input.optimizer.binder.GenerateTableIndex().index; + auto window = make_uniq(TableIndex(window_index)); + auto row_number = BindRankingWindow(input.context, ExpressionType::WINDOW_ROW_NUMBER); + row_number->WindowStartMutable() = WindowBoundary::UNBOUNDED_PRECEDING; + row_number->WindowEndMutable() = WindowBoundary::CURRENT_ROW_ROWS; window->expressions.push_back(std::move(row_number)); window->children.push_back(std::move(*slot)); window->ResolveOperatorTypes(); @@ -2014,15 +2034,17 @@ static NoiseProjection WrapSampleMedianProjection( vector> proj_exprs; proj_exprs.reserve(n_groups + n_aggs); for (idx_t gi = 0; gi < n_groups; gi++) { - proj_exprs.push_back(make_uniq(agg->types[gi], ColumnBinding(agg->group_index, gi))); + proj_exprs.push_back(make_uniq( + agg->types[gi], ColumnBinding(TableIndex(agg->group_index.index), ProjectionIndex(gi)))); } int sample_lanes = GetDpSampleLanes(input.context); for (idx_t ai = 0; ai < n_aggs; ai++) { idx_t agg_pos = agg_positions[ai]; vector> children; - children.push_back(make_uniq(agg->types[n_groups + agg_pos], - ColumnBinding(agg->aggregate_index, agg_pos))); + children.push_back(make_uniq( + agg->types[n_groups + agg_pos], + ColumnBinding(TableIndex(agg->aggregate_index.index), ProjectionIndex(agg_pos)))); if (stability_query_mode) { children.push_back(make_uniq(Value::INTEGER(static_cast(ai)))); unique_ptr recorded = BindScalarLocal(input, "dp_sass_record_stability", std::move(children)); @@ -2059,7 +2081,8 @@ static NoiseProjection WrapSampleMedianProjection( children.push_back(make_uniq(Value::INTEGER(static_cast(ai)))); children.push_back(make_uniq(Value::INTEGER(static_cast(n_aggs)))); children.push_back(make_uniq( - LogicalType::BIGINT, ColumnBinding(noise_identity_index.GetIndex(), 0))); + LogicalType::BIGINT, + ColumnBinding(TableIndex(noise_identity_index.GetIndex()), ProjectionIndex(0)))); terminal = "dp_nonempty_mean_noise"; } } else if (noise_scale_query_mode) { @@ -2076,8 +2099,8 @@ static NoiseProjection WrapSampleMedianProjection( D_ASSERT(noise_identity_index.IsValid()); children.push_back(make_uniq(Value::INTEGER(static_cast(ai)))); children.push_back(make_uniq(Value::INTEGER(static_cast(n_aggs)))); - children.push_back(make_uniq(LogicalType::BIGINT, - ColumnBinding(noise_identity_index.GetIndex(), 0))); + children.push_back(make_uniq( + LogicalType::BIGINT, ColumnBinding(TableIndex(noise_identity_index.GetIndex()), ProjectionIndex(0)))); } unique_ptr noised = BindScalarLocal(input, terminal, std::move(children)); if (output_types[ai] != LogicalType::DOUBLE) { @@ -2086,10 +2109,10 @@ static NoiseProjection WrapSampleMedianProjection( proj_exprs.push_back(std::move(noised)); } - idx_t proj_idx = input.optimizer.binder.GenerateTableIndex(); - auto projection = make_uniq(proj_idx, std::move(proj_exprs)); - return InsertProjectionAndRemap(plan, std::move(projection), insert_above, proj_idx, agg->group_index, - agg->aggregate_index, n_groups, n_aggs, 0, + idx_t proj_idx = input.optimizer.binder.GenerateTableIndex().index; + auto projection = make_uniq(TableIndex(proj_idx), std::move(proj_exprs)); + return InsertProjectionAndRemap(plan, std::move(projection), insert_above, proj_idx, agg->group_index.index, + agg->aggregate_index.index, n_groups, n_aggs, 0, "dp_sample_median: could not locate aggregate slot for median projection"); } @@ -2098,8 +2121,8 @@ static bool IsDpSampleCountDistinct(const LogicalAggregate *agg) { return false; } auto &aggr = agg->expressions[0]->Cast(); - string name = StringUtil::Lower(aggr.function.name); - return aggr.IsDistinct() && (name == "count" || name == "count_star") && aggr.children.size() == 1; + string name = StringUtil::Lower(aggr.FunctionMutable().GetName().GetIdentifierName()); + return aggr.IsDistinct() && (name == "count" || name == "count_star") && aggr.GetChildrenMutable().size() == 1; } static bool IsSinglePuKeyDistinctCount(unique_ptr &plan, LogicalAggregate *agg, @@ -2138,15 +2161,15 @@ static bool IsSinglePuKeyDistinctCount(unique_ptr &plan, Logica } auto col_index = key_get->GetColumnIds()[proj_idx]; auto col_type = key_get->GetColumnType(col_index); - ColumnBinding source_binding(key_get->table_index, proj_idx); - auto propagated = PropagateSingleBinding(*plan, key_get->table_index, source_binding, col_type, agg); - if (propagated.table_index == DConstants::INVALID_INDEX) { + ColumnBinding source_binding {TableIndex(key_get->table_index.index), ProjectionIndex(proj_idx)}; + auto propagated = PropagateSingleBinding(*plan, key_get->table_index.index, source_binding, col_type, agg); + if (propagated.table_index.index == DConstants::INVALID_INDEX) { return false; } auto &aggr = agg->expressions[0]->Cast(); auto key_ref = make_uniq(col_type, propagated); - return Expression::Equals(*aggr.children[0], *key_ref); + return Expression::Equals(*aggr.GetChildrenMutable()[0], *key_ref); } struct DpSassContributionBounds { @@ -2198,7 +2221,7 @@ static DpSassContributionBounds GetDpSassContributionBounds(ClientContext &conte idx_t explicit_sum_count = 0; for (idx_t i = 0; i < agg->expressions.size(); i++) { auto &aggr = agg->expressions[i]->Cast(); - string name = StringUtil::Lower(aggr.function.name); + string name = StringUtil::Lower(aggr.FunctionMutable().GetName().GetIdentifierName()); if (name == "sum" && !FindAvgInfoForSumPos(avg_infos, i)) { explicit_sum_count++; } @@ -2224,7 +2247,7 @@ static DpSassContributionBounds GetDpSassContributionBounds(ClientContext &conte idx_t sum_list_idx = 0; for (idx_t i = 0; i < agg->expressions.size(); i++) { auto &aggr = agg->expressions[i]->Cast(); - string name = StringUtil::Lower(aggr.function.name); + string name = StringUtil::Lower(aggr.FunctionMutable().GetName().GetIdentifierName()); if (FindAvgInfoForSumPos(avg_infos, i)) { RequireDpSassSumBound(context, bounds); bounds.sum_bounds_by_aggregate[i] = bounds.sum_bound; @@ -2252,11 +2275,11 @@ static void RewriteDistinctCountToSampleMedian(OptimizerExtensionInput &input, L int64_t group_bound, bool apply_support_filter, bool variable_m) { auto &binder = input.optimizer.binder; auto &old_aggr = agg->expressions[0]->Cast(); - auto distinct_value = old_aggr.children[0]->Copy(); - auto distinct_type = distinct_value->return_type; + auto distinct_value = old_aggr.GetChildrenMutable()[0]->Copy(); + auto distinct_type = distinct_value->GetReturnType(); - idx_t inner_group_index = binder.GenerateTableIndex(); - idx_t inner_agg_index = binder.GenerateTableIndex(); + idx_t inner_group_index = binder.GenerateTableIndex().index; + idx_t inner_agg_index = binder.GenerateTableIndex().index; vector> inner_expressions; if (!variable_m) { vector> mask_children; @@ -2265,7 +2288,8 @@ static void RewriteDistinctCountToSampleMedian(OptimizerExtensionInput &input, L auto bit_or_expr = BindBitOrAggregate(input, std::move(sample_mask), distinct_value->Copy()); inner_expressions.push_back(std::move(bit_or_expr)); } - auto inner_agg = make_uniq(inner_group_index, inner_agg_index, std::move(inner_expressions)); + auto inner_agg = make_uniq(TableIndex(inner_group_index), TableIndex(inner_agg_index), + std::move(inner_expressions)); idx_t num_original_groups = agg->groups.size(); for (auto &group : agg->groups) { @@ -2277,7 +2301,7 @@ static void RewriteDistinctCountToSampleMedian(OptimizerExtensionInput &input, L inner_agg->ResolveOperatorTypes(); auto *inner_agg_ptr = inner_agg.get(); - idx_t inner_gi = inner_agg_ptr->group_index; + idx_t inner_gi = inner_agg_ptr->group_index.index; unique_ptr distinct_rows = std::move(inner_agg); // Each inner row is one (PU, group, distinct value). Bound a PU's contribution with two @@ -2320,20 +2344,19 @@ static void RewriteDistinctCountToSampleMedian(OptimizerExtensionInput &input, L optional_idx support_window_index; if (apply_support_filter) { - idx_t window_index = binder.GenerateTableIndex(); - auto window = make_uniq(window_index); - auto rn = - make_uniq(ExpressionType::WINDOW_ROW_NUMBER, LogicalType::BIGINT, nullptr, nullptr); + idx_t window_index = binder.GenerateTableIndex().index; + auto window = make_uniq(TableIndex(window_index)); + auto rn = BindRankingWindow(input.context, ExpressionType::WINDOW_ROW_NUMBER); for (idx_t gi = 0; gi < num_original_groups; gi++) { - rn->partitions.push_back(BuildRankSourceRef(inner_agg_ptr->types, inner_gi, gi)); + rn->PartitionsMutable().push_back(BuildRankSourceRef(inner_agg_ptr->types, inner_gi, gi)); } - rn->partitions.push_back(BuildRankSourceRef(inner_agg_ptr->types, inner_gi, pu_group_col)); + rn->PartitionsMutable().push_back(BuildRankSourceRef(inner_agg_ptr->types, inner_gi, pu_group_col)); vector> distinct_hash_children; distinct_hash_children.push_back(BuildRankSourceRef(inner_agg_ptr->types, inner_gi, distinct_col)); auto distinct_hash = BindScalarLocal(input, "hash", std::move(distinct_hash_children)); - rn->orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, std::move(distinct_hash)); - rn->start = WindowBoundary::UNBOUNDED_PRECEDING; - rn->end = WindowBoundary::CURRENT_ROW_ROWS; + rn->OrderByMutable().emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, std::move(distinct_hash)); + rn->WindowStartMutable() = WindowBoundary::UNBOUNDED_PRECEDING; + rn->WindowEndMutable() = WindowBoundary::CURRENT_ROW_ROWS; window->expressions.push_back(std::move(rn)); window->children.push_back(std::move(distinct_rows)); distinct_rows = std::move(window); @@ -2344,46 +2367,49 @@ static void RewriteDistinctCountToSampleMedian(OptimizerExtensionInput &input, L // A per-PU COUNT(DISTINCT) cannot be summed directly: two PUs in the same lane // may carry the same value. Regroup by (query group, lane, value) to produce one // row per lane-level distinct value, then count those rows into the m-lane list. - idx_t lane_group_index = binder.GenerateTableIndex(); - idx_t lane_agg_index = binder.GenerateTableIndex(); + idx_t lane_group_index = binder.GenerateTableIndex().index; + idx_t lane_agg_index = binder.GenerateTableIndex().index; vector> lane_expressions; if (apply_support_filter) { - auto rn_ref = make_uniq(LogicalType::BIGINT, - ColumnBinding(support_window_index.GetIndex(), 0)); - auto is_first = make_uniq(ExpressionType::COMPARE_EQUAL, std::move(rn_ref), - make_uniq(Value::BIGINT(1))); + auto rn_ref = make_uniq( + LogicalType::BIGINT, ColumnBinding(TableIndex(support_window_index.GetIndex()), ProjectionIndex(0))); + auto is_first = BoundComparisonExpression::Create(ExpressionType::COMPARE_EQUAL, std::move(rn_ref), + make_uniq(Value::BIGINT(1))); auto marker = make_uniq(LogicalType::BIGINT); BoundCaseCheck check; check.when_expr = std::move(is_first); check.then_expr = make_uniq(Value::BIGINT(1)); - marker->case_checks.push_back(std::move(check)); - marker->else_expr = make_uniq(Value::BIGINT(0)); + marker->CaseChecksMutable().push_back(std::move(check)); + marker->ElseMutable() = make_uniq(Value::BIGINT(0)); lane_expressions.push_back(BindPlainAggregate(input, "sum", std::move(marker))); } - auto lane_agg = make_uniq(lane_group_index, lane_agg_index, std::move(lane_expressions)); + auto lane_agg = make_uniq(TableIndex(lane_group_index), TableIndex(lane_agg_index), + std::move(lane_expressions)); for (idx_t gi = 0; gi < num_original_groups; gi++) { - lane_agg->groups.push_back(make_uniq(agg->groups[gi]->return_type, - ColumnBinding(inner_group_index, gi))); + lane_agg->groups.push_back(make_uniq( + agg->groups[gi]->GetReturnType(), ColumnBinding(TableIndex(inner_group_index), ProjectionIndex(gi)))); } auto lane_source = BuildRankSourceRef(inner_agg_ptr->types, inner_gi, pu_group_col); auto lane_mask = make_uniq(Value::UBIGINT(static_cast(GetDpSassM(input.context) - 1))); lane_agg->groups.push_back( input.optimizer.BindScalarFunction("&", std::move(lane_source), std::move(lane_mask))); - lane_agg->groups.push_back( - make_uniq(distinct_type, ColumnBinding(inner_group_index, distinct_col))); + lane_agg->groups.push_back(make_uniq( + distinct_type, ColumnBinding(TableIndex(inner_group_index), ProjectionIndex(distinct_col)))); lane_agg->children.push_back(std::move(distinct_rows)); lane_agg->ResolveOperatorTypes(); for (idx_t gi = 0; gi < num_original_groups; gi++) { - agg->groups[gi] = - make_uniq(lane_agg->types[gi], ColumnBinding(lane_group_index, gi)); + agg->groups[gi] = make_uniq( + lane_agg->types[gi], ColumnBinding(TableIndex(lane_group_index), ProjectionIndex(gi))); } - auto lane_ref = make_uniq(lane_agg->types[num_original_groups], - ColumnBinding(lane_group_index, num_original_groups)); + auto lane_ref = make_uniq( + lane_agg->types[num_original_groups], + ColumnBinding(TableIndex(lane_group_index), ProjectionIndex(num_original_groups))); auto distinct_ref = make_uniq( - lane_agg->types[num_original_groups + 1], ColumnBinding(lane_group_index, num_original_groups + 1)); + lane_agg->types[num_original_groups + 1], + ColumnBinding(TableIndex(lane_group_index), ProjectionIndex(num_original_groups + 1))); LogicalType support_type; if (apply_support_filter) { support_type = lane_agg->types[lane_agg->groups.size()]; @@ -2397,7 +2423,8 @@ static void RewriteDistinctCountToSampleMedian(OptimizerExtensionInput &input, L top_expressions.push_back( BindAggregateLocal(input, "as_sample_m_count_distinct_if", std::move(count_children))); if (apply_support_filter) { - auto support_ref = make_uniq(support_type, ColumnBinding(lane_agg_index, 0)); + auto support_ref = make_uniq( + support_type, ColumnBinding(TableIndex(lane_agg_index), ProjectionIndex(0))); top_expressions.push_back(BindPlainAggregate(input, "sum", std::move(support_ref))); } agg->expressions = std::move(top_expressions); @@ -2405,61 +2432,64 @@ static void RewriteDistinctCountToSampleMedian(OptimizerExtensionInput &input, L return; } - idx_t merge_group_index = binder.GenerateTableIndex(); - idx_t merge_agg_index = binder.GenerateTableIndex(); + idx_t merge_group_index = binder.GenerateTableIndex().index; + idx_t merge_agg_index = binder.GenerateTableIndex().index; vector> merge_expressions; - auto mask_ref = make_uniq(LogicalType::UBIGINT, ColumnBinding(inner_agg_index, 0)); + auto mask_ref = make_uniq(LogicalType::UBIGINT, + ColumnBinding(TableIndex(inner_agg_index), ProjectionIndex(0))); merge_expressions.push_back(BindPlainAggregate(input, "bit_or", std::move(mask_ref))); if (apply_support_filter) { - auto rn_ref = - make_uniq(LogicalType::BIGINT, ColumnBinding(support_window_index.GetIndex(), 0)); - auto is_first = make_uniq(ExpressionType::COMPARE_EQUAL, std::move(rn_ref), - make_uniq(Value::BIGINT(1))); + auto rn_ref = make_uniq( + LogicalType::BIGINT, ColumnBinding(TableIndex(support_window_index.GetIndex()), ProjectionIndex(0))); + auto is_first = BoundComparisonExpression::Create(ExpressionType::COMPARE_EQUAL, std::move(rn_ref), + make_uniq(Value::BIGINT(1))); auto marker = make_uniq(LogicalType::BIGINT); BoundCaseCheck check; check.when_expr = std::move(is_first); check.then_expr = make_uniq(Value::BIGINT(1)); - marker->case_checks.push_back(std::move(check)); - marker->else_expr = make_uniq(Value::BIGINT(0)); + marker->CaseChecksMutable().push_back(std::move(check)); + marker->ElseMutable() = make_uniq(Value::BIGINT(0)); merge_expressions.push_back(BindPlainAggregate(input, "sum", std::move(marker))); } - auto merge_agg = make_uniq(merge_group_index, merge_agg_index, std::move(merge_expressions)); + auto merge_agg = make_uniq(TableIndex(merge_group_index), TableIndex(merge_agg_index), + std::move(merge_expressions)); for (idx_t gi = 0; gi < num_original_groups; gi++) { - merge_agg->groups.push_back( - make_uniq(agg->groups[gi]->return_type, ColumnBinding(inner_group_index, gi))); + merge_agg->groups.push_back(make_uniq( + agg->groups[gi]->GetReturnType(), ColumnBinding(TableIndex(inner_group_index), ProjectionIndex(gi)))); } - merge_agg->groups.push_back( - make_uniq(distinct_type, ColumnBinding(inner_group_index, num_original_groups + 1))); + merge_agg->groups.push_back(make_uniq( + distinct_type, ColumnBinding(TableIndex(inner_group_index), ProjectionIndex(num_original_groups + 1)))); merge_agg->children.push_back(std::move(distinct_rows)); merge_agg->ResolveOperatorTypes(); - idx_t distinct_proj_index = binder.GenerateTableIndex(); + idx_t distinct_proj_index = binder.GenerateTableIndex().index; vector> distinct_proj_exprs; distinct_proj_exprs.reserve(num_original_groups + 1 + (apply_support_filter ? 1 : 0)); for (idx_t gi = 0; gi < num_original_groups; gi++) { - distinct_proj_exprs.push_back( - make_uniq(merge_agg->types[gi], ColumnBinding(merge_group_index, gi))); + distinct_proj_exprs.push_back(make_uniq( + merge_agg->types[gi], ColumnBinding(TableIndex(merge_group_index), ProjectionIndex(gi)))); } if (apply_support_filter) { - distinct_proj_exprs.push_back(make_uniq( - merge_agg->types[merge_agg->groups.size() + 1], ColumnBinding(merge_agg_index, 1))); + distinct_proj_exprs.push_back( + make_uniq(merge_agg->types[merge_agg->groups.size() + 1], + ColumnBinding(TableIndex(merge_agg_index), ProjectionIndex(1)))); } - distinct_proj_exprs.push_back(make_uniq(merge_agg->types[merge_agg->groups.size()], - ColumnBinding(merge_agg_index, 0))); - auto distinct_proj = make_uniq(distinct_proj_index, std::move(distinct_proj_exprs)); + distinct_proj_exprs.push_back(make_uniq( + merge_agg->types[merge_agg->groups.size()], ColumnBinding(TableIndex(merge_agg_index), ProjectionIndex(0)))); + auto distinct_proj = make_uniq(TableIndex(distinct_proj_index), std::move(distinct_proj_exprs)); distinct_proj->children.push_back(std::move(merge_agg)); distinct_proj->ResolveOperatorTypes(); unique_ptr final_input = std::move(distinct_proj); for (idx_t gi = 0; gi < num_original_groups; gi++) { - agg->groups[gi] = - make_uniq(final_input->types[gi], ColumnBinding(distinct_proj_index, gi)); + agg->groups[gi] = make_uniq( + final_input->types[gi], ColumnBinding(TableIndex(distinct_proj_index), ProjectionIndex(gi))); } idx_t support_proj_pos = num_original_groups; idx_t mask_proj_pos = apply_support_filter ? num_original_groups + 1 : num_original_groups; - auto merged_mask_ref = - make_uniq(LogicalType::UBIGINT, ColumnBinding(distinct_proj_index, mask_proj_pos)); + auto merged_mask_ref = make_uniq( + LogicalType::UBIGINT, ColumnBinding(TableIndex(distinct_proj_index), ProjectionIndex(mask_proj_pos))); agg->children[0] = std::move(final_input); vector> top_expressions; // expr[0] = the 64-counter mask, released via the smooth-sensitivity median. When partition @@ -2468,8 +2498,9 @@ static void RewriteDistinctCountToSampleMedian(OptimizerExtensionInput &input, L // the same projection-based path the generic aggregates use, so the noise is keyed on the group. top_expressions.push_back(BindPlainAggregate(input, "as_sample_count_mask", std::move(merged_mask_ref))); if (apply_support_filter) { - auto support_ref = make_uniq(agg->children[0]->types[support_proj_pos], - ColumnBinding(distinct_proj_index, support_proj_pos)); + auto support_ref = make_uniq( + agg->children[0]->types[support_proj_pos], + ColumnBinding(TableIndex(distinct_proj_index), ProjectionIndex(support_proj_pos))); top_expressions.push_back(BindPlainAggregate(input, "sum", std::move(support_ref))); } agg->expressions = std::move(top_expressions); @@ -2482,20 +2513,23 @@ static void RewritePuDistinctCountToSampleMedian(OptimizerExtensionInput &input, // PU rows, every remaining row contributes exactly one count to its own PU lane. auto &binder = input.optimizer.binder; auto &old_aggr = agg->expressions[0]->Cast(); - auto distinct_value = old_aggr.children[0]->Copy(); - auto distinct_type = distinct_value->return_type; + auto distinct_value = old_aggr.GetChildrenMutable()[0]->Copy(); + auto distinct_type = distinct_value->GetReturnType(); - idx_t inner_group_index = binder.GenerateTableIndex(); - idx_t inner_agg_index = binder.GenerateTableIndex(); + idx_t inner_group_index = binder.GenerateTableIndex().index; + idx_t inner_agg_index = binder.GenerateTableIndex().index; vector> inner_expressions; - auto inner_agg = make_uniq(inner_group_index, inner_agg_index, std::move(inner_expressions)); + auto inner_agg = make_uniq(TableIndex(inner_group_index), TableIndex(inner_agg_index), + std::move(inner_expressions)); inner_agg->groups.push_back(std::move(pu_hash_expr)); inner_agg->groups.push_back(std::move(distinct_value)); inner_agg->children.push_back(std::move(agg->children[0])); inner_agg->ResolveOperatorTypes(); - auto hash_ref = make_uniq(LogicalType::UBIGINT, ColumnBinding(inner_group_index, 0)); - auto value_ref = make_uniq(distinct_type, ColumnBinding(inner_group_index, 1)); + auto hash_ref = make_uniq( + LogicalType::UBIGINT, ColumnBinding(TableIndex(inner_group_index), ProjectionIndex(0))); + auto value_ref = make_uniq( + distinct_type, ColumnBinding(TableIndex(inner_group_index), ProjectionIndex(1))); vector> top_expressions; vector> count_children; count_children.push_back(std::move(hash_ref)); @@ -2511,11 +2545,11 @@ static void RewritePuDistinctCountToSampleMedian(OptimizerExtensionInput &input, } static void RemapExpressionBindingsLocal(Expression &e, const std::unordered_map &map) { - if (e.type == ExpressionType::BOUND_COLUMN_REF) { + if (e.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e.Cast(); - auto it = map.find(col_ref.binding.table_index); + auto it = map.find(col_ref.BindingMutable().table_index.index); if (it != map.end()) { - col_ref.binding.table_index = it->second; + col_ref.BindingMutable().table_index.index = it->second; } } ExpressionIterator::EnumerateChildren(e, [&](Expression &child) { RemapExpressionBindingsLocal(child, map); }); @@ -2532,10 +2566,10 @@ static unique_ptr CopyMaybeRemapped(const Expression &expr, static unique_ptr CopyFilterMaybeRemapped(const BoundAggregateExpression &aggr, const std::unordered_map *index_map) { - if (!aggr.filter) { + if (!aggr.GetFilter()) { return nullptr; } - return CopyMaybeRemapped(*aggr.filter, index_map); + return CopyMaybeRemapped(*aggr.GetFilter(), index_map); } struct SampleAggregateRewrite { @@ -2556,7 +2590,7 @@ static void BuildSampleMedianLowerExpression(OptimizerExtensionInput &input, con vector> &lower_expressions, idx_t aggregate_pos, vector &rewrites, bool variable_m, bool nullable_sum, const Expression *extra_filter = nullptr) { - string name = StringUtil::Lower(aggr.function.name); + string name = StringUtil::Lower(aggr.Function().GetName().GetIdentifierName()); string sample_sum_name = nullable_sum ? (variable_m ? "as_sample_m_nullable_sum" : "as_sample_nullable_sum") : (variable_m ? "as_sample_m_sum" : "as_sample_sum"); string sample_avg_name = variable_m ? "as_sample_m_avg" : "as_sample_avg"; @@ -2569,8 +2603,8 @@ static void BuildSampleMedianLowerExpression(OptimizerExtensionInput &input, con rewrites.push_back( {sample_sum_name, lower_value_pos, optional_idx(), true, 0.0, bounds.count_bound, false, 0.0, false}); } else if (name == "count") { - if (!aggr.children.empty()) { - children.push_back(CopyMaybeRemapped(*aggr.children[0], index_map)); + if (!aggr.GetChildren().empty()) { + children.push_back(CopyMaybeRemapped(*aggr.GetChildren()[0], index_map)); } if (variable_m && aggr.IsDistinct()) { lower_expressions.push_back( @@ -2580,7 +2614,7 @@ static void BuildSampleMedianLowerExpression(OptimizerExtensionInput &input, con if (extra_filter) { predicates.push_back(extra_filter->Copy()); } - lower_expressions.back()->Cast().filter = + lower_expressions.back()->Cast().GetFilterMutable() = ConjoinPredicates(std::move(predicates)); rewrites.push_back({"as_sample_m_count_distinct_values", lower_value_pos, optional_idx(), false, 0.0, 0.0, false, 0.0, true}); @@ -2593,23 +2627,23 @@ static void BuildSampleMedianLowerExpression(OptimizerExtensionInput &input, con {sample_sum_name, lower_value_pos, optional_idx(), true, 0.0, bounds.count_bound, false, 0.0, false}); } } else if (name == "sum") { - children.push_back(CopyMaybeRemapped(*aggr.children[0], index_map)); + children.push_back(CopyMaybeRemapped(*aggr.GetChildren()[0], index_map)); lower_expressions.push_back(BindPlainAggregate(input, "sum", std::move(children))); double sum_bound = GetDpSassSumContributionBound(bounds, aggregate_pos); rewrites.push_back( {sample_sum_name, lower_value_pos, optional_idx(), true, -sum_bound, sum_bound, false, 0.0, false}); } else if (name == "avg") { - auto sum_arg = CopyMaybeRemapped(*aggr.children[0], index_map); - auto count_arg = CopyMaybeRemapped(*aggr.children[0], index_map); + auto sum_arg = CopyMaybeRemapped(*aggr.GetChildren()[0], index_map); + auto count_arg = CopyMaybeRemapped(*aggr.GetChildren()[0], index_map); auto filter_for_sum = CopyFilterMaybeRemapped(aggr, index_map); auto filter_for_count = CopyFilterMaybeRemapped(aggr, index_map); lower_expressions.push_back(BindPlainAggregate(input, "sum", std::move(sum_arg))); - lower_expressions.back()->Cast().filter = std::move(filter_for_sum); + lower_expressions.back()->Cast().GetFilterMutable() = std::move(filter_for_sum); idx_t lower_count_pos = lower_expressions.size(); lower_expressions.push_back(BindPlainAggregate(input, "count", std::move(count_arg))); - lower_expressions.back()->Cast().filter = std::move(filter_for_count); + lower_expressions.back()->Cast().GetFilterMutable() = std::move(filter_for_count); rewrites.push_back({sample_avg_name, lower_value_pos, optional_idx(lower_count_pos), true, -bounds.sum_bound, bounds.sum_bound, true, bounds.count_bound, false}); return; @@ -2617,7 +2651,7 @@ static void BuildSampleMedianLowerExpression(OptimizerExtensionInput &input, con // Per-PU MIN/MAX of the value; per-lane sample MIN/MAX then median/mean release. No per-lane // magnitude clip — the release terminal already clamps each lane answer to the public // [dp_sass_minmax_lower_bound, dp_sass_minmax_upper_bound] output domain. - children.push_back(CopyMaybeRemapped(*aggr.children[0], index_map)); + children.push_back(CopyMaybeRemapped(*aggr.GetChildren()[0], index_map)); lower_expressions.push_back(BindPlainAggregate(input, name, std::move(children))); rewrites.push_back({name == "min" ? sample_min_name : sample_max_name, lower_value_pos, optional_idx(), false, 0.0, 0.0, false, 0.0, false}); @@ -2625,7 +2659,8 @@ static void BuildSampleMedianLowerExpression(OptimizerExtensionInput &input, con throw InvalidInputException("dp_sample_median: only COUNT, COUNT(DISTINCT), SUM, AVG, MIN, and MAX aggregates " "are supported"); } - lower_expressions.back()->Cast().filter = CopyFilterMaybeRemapped(aggr, index_map); + lower_expressions.back()->Cast().GetFilterMutable() = + CopyFilterMaybeRemapped(aggr, index_map); } static unique_ptr BuildSampleMedianAggregateRef(OptimizerExtensionInput &input, @@ -2633,7 +2668,7 @@ static unique_ptr BuildSampleMedianAggregateRef(OptimizerExtensionIn const SampleAggregateRewrite &rewrite) { auto lower_type = pre_agg.lower_agg->types[pre_agg.num_original_groups + 1 + rewrite.lower_value_pos]; unique_ptr lower_ref = make_uniq( - lower_type, ColumnBinding(pre_agg.lower_agg_index, rewrite.lower_value_pos)); + lower_type, ColumnBinding(TableIndex(pre_agg.lower_agg_index), ProjectionIndex(rewrite.lower_value_pos))); if (!rewrite.raw_lower_value) { lower_ref = BoundCastExpression::AddCastToType(input.context, std::move(lower_ref), LogicalType::DOUBLE); if (rewrite.clip_value) { @@ -2649,7 +2684,8 @@ static unique_ptr BuildSampleMedianAggregateRef(OptimizerExtensionIn auto lower_count_type = pre_agg.lower_agg->types[pre_agg.num_original_groups + 1 + rewrite.lower_count_pos.GetIndex()]; unique_ptr lower_count_ref = make_uniq( - lower_count_type, ColumnBinding(pre_agg.lower_agg_index, rewrite.lower_count_pos.GetIndex())); + lower_count_type, + ColumnBinding(TableIndex(pre_agg.lower_agg_index), ProjectionIndex(rewrite.lower_count_pos.GetIndex()))); auto count_ref = BoundCastExpression::AddCastToType(input.context, std::move(lower_count_ref), LogicalType::DOUBLE); if (rewrite.clip_count) { @@ -2692,7 +2728,7 @@ static void RewriteAggregateToSampleMedian(OptimizerExtensionInput &input, Logic bool has_variable_m_distinct = false; for (auto &expr : agg->expressions) { auto &aggr = expr->Cast(); - string name = StringUtil::Lower(aggr.function.name); + string name = StringUtil::Lower(aggr.FunctionMutable().GetName().GetIdentifierName()); if (name == "count" && aggr.IsDistinct()) { has_variable_m_distinct = true; break; @@ -2723,7 +2759,8 @@ static void RewriteAggregateToSampleMedian(OptimizerExtensionInput &input, Logic for (idx_t ai = 0; ai < agg->expressions.size(); ai++) { auto &rewrite = rewrites[ai]; auto &original = agg->expressions[ai]->Cast(); - PRIVACY_DEBUG_PRINT("[DP_SAMPLE_MEDIAN] rewrite " + original.function.name + " -> " + rewrite.sample_name); + PRIVACY_DEBUG_PRINT("[DP_SAMPLE_MEDIAN] rewrite " + original.Function().GetName().GetIdentifierName() + " -> " + + rewrite.sample_name); agg->expressions[ai] = BuildSampleMedianAggregateRef(input, pre_agg, rewrite); } agg->ResolveOperatorTypes(); @@ -2770,16 +2807,16 @@ BuildStableDpSassPuHashExpression(OptimizerExtensionInput &input, unique_ptrGetColumnIds()[proj_idx]; auto col_type = key_get->GetColumnType(col_index); - ColumnBinding source_binding(key_get->table_index, proj_idx); - auto propagated = PropagateSingleBinding(*plan, key_get->table_index, source_binding, col_type, agg); - if (propagated.table_index == DConstants::INVALID_INDEX) { + ColumnBinding source_binding {TableIndex(key_get->table_index.index), ProjectionIndex(proj_idx)}; + auto propagated = PropagateSingleBinding(*plan, key_get->table_index.index, source_binding, col_type, agg); + if (propagated.table_index.index == DConstants::INVALID_INDEX) { return nullptr; } key_exprs.push_back(make_uniq(col_type, propagated)); } auto pu_hash_expr = BuildXorHash(input, std::move(key_exprs)); pu_hash_expr = BoundCastExpression::AddCastToType(input.context, std::move(pu_hash_expr), LogicalType::UBIGINT); - pu_hash_expr->alias = "dp_sass_lane_key"; + pu_hash_expr->SetAlias(Identifier("dp_sass_lane_key")); PRIVACY_DEBUG_PRINT("[DP_SAMPLE_MEDIAN] using stable hash-based sample lanes"); return pu_hash_expr; } @@ -3037,8 +3074,8 @@ static unique_ptr BuildFilterlessSamplePredicate(OptimizerExtensionI } auto shift = make_uniq(Value::INTEGER(64 - sample_bits)); auto prefix = input.optimizer.BindScalarFunction(">>", pu_hash.Copy(), std::move(shift)); - return make_uniq(ExpressionType::COMPARE_EQUAL, std::move(prefix), - make_uniq(Value::UBIGINT(0))); + return BoundComparisonExpression::Create(ExpressionType::COMPARE_EQUAL, std::move(prefix), + make_uniq(Value::UBIGINT(0))); } static unique_ptr BuildFilterlessEncodedPu(OptimizerExtensionInput &input, const Expression &pu_hash, @@ -3051,11 +3088,11 @@ static unique_ptr BuildFilterlessEncodedPu(OptimizerExtensionInput & BoundCaseCheck active_check; active_check.when_expr = std::move(active); active_check.then_expr = make_uniq(Value::BOOLEAN(true)); - active_value->case_checks.push_back(std::move(active_check)); - active_value->else_expr = make_uniq(Value::BOOLEAN(false)); + active_value->CaseChecksMutable().push_back(std::move(active_check)); + active_value->ElseMutable() = make_uniq(Value::BOOLEAN(false)); auto active_bit = BoundCastExpression::AddCastToType(input.context, std::move(active_value), LogicalType::UBIGINT); auto encoded = input.optimizer.BindScalarFunction("|", std::move(logical_pu), std::move(active_bit)); - encoded->alias = "dp_filterless_encoded_pu"; + encoded->SetAlias(Identifier("dp_filterless_encoded_pu")); return encoded; } @@ -3078,8 +3115,8 @@ static vector InsertFilterlessProjection(OptimizerExtensionInput expressions.push_back(std::move(expression)); } - idx_t projection_index = input.optimizer.binder.GenerateTableIndex(); - auto projection = make_uniq(projection_index, std::move(expressions)); + idx_t projection_index = input.optimizer.binder.GenerateTableIndex().index; + auto projection = make_uniq(TableIndex(projection_index), std::move(expressions)); auto *slot = FindOperatorSlotByPointer(plan, insert_above); if (!slot) { throw InternalException("dp_filterless: could not locate encoded-PU projection point"); @@ -3091,14 +3128,15 @@ static vector InsertFilterlessProjection(OptimizerExtensionInput ColumnBindingReplacer replacer; for (idx_t i = 0; i < old_bindings.size(); i++) { - replacer.replacement_bindings.emplace_back(old_bindings[i], ColumnBinding(projection_index, i), old_types[i]); + replacer.replacement_bindings.emplace_back( + old_bindings[i], ColumnBinding(TableIndex(projection_index), ProjectionIndex(i)), old_types[i]); } replacer.stop_operator = projection_ptr; replacer.VisitOperator(*plan); vector result; result.reserve(extra_expressions.size()); for (idx_t i = 0; i < extra_expressions.size(); i++) { - result.emplace_back(projection_index, old_bindings.size() + i); + result.emplace_back(TableIndex(projection_index), ProjectionIndex(old_bindings.size() + i)); } return result; } @@ -3159,8 +3197,8 @@ static unique_ptr BuildFilterlessEncodedPuInput(OptimizerExtensionIn auto encoded_bindings = InsertFilterlessProjection(input, plan, projection_child, std::move(encoded_expression)); auto encoded_binding = encoded_bindings[0]; auto propagated = - PropagateSingleBinding(*plan, encoded_binding.table_index, encoded_binding, LogicalType::UBIGINT, agg); - if (propagated.table_index == DConstants::INVALID_INDEX) { + PropagateSingleBinding(*plan, encoded_binding.table_index.index, encoded_binding, LogicalType::UBIGINT, agg); + if (propagated.table_index.index == DConstants::INVALID_INDEX) { throw InternalException("dp_filterless: could not propagate encoded PU to aggregate input"); } return make_uniq(LogicalType::UBIGINT, propagated); @@ -3176,16 +3214,19 @@ struct FilterlessPreAggregationInput { vector aggregate_types; unique_ptr GroupRef(idx_t index) const { - return make_uniq(group_types[index], ColumnBinding(group_table_index, index)); + return make_uniq( + group_types[index], ColumnBinding(TableIndex(group_table_index), ProjectionIndex(index))); } unique_ptr PuRef() const { - return make_uniq(LogicalType::UBIGINT, ColumnBinding(group_table_index, group_count)); + return make_uniq( + LogicalType::UBIGINT, ColumnBinding(TableIndex(group_table_index), ProjectionIndex(group_count))); } unique_ptr AggregateRef(idx_t index) const { return make_uniq( - aggregate_types[index], ColumnBinding(aggregate_table_index, component_column_offset + index)); + aggregate_types[index], + ColumnBinding(TableIndex(aggregate_table_index), ProjectionIndex(component_column_offset + index))); } unique_ptr ComponentRef(OptimizerExtensionInput &input, idx_t index, bool answer) const { @@ -3211,7 +3252,7 @@ struct FilterlessPreAggregationInput { static FilterlessPreAggregationInput BuildFilterlessPreAggregationInput(const PuPreAggregationInfo &pre, idx_t component_count) { FilterlessPreAggregationInput result; - result.group_table_index = pre.lower_agg->group_index; + result.group_table_index = pre.lower_agg->group_index.index; result.aggregate_table_index = pre.lower_agg_index; result.group_count = pre.num_original_groups; result.component_column_offset = 0; @@ -3237,18 +3278,19 @@ static FilterlessPreAggregationInput ApplyFilterlessMaxGroups(OptimizerExtension // caps the group set without needing duplicate-aware DENSE_RANK. DuckDB can optimize this // single hashed ordering into a bounded per-PU top-k aggregate. idx_t lower_aggregate_count = component_count + 1; - idx_t projection_index = input.optimizer.binder.GenerateTableIndex(); + idx_t projection_index = input.optimizer.binder.GenerateTableIndex().index; vector> expressions; expressions.reserve(pre.num_original_groups + 1 + lower_aggregate_count); for (idx_t i = 0; i < pre.num_original_groups + 1; i++) { - expressions.push_back( - make_uniq(pre.lower_agg->types[i], ColumnBinding(pre.lower_agg->group_index, i))); + expressions.push_back(make_uniq( + pre.lower_agg->types[i], ColumnBinding(TableIndex(pre.lower_agg->group_index.index), ProjectionIndex(i)))); } for (idx_t i = 0; i < lower_aggregate_count; i++) { - expressions.push_back(make_uniq(pre.lower_agg->types[pre.num_original_groups + 1 + i], - ColumnBinding(pre.lower_agg_index, i))); + expressions.push_back( + make_uniq(pre.lower_agg->types[pre.num_original_groups + 1 + i], + ColumnBinding(TableIndex(pre.lower_agg_index), ProjectionIndex(i)))); } - auto projection = make_uniq(projection_index, std::move(expressions)); + auto projection = make_uniq(TableIndex(projection_index), std::move(expressions)); projection->children.push_back(std::move(agg->children[0])); projection->ResolveOperatorTypes(); auto projection_types = projection->types; @@ -3271,8 +3313,8 @@ static FilterlessPreAggregationInput ApplyFilterlessMaxGroups(OptimizerExtension agg->children[0]->ResolveOperatorTypes(); for (idx_t i = 0; i < pre.num_original_groups; i++) { - agg->groups[i] = - make_uniq(pre.lower_agg->types[i], ColumnBinding(projection_index, i)); + agg->groups[i] = make_uniq( + pre.lower_agg->types[i], ColumnBinding(TableIndex(projection_index), ProjectionIndex(i))); } FilterlessPreAggregationInput result; result.group_table_index = projection_index; @@ -3294,22 +3336,22 @@ static FilterlessPreAggregationInput ApplyFilterlessMaxGroups(OptimizerExtension static unique_ptr BuildFilterlessActiveSupportKey(OptimizerExtensionInput &input, const FilterlessPreAggregationInput &pre_input) { auto active_count = pre_input.ActiveCountRef(); - auto is_active = make_uniq(ExpressionType::COMPARE_GREATERTHAN, std::move(active_count), - make_uniq(Value::BIGINT(0))); + auto is_active = BoundComparisonExpression::Create(ExpressionType::COMPARE_GREATERTHAN, std::move(active_count), + make_uniq(Value::BIGINT(0))); auto active_pu = make_uniq(LogicalType::UBIGINT); BoundCaseCheck check; check.when_expr = std::move(is_active); check.then_expr = pre_input.PuRef(); - active_pu->case_checks.push_back(std::move(check)); - active_pu->else_expr = make_uniq(Value(LogicalType::UBIGINT)); + active_pu->CaseChecksMutable().push_back(std::move(check)); + active_pu->ElseMutable() = make_uniq(Value(LogicalType::UBIGINT)); return active_pu; } static unique_ptr BuildFilterlessIsActive(OptimizerExtensionInput &input, const Expression &encoded_pu) { auto active_bit = input.optimizer.BindScalarFunction("&", encoded_pu.Copy(), make_uniq(Value::UBIGINT(1))); - return make_uniq(ExpressionType::COMPARE_NOTEQUAL, std::move(active_bit), - make_uniq(Value::UBIGINT(0))); + return BoundComparisonExpression::Create(ExpressionType::COMPARE_NOTEQUAL, std::move(active_bit), + make_uniq(Value::UBIGINT(0))); } static unique_ptr BuildFilterlessLogicalPu(OptimizerExtensionInput &input, @@ -3323,13 +3365,13 @@ static unique_ptr BuildFilterlessLowerPair(OptimizerExtensionInput & const Expression &encoded_pu, int sample_bits) { vector> children; if (is_count) { - if (aggregate.function.name == "count" && !aggregate.children.empty()) { - children.push_back(BuildIsNotNullPredicate(aggregate.children[0]->Copy())); + if (aggregate.Function().GetName().GetIdentifierName() == "count" && !aggregate.GetChildren().empty()) { + children.push_back(BuildIsNotNullPredicate(aggregate.GetChildren()[0]->Copy())); } else { children.push_back(make_uniq(Value::BOOLEAN(true))); } } else { - children.push_back(aggregate.children[0]->Copy()); + children.push_back(aggregate.GetChildren()[0]->Copy()); } children.push_back(BuildFilterlessIsActive(input, encoded_pu)); children.push_back(BuildFilterlessSamplePredicate(input, encoded_pu, sample_bits)); @@ -3348,7 +3390,7 @@ void CompileDPFilterlessQuery(const PrivacyCompatibilityResult &check, Optimizer idx_t n_groups = agg->groups.size(); for (auto &expression : agg->expressions) { auto &aggregate = expression->Cast(); - if (aggregate.filter) { + if (aggregate.GetFilterMutable()) { throw NotImplementedException("dp_filterless: aggregate FILTER clauses are not implemented yet"); } } @@ -3408,7 +3450,7 @@ void CompileDPFilterlessQuery(const PrivacyCompatibilityResult &check, Optimizer BuildFilterlessLowerPair(input, aggregate, is_count, *encoded_pu, filterless_settings.sample_bits)); } auto active_count = BindPlainAggregate(input, "count_star", nullptr); - active_count->Cast().filter = BuildFilterlessIsActive(input, *encoded_pu); + active_count->Cast().GetFilterMutable() = BuildFilterlessIsActive(input, *encoded_pu); lower_expressions.push_back(std::move(active_count)); auto logical_pu = BuildFilterlessLogicalPu(input, std::move(encoded_pu)); @@ -3421,8 +3463,8 @@ void CompileDPFilterlessQuery(const PrivacyCompatibilityResult &check, Optimizer upper_expressions.reserve(original_aggregate_count + (n_groups > 0 ? 1 : 0)); for (idx_t i = 0; i < original_aggregate_count; i++) { auto is_active = - make_uniq(ExpressionType::COMPARE_GREATERTHAN, pre_input.ActiveCountRef(), - make_uniq(Value::BIGINT(0))); + BoundComparisonExpression::Create(ExpressionType::COMPARE_GREATERTHAN, pre_input.ActiveCountRef(), + make_uniq(Value::BIGINT(0))); vector> children; children.push_back(pre_input.PuRef()); children.push_back(std::move(is_active)); @@ -3649,7 +3691,7 @@ void CompileDPSampleMedianQuery(const PrivacyCompatibilityResult &check, Optimiz idx_t sum_count = 0; for (idx_t ai = 0; ai < agg->expressions.size(); ai++) { auto &aggr = agg->expressions[ai]->Cast(); - string name = StringUtil::Lower(aggr.function.name); + string name = StringUtil::Lower(aggr.FunctionMutable().GetName().GetIdentifierName()); if (avg_components.count(ai)) { continue; } @@ -3665,9 +3707,9 @@ void CompileDPSampleMedianQuery(const PrivacyCompatibilityResult &check, Optimiz idx_t sum_output_idx = 0; for (idx_t ai = 0; ai < agg->expressions.size(); ai++) { auto &aggr = agg->expressions[ai]->Cast(); - string name = StringUtil::Lower(aggr.function.name); + string name = StringUtil::Lower(aggr.FunctionMutable().GetName().GetIdentifierName()); bool avg_sum_component = FindAvgInfoForSumPos(avg_infos, ai) != nullptr; - output_types.push_back(name == "avg" ? LogicalType::DOUBLE : aggr.return_type); + output_types.push_back(name == "avg" ? LogicalType::DOUBLE : aggr.GetReturnType()); if (avg_sum_component) { double output_bound = GetRequiredDpBound(input.context, "dp_sass_sum_output_bound", "dp_sass"); lower_bounds.push_back(-output_bound); diff --git a/src/core/privacy_extension.cpp b/src/core/privacy_extension.cpp index b4258c7e..0afc2849 100644 --- a/src/core/privacy_extension.cpp +++ b/src/core/privacy_extension.cpp @@ -2,6 +2,7 @@ #include "privacy_extension.hpp" #include "duckdb.hpp" +#include "duckdb/main/database_manager.hpp" #include "duckdb/common/exception.hpp" #include "duckdb/function/scalar_function.hpp" #include @@ -92,7 +93,7 @@ static string QuoteIdentifier(const string &identifier) { static void SetExtensionSetting(ClientContext &context, const string &name, const Value &value) { auto &config = DBConfig::GetConfig(context); ExtensionOption extension_option; - if (!config.TryGetExtensionOption(name, extension_option)) { + if (!config.TryGetExtensionOption(Identifier(name), extension_option)) { throw InternalException("refresh_dp_stats: extension setting '" + name + "' was not registered"); } auto target_value = value.CastAs(context, extension_option.type); diff --git a/src/core/privacy_optimizer.cpp b/src/core/privacy_optimizer.cpp index 2670ccab..b1fbb04c 100644 --- a/src/core/privacy_optimizer.cpp +++ b/src/core/privacy_optimizer.cpp @@ -22,7 +22,7 @@ #include "duckdb/optimizer/optimizer.hpp" // Include DuckDB headers for DROP operation handling #include "duckdb/parser/parsed_data/drop_info.hpp" -#include "duckdb/planner/operator/logical_simple.hpp" +#include "duckdb/planner/operator/logical_drop.hpp" // Include DuckDB headers for CTAS metadata propagation #include "duckdb/planner/operator/logical_create_table.hpp" #include "duckdb/planner/operator/logical_get.hpp" @@ -47,7 +47,7 @@ void PACDropTableRule::PACDropTableRuleFunction(OptimizerExtensionInput &input, return; } // Cast to LogicalSimple to access the parse info (DROP operations use LogicalSimple) - auto &simple = plan->Cast(); + auto &simple = plan->Cast(); if (simple.info->info_type != ParseInfoType::DROP_INFO) { return; } @@ -56,7 +56,7 @@ void PACDropTableRule::PACDropTableRuleFunction(OptimizerExtensionInput &input, if (drop_info.type != CatalogType::TABLE_ENTRY) { return; // Only handle DROP TABLE operations } - string table_name = drop_info.name; + string table_name = drop_info.GetQualifiedName().Name().GetIdentifierName(); // Check if this table has PAC metadata auto &metadata_mgr = PrivacyMetadataManager::Get(); @@ -140,7 +140,7 @@ void PACDropTableRule::PACDropTableRuleFunction(OptimizerExtensionInput &input, static void PropagateCTASMetadata(unique_ptr &outer_plan, unique_ptr &select_plan, OptimizerExtensionInput &input) { auto &create_table = outer_plan->Cast(); - string new_table = create_table.info->Base().table; + string new_table = create_table.info->Base().GetQualifiedName().Name().GetIdentifierName(); auto &mgr = PrivacyMetadataManager::Get(); if (mgr.HasMetadata(new_table)) { return; @@ -167,7 +167,7 @@ static void PropagateCTASMetadata(unique_ptr &outer_plan, uniqu if (!entry) { continue; } - auto *meta = mgr.GetTableMetadata(entry->name); + auto *meta = mgr.GetTableMetadata(entry->name.GetIdentifierName()); if (meta && (meta->is_privacy_unit || meta->derived_pu)) { source_meta = meta; source_get = &get; @@ -187,7 +187,7 @@ static void PropagateCTASMetadata(unique_ptr &outer_plan, uniqu std::unordered_map oid_to_name; if (entry) { for (auto &col : entry->GetColumns().Logical()) { - oid_to_name[col.Oid()] = col.Name(); + oid_to_name[col.Oid()] = col.Name().GetIdentifierName(); } } @@ -215,7 +215,7 @@ static void PropagateCTASMetadata(unique_ptr &outer_plan, uniqu for (auto &new_col : new_columns.Logical()) { auto name_it = get_col_idx_to_name.find(dest_idx); if (name_it != get_col_idx_to_name.end()) { - src_to_dest_name[name_it->second] = new_col.Name(); + src_to_dest_name[name_it->second] = new_col.Name().GetIdentifierName(); } dest_idx++; } @@ -247,7 +247,7 @@ static void PropagateCTASMetadata(unique_ptr &outer_plan, uniqu // fall back to a simple name check against the output column names. std::unordered_set new_col_names; for (auto &new_col : new_columns.Logical()) { - new_col_names.insert(new_col.Name()); + new_col_names.insert(new_col.Name().GetIdentifierName()); } for (auto &prot : source_meta->protected_columns) { if (new_col_names.count(prot) > 0) { @@ -495,7 +495,7 @@ void PACRewriteRule::PACPreOptimizeFunction(OptimizerExtensionInput &input, uniq bool do_diff = false; { Value val; - if (input.context.TryGetCurrentSetting("privacy_diffcols", val) && !val.IsNull()) { + if (input.context.TryGetCurrentSetting(Identifier("privacy_diffcols"), val) && !val.IsNull()) { string val_str = val.ToString(); if (!val_str.empty()) { if (std::isdigit(val_str[0])) { @@ -541,10 +541,10 @@ void PACRewriteRule::PACPreOptimizeFunction(OptimizerExtensionInput &input, uniq string target_table; if (outer_plan->type == LogicalOperatorType::LOGICAL_CREATE_TABLE) { auto &create = outer_plan->Cast(); - target_table = create.info->Base().table; + target_table = create.info->Base().GetQualifiedName().Name().GetIdentifierName(); } else if (outer_plan->type == LogicalOperatorType::LOGICAL_INSERT) { auto &insert = outer_plan->Cast(); - target_table = insert.table.name; + target_table = insert.table.name.GetIdentifierName(); } if (!target_table.empty()) { auto &mgr = PrivacyMetadataManager::Get(); @@ -570,7 +570,7 @@ void PACRewriteRule::PACPreOptimizeFunction(OptimizerExtensionInput &input, uniq col.SetType(plan_types[i]); } if (plan_types[i] == counter_list_type && mut_meta) { - mut_meta->counter_columns.push_back(col.Name()); + mut_meta->counter_columns.push_back(col.Name().GetIdentifierName()); } } } diff --git a/src/diff/pac_utility_diff.cpp b/src/diff/pac_utility_diff.cpp index fa3f0ae4..524deb60 100644 --- a/src/diff/pac_utility_diff.cpp +++ b/src/diff/pac_utility_diff.cpp @@ -60,14 +60,13 @@ static bool IsNumericType(const LogicalType &type) { } // Create a ROW_NUMBER() OVER () window expression for positional matching. -static unique_ptr AddRowNumber(unique_ptr child, idx_t table_index, - ColumnBinding &rn_binding_out) { - auto window = make_uniq(table_index); +static unique_ptr AddRowNumber(ClientContext &context, unique_ptr child, + idx_t table_index, ColumnBinding &rn_binding_out) { + auto window = make_uniq(TableIndex(table_index)); - auto row_number = - make_uniq(ExpressionType::WINDOW_ROW_NUMBER, LogicalType::BIGINT, nullptr, nullptr); + auto row_number = BindRankingWindow(context, ExpressionType::WINDOW_ROW_NUMBER); - rn_binding_out = ColumnBinding(table_index, 0); + rn_binding_out = ColumnBinding(TableIndex(table_index), ProjectionIndex(0)); window->expressions.push_back(std::move(row_number)); window->children.push_back(std::move(child)); @@ -77,7 +76,7 @@ static unique_ptr AddRowNumber(unique_ptr chil // Create an IS NULL check expression. static unique_ptr MakeIsNull(unique_ptr child) { auto op = make_uniq(ExpressionType::OPERATOR_IS_NULL, LogicalType::BOOLEAN); - op->children.push_back(std::move(child)); + op->GetChildrenMutable().push_back(std::move(child)); return std::move(op); } @@ -95,7 +94,8 @@ static vector ExtractColumnNames(LogicalOperator *node) { if (node && node->type == LogicalOperatorType::LOGICAL_PROJECTION) { auto &proj = node->Cast(); for (auto &expr : proj.expressions) { - names.push_back(expr->alias.empty() ? expr->GetName() : expr->alias); + names.push_back(expr->GetAlias().empty() ? expr->GetName().GetIdentifierName() + : expr->GetAlias().GetIdentifierName()); } } return names; @@ -153,13 +153,14 @@ void ApplyUtilityDiff(OptimizerExtensionInput &input, unique_ptrCast(); auto child_bindings = node->children[0]->GetColumnBindings(); for (auto &o : order_op.orders) { - if (o.expression->type == ExpressionType::BOUND_COLUMN_REF) { + if (o.expression->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &ref = o.expression->Cast(); for (idx_t i = 0; i < child_bindings.size(); i++) { - if (child_bindings[i] == ref.binding) { + if (child_bindings[i] == ref.BindingMutable()) { saved_orders.emplace_back( o.type, o.null_order, - make_uniq(ref.return_type, ColumnBinding(0, i))); + make_uniq(ref.GetReturnType(), + ColumnBinding(TableIndex(0), ProjectionIndex(i)))); break; } } @@ -217,7 +218,7 @@ void ApplyUtilityDiff(OptimizerExtensionInput &input, unique_ptr(col_types[i], bindings[i])); } - auto proj = make_uniq(binder.GenerateTableIndex(), std::move(exprs)); + auto proj = make_uniq(TableIndex(binder.GenerateTableIndex().index), std::move(exprs)); proj->children.push_back(std::move(child)); return std::move(proj); }; @@ -231,25 +232,25 @@ void ApplyUtilityDiff(OptimizerExtensionInput &input, unique_ptrGetColumnBindings(); if (num_key_cols == 0) { - pac_side = AddRowNumber(std::move(pac_side), binder.GenerateTableIndex(), pac_rn_binding); - ref_side = AddRowNumber(std::move(ref_side), binder.GenerateTableIndex(), ref_rn_binding); + pac_side = AddRowNumber(input.context, std::move(pac_side), binder.GenerateTableIndex().index, pac_rn_binding); + ref_side = AddRowNumber(input.context, std::move(ref_side), binder.GenerateTableIndex().index, ref_rn_binding); } // ---- Step 2: Create the FULL OUTER JOIN ---- auto join = make_uniq(JoinType::OUTER); if (num_key_cols == 0) { - JoinCondition cond; - cond.comparison = ExpressionType::COMPARE_EQUAL; - cond.left = make_uniq(LogicalType::BIGINT, pac_rn_binding); - cond.right = make_uniq(LogicalType::BIGINT, ref_rn_binding); + JoinCondition cond(nullptr, nullptr, ExpressionType::COMPARE_EQUAL); + + cond.LeftReference() = make_uniq(LogicalType::BIGINT, pac_rn_binding); + cond.RightReference() = make_uniq(LogicalType::BIGINT, ref_rn_binding); join->conditions.push_back(std::move(cond)); } else { for (idx_t i = 0; i < num_key_cols; i++) { - JoinCondition cond; - cond.comparison = ExpressionType::COMPARE_EQUAL; - cond.left = make_uniq(col_types[i], pac_bindings[i]); - cond.right = make_uniq(col_types[i], ref_bindings[i]); + JoinCondition cond(nullptr, nullptr, ExpressionType::COMPARE_EQUAL); + + cond.LeftReference() = make_uniq(col_types[i], pac_bindings[i]); + cond.RightReference() = make_uniq(col_types[i], ref_bindings[i]); join->conditions.push_back(std::move(cond)); } } @@ -283,16 +284,16 @@ void ApplyUtilityDiff(OptimizerExtensionInput &input, unique_ptr(probe_type, pac_probe_binding)); check.then_expr = make_uniq(Value(col_types[i])); - case_expr->case_checks.push_back(std::move(check)); + case_expr->CaseChecksMutable().push_back(std::move(check)); } { BoundCaseCheck check; check.when_expr = MakeIsNull(make_uniq(probe_type, ref_probe_binding)); check.then_expr = make_uniq(col_types[i], pac_bindings[i]); - case_expr->case_checks.push_back(std::move(check)); + case_expr->CaseChecksMutable().push_back(std::move(check)); } - case_expr->else_expr = make_uniq(col_types[i], ref_bindings[i]); - case_expr->alias = col_names[i]; + case_expr->ElseMutable() = make_uniq(col_types[i], ref_bindings[i]); + case_expr->SetAlias(Identifier(col_names[i])); new_expressions.push_back(std::move(case_expr)); } else if (is_numeric) { // Numeric measure columns: @@ -318,7 +319,7 @@ void ApplyUtilityDiff(OptimizerExtensionInput &input, unique_ptr(probe_type, ref_probe_binding)); check.then_expr = make_uniq(Value(col_types[i])); - case_expr->case_checks.push_back(std::move(check)); + case_expr->CaseChecksMutable().push_back(std::move(check)); } // "-" row → 0 (missing from PAC result = zero utility) @@ -327,7 +328,7 @@ void ApplyUtilityDiff(OptimizerExtensionInput &input, unique_ptr(probe_type, pac_probe_binding)); check.then_expr = BoundCastExpression::AddDefaultCastToType( make_uniq(Value::DOUBLE(0.0)), col_types[i]); - case_expr->case_checks.push_back(std::move(check)); + case_expr->CaseChecksMutable().push_back(std::move(check)); } // Both NULL → 0 (perfect match) @@ -339,7 +340,7 @@ void ApplyUtilityDiff(OptimizerExtensionInput &input, unique_ptr(Value::DOUBLE(0.0)), col_types[i]); - case_expr->case_checks.push_back(std::move(check)); + case_expr->CaseChecksMutable().push_back(std::move(check)); } // One NULL → 50 @@ -351,7 +352,7 @@ void ApplyUtilityDiff(OptimizerExtensionInput &input, unique_ptr(Value::DOUBLE(50.0)), col_types[i]); - case_expr->case_checks.push_back(std::move(check)); + case_expr->CaseChecksMutable().push_back(std::move(check)); } // ELSE: CAST(100 * |ref - pac| / max(0.00001, |ref|) AS original_type) @@ -360,12 +361,12 @@ void ApplyUtilityDiff(OptimizerExtensionInput &input, unique_ptr(LogicalType::DOUBLE); { BoundCaseCheck check; - check.when_expr = make_uniq(ExpressionType::COMPARE_GREATERTHANOREQUALTO, - make_ref_dbl(), make_pac_dbl()); + check.when_expr = BoundComparisonExpression::Create(ExpressionType::COMPARE_GREATERTHANOREQUALTO, + make_ref_dbl(), make_pac_dbl()); check.then_expr = optimizer.BindScalarFunction("-", make_ref_dbl(), make_pac_dbl()); - abs_diff->case_checks.push_back(std::move(check)); + abs_diff->CaseChecksMutable().push_back(std::move(check)); } - abs_diff->else_expr = optimizer.BindScalarFunction("-", make_pac_dbl(), make_ref_dbl()); + abs_diff->ElseMutable() = optimizer.BindScalarFunction("-", make_pac_dbl(), make_ref_dbl()); // max(0.00001, |ref|): // CASE WHEN ref >= 0.00001 THEN ref @@ -374,32 +375,32 @@ void ApplyUtilityDiff(OptimizerExtensionInput &input, unique_ptr(LogicalType::DOUBLE); { BoundCaseCheck check; - check.when_expr = make_uniq( - ExpressionType::COMPARE_GREATERTHANOREQUALTO, make_ref_dbl(), - make_uniq(Value::DOUBLE(0.00001))); + check.when_expr = + BoundComparisonExpression::Create(ExpressionType::COMPARE_GREATERTHANOREQUALTO, make_ref_dbl(), + make_uniq(Value::DOUBLE(0.00001))); check.then_expr = make_ref_dbl(); - denom->case_checks.push_back(std::move(check)); + denom->CaseChecksMutable().push_back(std::move(check)); } { BoundCaseCheck check; - check.when_expr = make_uniq( - ExpressionType::COMPARE_LESSTHANOREQUALTO, make_ref_dbl(), - make_uniq(Value::DOUBLE(-0.00001))); + check.when_expr = + BoundComparisonExpression::Create(ExpressionType::COMPARE_LESSTHANOREQUALTO, make_ref_dbl(), + make_uniq(Value::DOUBLE(-0.00001))); check.then_expr = optimizer.BindScalarFunction( "-", make_uniq(Value::DOUBLE(0.0)), make_ref_dbl()); - denom->case_checks.push_back(std::move(check)); + denom->CaseChecksMutable().push_back(std::move(check)); } - denom->else_expr = make_uniq(Value::DOUBLE(0.00001)); + denom->ElseMutable() = make_uniq(Value::DOUBLE(0.00001)); // 100 * abs_diff / denom auto hundred = make_uniq(Value::DOUBLE(100.0)); auto numerator = optimizer.BindScalarFunction("*", std::move(hundred), std::move(abs_diff)); auto result = optimizer.BindScalarFunction("/", std::move(numerator), std::move(denom)); - case_expr->else_expr = BoundCastExpression::AddDefaultCastToType(std::move(result), col_types[i]); + case_expr->ElseMutable() = BoundCastExpression::AddDefaultCastToType(std::move(result), col_types[i]); } - case_expr->alias = col_names[i]; + case_expr->SetAlias(Identifier(col_names[i])); new_expressions.push_back(std::move(case_expr)); } else { // Non-numeric, non-key columns: @@ -411,35 +412,37 @@ void ApplyUtilityDiff(OptimizerExtensionInput &input, unique_ptr(probe_type, ref_probe_binding)); check.then_expr = make_uniq(Value(col_types[i])); - case_expr->case_checks.push_back(std::move(check)); + case_expr->CaseChecksMutable().push_back(std::move(check)); } { BoundCaseCheck check; check.when_expr = MakeIsNull(make_uniq(probe_type, pac_probe_binding)); check.then_expr = make_uniq(Value(col_types[i])); - case_expr->case_checks.push_back(std::move(check)); + case_expr->CaseChecksMutable().push_back(std::move(check)); } auto coalesce = make_uniq(ExpressionType::OPERATOR_COALESCE, col_types[i]); - coalesce->children.push_back(make_uniq(col_types[i], pac_bindings[i])); - coalesce->children.push_back(make_uniq(col_types[i], ref_bindings[i])); - case_expr->else_expr = std::move(coalesce); - case_expr->alias = col_names[i]; + coalesce->GetChildrenMutable().push_back( + make_uniq(col_types[i], pac_bindings[i])); + coalesce->GetChildrenMutable().push_back( + make_uniq(col_types[i], ref_bindings[i])); + case_expr->ElseMutable() = std::move(coalesce); + case_expr->SetAlias(Identifier(col_names[i])); new_expressions.push_back(std::move(case_expr)); } } // ---- Step 4: Create a new top-level diff projection ---- - idx_t diff_proj_idx = binder.GenerateTableIndex(); - plan = make_uniq(diff_proj_idx, std::move(new_expressions)); + idx_t diff_proj_idx = binder.GenerateTableIndex().index; + plan = make_uniq(TableIndex(diff_proj_idx), std::move(new_expressions)); plan->children.push_back(std::move(join)); // ---- Step 4b: Re-apply ORDER BY if the original query had one ---- if (!saved_orders.empty()) { for (auto &o : saved_orders) { auto &ref = o.expression->Cast(); - idx_t col_idx = ref.binding.column_index; - ref.binding = ColumnBinding(diff_proj_idx, col_idx); - ref.return_type = col_types[col_idx]; + idx_t col_idx = ref.BindingMutable().column_index; + ref.BindingMutable() = ColumnBinding(TableIndex(diff_proj_idx), ProjectionIndex(col_idx)); + ref.SetReturnType(col_types[col_idx]); } auto order = make_uniq(std::move(saved_orders)); order->children.push_back(std::move(plan)); diff --git a/src/include/aggregates/as_aggregate.hpp b/src/include/aggregates/as_aggregate.hpp index 249f4a32..e062f81a 100644 --- a/src/include/aggregates/as_aggregate.hpp +++ b/src/include/aggregates/as_aggregate.hpp @@ -3,6 +3,7 @@ #include "duckdb.hpp" #include "duckdb/common/random_engine.hpp" +#include "duckdb/common/clustered_aggregate.hpp" #include "duckdb/execution/expression_executor.hpp" #include "duckdb/transaction/meta_transaction.hpp" #include "pcg_random.hpp" @@ -49,7 +50,8 @@ static inline int ValidateDpSampleLanes(int64_t sample_lanes) { static inline int GetDpSampleLanes(duckdb::ClientContext &ctx) { duckdb::Value sample_lanes_val; - if (ctx.TryGetCurrentSetting("dp_sample_lanes", sample_lanes_val) && !sample_lanes_val.IsNull()) { + if (ctx.TryGetCurrentSetting(duckdb::Identifier("dp_sample_lanes"), sample_lanes_val) && + !sample_lanes_val.IsNull()) { return ValidateDpSampleLanes(sample_lanes_val.GetValue()); } return DP_SAMPLE_DEFAULT_LANES; @@ -64,7 +66,7 @@ static inline int ValidateDpSassM(int64_t m) { static inline int GetDpSassM(duckdb::ClientContext &ctx) { duckdb::Value m_val; - if (ctx.TryGetCurrentSetting("dp_sass_m", m_val) && !m_val.IsNull()) { + if (ctx.TryGetCurrentSetting(duckdb::Identifier("dp_sass_m"), m_val) && !m_val.IsNull()) { return ValidateDpSassM(m_val.GetValue()); } return DP_SASS_DEFAULT_M; @@ -88,7 +90,7 @@ static inline double DpSampleRescale(int sample_lanes) { static inline bool GetDpSassRescale(duckdb::ClientContext &ctx) { duckdb::Value rescale_val; - if (ctx.TryGetCurrentSetting("dp_sass_rescale", rescale_val) && !rescale_val.IsNull()) { + if (ctx.TryGetCurrentSetting(duckdb::Identifier("dp_sass_rescale"), rescale_val) && !rescale_val.IsNull()) { return rescale_val.GetValue(); } return true; @@ -103,7 +105,7 @@ static inline int ValidatePacM(int64_t m) { static inline int GetPacSampleM(duckdb::ClientContext &ctx) { duckdb::Value m_val; - if (ctx.TryGetCurrentSetting("pac_m", m_val) && !m_val.IsNull()) { + if (ctx.TryGetCurrentSetting(duckdb::Identifier("pac_m"), m_val) && !m_val.IsNull()) { return ValidatePacM(m_val.GetValue()); } return DP_SASS_DEFAULT_M; @@ -111,7 +113,7 @@ static inline int GetPacSampleM(duckdb::ClientContext &ctx) { static inline bool GetPacHashRepair(duckdb::ClientContext &ctx) { duckdb::Value hash_repair_val; - if (ctx.TryGetCurrentSetting("pac_hash_repair", hash_repair_val) && !hash_repair_val.IsNull()) { + if (ctx.TryGetCurrentSetting(duckdb::Identifier("pac_hash_repair"), hash_repair_val) && !hash_repair_val.IsNull()) { return hash_repair_val.GetValue(); } return true; @@ -185,6 +187,33 @@ static inline int pac_ctzll(uint64_t x) { namespace duckdb { +// Preserve the single-state kernels for ungrouped input and selected group runs. +template +void ExecutePrivClusters(Vector inputs[], idx_t input_count, const ClusteredAggr &clusters, UPDATE update) { + vector selected; + sel_t selection_data[STANDARD_VECTOR_SIZE]; + SelectionVector selection(selection_data, STANDARD_VECTOR_SIZE); + for (auto &run : clusters.runs()) { + if (!run.sel) { + update(inputs, run.state, run.count); + continue; + } + if (selected.empty()) { + selected.reserve(input_count); + for (idx_t col = 0; col < input_count; col++) { + selected.emplace_back(Vector::Ref(inputs[col])); + } + } + for (idx_t row = 0; row < run.count; row++) { + selection.set_index(row, run.sel[row]); + } + for (idx_t col = 0; col < input_count; col++) { + selected[col].Slice(inputs[col], selection, run.count); + } + update(selected.data(), run.state, run.count); + } +} + // DuckDB's PCG32 core, exposed as a 64-bit URBG for the standard distributions. class PacRandomEngine { public: @@ -313,7 +342,7 @@ uint64_t PacGenerateRandomSeed(); // Returns 1.0/128 if not found or null. inline double GetPacMiFromSetting(ClientContext &ctx) { Value pac_mi_val; - if (ctx.TryGetCurrentSetting("pac_mi", pac_mi_val) && !pac_mi_val.IsNull()) { + if (ctx.TryGetCurrentSetting(Identifier("pac_mi"), pac_mi_val) && !pac_mi_val.IsNull()) { return pac_mi_val.GetValue(); } return 1.0 / 128; // default @@ -357,12 +386,12 @@ struct PrivBindData : public FunctionData { sample_rescale(sample_rescale_val), utility_threshold(std::numeric_limits::quiet_NaN()), random_state(GetPacRandomState(ctx)), total_update_count(0), suspicious_count(0), nonsuspicious_count(0) { Value sd_val; - if (ctx.TryGetCurrentSetting("pac_sample_diversity_check", sd_val) && !sd_val.IsNull()) { + if (ctx.TryGetCurrentSetting(Identifier("pac_sample_diversity_check"), sd_val) && !sd_val.IsNull()) { sample_diversity_check = sd_val.GetValue(); } // Read utility threshold: if set (non-null), enables probabilistic NULLing of low-SNR cells Value ut_val; - if (ctx.TryGetCurrentSetting("privacy_min_group_count", ut_val) && !ut_val.IsNull()) { + if (ctx.TryGetCurrentSetting(Identifier("privacy_min_group_count"), ut_val) && !ut_val.IsNull()) { utility_threshold = ut_val.GetValue(); } seed = GetPacSeed(ctx, mi); @@ -372,7 +401,7 @@ struct PrivBindData : public FunctionData { if (mi > 0.0) { Value pt_val; bool ptracking_enabled = true; // default: enabled - if (ctx.TryGetCurrentSetting("pac_ptracking", pt_val) && !pt_val.IsNull()) { + if (ctx.TryGetCurrentSetting(Identifier("pac_ptracking"), pt_val) && !pt_val.IsNull()) { ptracking_enabled = pt_val.GetValue(); } if (ptracking_enabled) { diff --git a/src/include/aggregates/as_clip_aggr.hpp b/src/include/aggregates/as_clip_aggr.hpp index 323b11bd..e96d5c2c 100644 --- a/src/include/aggregates/as_clip_aggr.hpp +++ b/src/include/aggregates/as_clip_aggr.hpp @@ -158,29 +158,32 @@ static unique_ptr PacClipBindWithScale(ClientContext &ctx, vector< } int clip_support = 0; Value dc_val; - if (ctx.TryGetCurrentSetting("priv_clip_support", dc_val) && !dc_val.IsNull()) { + if (ctx.TryGetCurrentSetting(Identifier("priv_clip_support"), dc_val) && !dc_val.IsNull()) { clip_support = static_cast(dc_val.GetValue()); } bool clip_scale_val = false; Value cs_val; - if (ctx.TryGetCurrentSetting("priv_clip_scale", cs_val) && !cs_val.IsNull()) { + if (ctx.TryGetCurrentSetting(Identifier("priv_clip_scale"), cs_val) && !cs_val.IsNull()) { clip_scale_val = cs_val.GetValue(); } return make_uniq(ctx, mi, correction, clip_support, float_scale, clip_scale_val); } -static unique_ptr PacClipBind(ClientContext &ctx, AggregateFunction &, - vector> &args) { +static unique_ptr PacClipBind(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &args = bind_input.GetArguments(); return PacClipBindWithScale(ctx, args, "as_clip"); } -static unique_ptr PacClipBindFloat(ClientContext &ctx, AggregateFunction &, - vector> &args) { +static unique_ptr PacClipBindFloat(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &args = bind_input.GetArguments(); return PacClipBindWithScale(ctx, args, "as_clip", CLIP_FLOAT_SCALE); } -static unique_ptr PacClipBindDouble(ClientContext &ctx, AggregateFunction &, - vector> &args) { +static unique_ptr PacClipBindDouble(BindAggregateFunctionInput &bind_input) { + auto &ctx = bind_input.GetClientContext(); + auto &args = bind_input.GetArguments(); return PacClipBindWithScale(ctx, args, "as_clip", CLIP_DOUBLE_SCALE); } diff --git a/src/include/aggregates/as_sum.hpp b/src/include/aggregates/as_sum.hpp index 6d893bcd..44b1ca20 100644 --- a/src/include/aggregates/as_sum.hpp +++ b/src/include/aggregates/as_sum.hpp @@ -804,15 +804,15 @@ SubtractNegStateTotals(State *, PAC_FLOAT *, uint64_t &, ArenaAllocator &) { // Generic finalize template (used by as_sum) template -void PacSumFinalize(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset); +void PacSumFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, idx_t offset); // Bind for DECIMAL as_sum -unique_ptr BindDecimalPacSum(ClientContext &ctx, AggregateFunction &function, - vector> &args); +unique_ptr BindDecimalPacSum(BindAggregateFunctionInput &input); // FinalizeCounters for as_sum template -void PacSumAvgFinalizeCounters(Vector &states, AggregateInputData &input, Vector &result, idx_t count, idx_t offset); +void PacSumAvgFinalizeCounters(Vector &states, AggregateFinalizeInputData &input, Vector &result, idx_t count, + idx_t offset); } // namespace duckdb #endif // PAC_GODBOLT diff --git a/src/include/categorical/pac_categorical_detection.hpp b/src/include/categorical/pac_categorical_detection.hpp index 003b7b8a..0c505ecf 100644 --- a/src/include/categorical/pac_categorical_detection.hpp +++ b/src/include/categorical/pac_categorical_detection.hpp @@ -46,8 +46,8 @@ namespace duckdb { // Metadata about a PAC aggregate produced by the bitslice compiler. // Keyed by aggregate_index in PacAggregateInfoMap. struct PacAggregateInfo { - idx_t aggregate_index; // agg->aggregate_index - idx_t group_index; // agg->group_index + idx_t aggregate_index; // agg->aggregate_index.index + idx_t group_index; // agg->group_index.index ColumnBinding hash_binding; // the hash column binding used for this aggregate's children[0] }; // Map from aggregate_index → PacAggregateInfo @@ -179,7 +179,7 @@ inline bool IsNumericalType(const LogicalType &type) { // Hash a column binding to a unique 64-bit value for use in maps/sets static inline uint64_t HashBinding(const ColumnBinding &binding) { - return (uint64_t(binding.table_index) << 32) | binding.column_index; + return (uint64_t(binding.table_index.index) << 32) | binding.column_index.GetIndexUnsafe(); } // Rebind an aggregate function by name (catalog lookup + FunctionBinder). @@ -187,27 +187,27 @@ static inline uint64_t HashBinding(const ColumnBinding &binding) { static inline unique_ptr RebindAggregate(ClientContext &context, const string &func_name, vector> children, bool is_distinct) { auto &catalog = Catalog::GetSystemCatalog(context); - auto &func_entry = catalog.GetEntry(context, DEFAULT_SCHEMA, func_name); + auto &func_entry = catalog.GetEntry(context, DEFAULT_SCHEMA, Identifier(func_name)); vector arg_types; for (auto &child : children) { - arg_types.push_back(child->return_type); + arg_types.push_back(child->GetReturnType()); } ErrorData error; FunctionBinder function_binder(context); - auto best_function = function_binder.BindFunction(func_name, func_entry.functions, arg_types, error); + auto best_function = function_binder.BindFunction(Identifier(func_name), func_entry.functions, arg_types, error); if (!best_function.IsValid()) { return nullptr; } - AggregateFunction func = func_entry.functions.GetFunctionByOffset(best_function.GetIndex()); - return function_binder.BindAggregateFunction(func, std::move(children), nullptr, + const auto &func = func_entry.functions.GetFunctionByOffset(best_function.GetIndex()); + return function_binder.BindAggregateFunction(*func, std::move(children), nullptr, is_distinct ? AggregateType::DISTINCT : AggregateType::NON_DISTINCT); } // Walk through cast expressions to find the underlying expression. // Returns the innermost non-cast expression. static inline Expression *StripCasts(Expression *expr) { - while (expr->type == ExpressionType::OPERATOR_CAST) { - expr = expr->Cast().child.get(); + while (expr->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + expr = expr->Cast().GetChildrenMutable()[0].get(); } return expr; } @@ -215,24 +215,27 @@ static inline Expression *StripCasts(Expression *expr) { // Check if an expression is already wrapped in a categorical rewrite terminal function // This includes as_noised, priv_filter, list_transform, and list_zip // These functions indicate that the expression has already been processed by categorical rewriting -static inline bool IsAlreadyWrappedInPacNoised(Expression *expr) { +static inline bool IsAlreadyWrappedInPacNoised(const Expression *expr) { if (!expr) { return false; } - if (expr->type == ExpressionType::BOUND_FUNCTION) { + if (expr->GetExpressionType() == ExpressionType::BOUND_FUNCTION) { auto &func = expr->Cast(); // Check for all categorical rewrite terminal functions - if (func.function.name == "as_noised" || func.function.name == "as_noised_div" || - StringUtil::StartsWith(func.function.name, "priv_filter") || - StringUtil::StartsWith(func.function.name, "priv_select") || func.function.name == "priv_coalesce" || - func.function.name == "list_transform" || func.function.name == "list_zip") { + if (func.Function().GetName().GetIdentifierName() == "as_noised" || + func.Function().GetName().GetIdentifierName() == "as_noised_div" || + StringUtil::StartsWith(func.Function().GetName().GetIdentifierName(), "priv_filter") || + StringUtil::StartsWith(func.Function().GetName().GetIdentifierName(), "priv_select") || + func.Function().GetName().GetIdentifierName() == "priv_coalesce" || + func.Function().GetName().GetIdentifierName() == "list_transform" || + func.Function().GetName().GetIdentifierName() == "list_zip") { return true; } } // Check for CAST(terminal_function(...)) - if (expr->type == ExpressionType::OPERATOR_CAST) { - auto &cast = expr->Cast(); - return IsAlreadyWrappedInPacNoised(cast.child.get()); + if (expr->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + auto &cast = expr->Cast(); + return IsAlreadyWrappedInPacNoised(cast.GetChildren()[0].get()); } return false; } @@ -240,15 +243,15 @@ static inline bool IsAlreadyWrappedInPacNoised(Expression *expr) { // Check if a CASE expression is DuckDB's scalar subquery wrapper pattern: // CASE WHEN count>1 THEN error(...) ELSE value END static inline bool IsScalarSubqueryWrapper(const BoundCaseExpression &case_expr) { - if (case_expr.case_checks.size() != 1 || !case_expr.else_expr) { + if (case_expr.CaseChecks().size() != 1 || !case_expr.ElseExpression()) { return false; } - auto &check = case_expr.case_checks[0]; + auto &check = case_expr.CaseChecks()[0]; // Counter-list wrappers cast the error branch to the new result type. auto *then_expr = check.then_expr ? StripCasts(check.then_expr.get()) : nullptr; - if (then_expr && then_expr->type == ExpressionType::BOUND_FUNCTION) { + if (then_expr && then_expr->GetExpressionType() == ExpressionType::BOUND_FUNCTION) { auto &func = then_expr->Cast(); - return func.function.name == "error"; + return func.FunctionMutable().GetName().GetIdentifierName() == "error"; } return false; } @@ -264,10 +267,10 @@ static inline string FindPacAggregateInOperator(LogicalOperator *op) { if (op->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { auto &agg = op->Cast(); for (auto &agg_expr : agg.expressions) { - if (agg_expr->type == ExpressionType::BOUND_AGGREGATE) { + if (agg_expr->GetExpressionType() == ExpressionType::BOUND_AGGREGATE) { auto &bound_agg = agg_expr->Cast(); - if (IsAnyPacAggregate(bound_agg.function.name)) { - return GetBasePacAggregateName(bound_agg.function.name); + if (IsAnyPacAggregate(bound_agg.FunctionMutable().GetName().GetIdentifierName())) { + return GetBasePacAggregateName(bound_agg.FunctionMutable().GetName().GetIdentifierName()); } } } @@ -294,7 +297,7 @@ static inline LogicalOperator *RecognizeDuckDBScalarWrapper(LogicalOperator *op) return nullptr; } auto &expr = outer_proj.expressions[0]; - if (expr->type != ExpressionType::CASE_EXPR) { + if (expr->GetExpressionType() != ExpressionType::CASE_EXPR) { return nullptr; } if (!IsScalarSubqueryWrapper(expr->Cast())) { @@ -312,11 +315,11 @@ static inline LogicalOperator *RecognizeDuckDBScalarWrapper(LogicalOperator *op) return nullptr; } auto &agg_expr = agg.expressions[0]; - if (agg_expr->type != ExpressionType::BOUND_AGGREGATE) { + if (agg_expr->GetExpressionType() != ExpressionType::BOUND_AGGREGATE) { return nullptr; } auto &bound_agg = agg_expr->Cast(); - if (StringUtil::Lower(bound_agg.function.name) != "first") { + if (StringUtil::Lower(bound_agg.FunctionMutable().GetName().GetIdentifierName()) != "first") { return nullptr; } if (agg.children.empty()) { diff --git a/src/include/categorical/pac_categorical_rewriter.hpp b/src/include/categorical/pac_categorical_rewriter.hpp index 070444d7..2e795425 100644 --- a/src/include/categorical/pac_categorical_rewriter.hpp +++ b/src/include/categorical/pac_categorical_rewriter.hpp @@ -142,20 +142,20 @@ static inline ExpressionType FlipComparison(ExpressionType cmp_type) { // Try to extract a double value from an expression, looking through casts. // Returns true and sets out_val if the expression is a constant that can be cast to DOUBLE. static inline bool TryGetConstantDouble(Expression *expr, double &out_val) { - while (expr->type == ExpressionType::OPERATOR_CAST) { - expr = expr->Cast().child.get(); + while (expr->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + expr = expr->Cast().GetChildrenMutable()[0].get(); } - if (expr->type != ExpressionType::VALUE_CONSTANT) { + if (expr->GetExpressionType() != ExpressionType::VALUE_CONSTANT) { return false; } auto &const_expr = expr->Cast(); - if (const_expr.value.IsNull()) { + if (const_expr.GetValue().IsNull()) { return false; } - Value double_val; string error_msg; - if (const_expr.value.DefaultTryCastAs(LogicalType::DOUBLE, double_val, &error_msg)) { - out_val = double_val.GetValue(); + auto double_val = const_expr.GetValue().DefaultTryCastAs(LogicalType::DOUBLE, &error_msg); + if (double_val) { + out_val = double_val->GetValue(); return true; } return false; diff --git a/src/include/parser/privacy_parser.hpp b/src/include/parser/privacy_parser.hpp index 7d66d4fb..0eb30c7c 100644 --- a/src/include/parser/privacy_parser.hpp +++ b/src/include/parser/privacy_parser.hpp @@ -74,11 +74,12 @@ struct PACParseData : public ParserExtensionParseData { class PrivacyParserExtension : public ParserExtension { public: PrivacyParserExtension() { - parse_function = PACParseFunction; + parse_function = PACParseTokens; plan_function = PACPlanFunction; } static ParserExtensionParseResult PACParseFunction(ParserExtensionInfo *info, const string &query); + static ParserExtensionParseResult PACParseTokens(ParserExtensionInfo *info, const vector &tokens); static ParserExtensionPlanResult PACPlanFunction(ParserExtensionInfo *info, ClientContext &context, unique_ptr parse_data); diff --git a/src/include/query_processing/pac_subquery_handler.hpp b/src/include/query_processing/pac_subquery_handler.hpp index 7999a6cd..6360fcdf 100644 --- a/src/include/query_processing/pac_subquery_handler.hpp +++ b/src/include/query_processing/pac_subquery_handler.hpp @@ -24,7 +24,7 @@ struct DelimColumnResult { LogicalType type; bool IsValid() const { - return binding.table_index != DConstants::INVALID_INDEX; + return binding.table_index.index != DConstants::INVALID_INDEX; } }; diff --git a/src/include/query_processing/privacy_expression_builder.hpp b/src/include/query_processing/privacy_expression_builder.hpp index 75bc2d84..49888aab 100644 --- a/src/include/query_processing/privacy_expression_builder.hpp +++ b/src/include/query_processing/privacy_expression_builder.hpp @@ -38,7 +38,7 @@ unique_ptr BuildXorHashFromPKs(OptimizerExtensionInput &input, Logic ColumnBinding InsertHashProjectionAboveGet(OptimizerExtensionInput &input, unique_ptr &plan, LogicalGet &get, const vector &key_columns, bool use_rowid); -// Caching wrapper around InsertHashProjectionAboveGet. Keyed by get.table_index. +// Caching wrapper around InsertHashProjectionAboveGet. Keyed by get.table_index.index. // Returns cached binding if hash projection was already inserted for this get. ColumnBinding GetOrInsertHashProjection(OptimizerExtensionInput &input, unique_ptr &plan, LogicalGet &get, const vector &key_columns, bool use_rowid, diff --git a/src/include/query_processing/privacy_fk_resolver.hpp b/src/include/query_processing/privacy_fk_resolver.hpp index a23586c1..c4733e7e 100644 --- a/src/include/query_processing/privacy_fk_resolver.hpp +++ b/src/include/query_processing/privacy_fk_resolver.hpp @@ -38,7 +38,7 @@ FKToPUResult GetFKColumnsToPU(const string &table_name, const vector &pr // This is used when the primary FK table is blocked by SEMI/ANTI joins struct AccessibleFKTableResult { LogicalGet *get_node; - idx_t table_index; + TableIndex table_index; string table_name; vector fk_columns; bool found; diff --git a/src/include/utils/privacy_helpers.hpp b/src/include/utils/privacy_helpers.hpp index bf6482c8..65d1d229 100644 --- a/src/include/utils/privacy_helpers.hpp +++ b/src/include/utils/privacy_helpers.hpp @@ -8,6 +8,9 @@ namespace duckdb { class Binder; +class BoundWindowExpression; + +unique_ptr BindRankingWindow(ClientContext &context, ExpressionType type); // Sanitize a string to be used as a PAC privacy unit or table name (alphanumeric + underscores only) string Sanitize(const string &in); diff --git a/src/metadata/privacy_compatibility_check.cpp b/src/metadata/privacy_compatibility_check.cpp index 9878809e..9daf7787 100644 --- a/src/metadata/privacy_compatibility_check.cpp +++ b/src/metadata/privacy_compatibility_check.cpp @@ -135,6 +135,11 @@ static bool IsDPPrivacyMode(const string &privacy_mode) { } static bool ContainsUnsupportedDPShape(const LogicalOperator &op, string &shape) { + if (op.type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE && + StringUtil::StartsWith(op.Cast().ctename.GetIdentifierName(), "__duckdb_delim_")) { + shape = "lateral or correlated subqueries"; + return true; + } if (op.type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE || op.type == LogicalOperatorType::LOGICAL_CTE_REF || op.type == LogicalOperatorType::LOGICAL_RECURSIVE_CTE) { shape = "CTEs"; @@ -181,7 +186,7 @@ static bool ContainsAggregation(const LogicalOperator &op) { for (auto &expr : aggr.expressions) { if (expr && expr->IsAggregate()) { auto &ag = expr->Cast(); - if (IsAllowedAggregate(ag.function.name)) { + if (IsAllowedAggregate(ag.FunctionMutable().GetName().GetIdentifierName())) { return true; } } @@ -201,7 +206,7 @@ static void GetScannedTablesInScope(const LogicalOperator &op, std::unordered_se auto &get = op.Cast(); auto table_entry = get.GetTable(); if (table_entry) { - tables.insert(table_entry->name); + tables.insert(table_entry->name.GetIdentifierName()); } } @@ -250,10 +255,10 @@ static bool CheckPacAggregatesHaveProperJoins(const LogicalOperator &op, for (auto &expr : aggr.expressions) { if (expr && expr->IsAggregate()) { auto &ag = expr->Cast(); - if (IsPacAggregate(ag.function.name)) { + if (IsPacAggregate(ag.FunctionMutable().GetName().GetIdentifierName())) { has_pac_aggregate = true; found_pac_aggregate = true; - } else if (IsAllowedAggregate(ag.function.name)) { + } else if (IsAllowedAggregate(ag.FunctionMutable().GetName().GetIdentifierName())) { has_regular_aggregate = true; } } @@ -359,7 +364,7 @@ static bool CheckPacAggregatesHaveProperJoins(const LogicalOperator &op, static void TraceBindingToSources(LogicalOperator &root, const ColumnBinding &binding, const std::function &visit_source_binding) { // Find the operator that produces this binding's table_index - auto *source_op = FindOperatorByTableIndex(&root, binding.table_index); + auto *source_op = FindOperatorByTableIndex(&root, binding.table_index.index); if (!source_op) { return; // Can't find source, assume safe } @@ -370,35 +375,36 @@ static void TraceBindingToSources(LogicalOperator &root, const ColumnBinding &bi auto &aggr = source_op->Cast(); // Check if this binding is from the aggregate's group output or aggregate output - if (binding.table_index == aggr.group_index) { + if (binding.table_index.index == aggr.group_index.index) { // This is a grouped column - trace it further down idx_t group_idx = binding.column_index; if (group_idx < aggr.groups.size() && aggr.groups[group_idx]) { // Find column refs in this group expression and trace them ExpressionIterator::EnumerateExpression( const_cast &>(aggr.groups[group_idx]), [&](Expression &expr) { - if (expr.type == ExpressionType::BOUND_COLUMN_REF) { + if (expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr.Cast(); - TraceBindingToSources(root, col_ref.binding, visit_source_binding); + TraceBindingToSources(root, col_ref.BindingMutable(), visit_source_binding); } }); } } - // If binding.table_index == aggr.aggregate_index, it's an aggregate result - safe, don't trace further + // If binding.table_index.index == aggr.aggregate_index.index, it's an aggregate result - safe, don't trace + // further } else if (source_op->type == LogicalOperatorType::LOGICAL_PROJECTION) { auto &proj = source_op->Cast(); // Trace the expression that produces this column if (binding.column_index < proj.expressions.size() && proj.expressions[binding.column_index]) { ExpressionIterator::EnumerateExpression(proj.expressions[binding.column_index], [&](Expression &expr) { - if (expr.type == ExpressionType::BOUND_COLUMN_REF) { + if (expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr.Cast(); - TraceBindingToSources(root, col_ref.binding, visit_source_binding); + TraceBindingToSources(root, col_ref.BindingMutable(), visit_source_binding); } }); } } else if (source_op->type == LogicalOperatorType::LOGICAL_CTE_REF) { auto &cte_ref = source_op->Cast(); - auto *cte_def = FindMaterializedCTE(&root, cte_ref.cte_index); + auto *cte_def = FindMaterializedCTE(&root, cte_ref.cte_index.index); if (cte_def && !cte_def->children.empty() && cte_def->children[0]) { auto &cte_body = *cte_def->children[0]; auto body_bindings = cte_body.GetColumnBindings(); @@ -431,7 +437,7 @@ static std::pair GetProtectedColumnInfo(LogicalOperator &root, const ColumnBinding &binding, const std::unordered_map> &protected_columns) { // Find the LogicalGet that produces this binding - auto *source_op = FindOperatorByTableIndex(&root, binding.table_index); + auto *source_op = FindOperatorByTableIndex(&root, binding.table_index.index); if (!source_op || source_op->type != LogicalOperatorType::LOGICAL_GET) { return {"", ""}; } @@ -448,18 +454,18 @@ GetProtectedColumnInfo(LogicalOperator &root, const ColumnBinding &binding, string col_name; const auto &column_ids = get.GetColumnIds(); if (binding.column_index < column_ids.size()) { - col_name = get.GetColumnName(column_ids[binding.column_index]); + col_name = get.GetColumnName(column_ids[binding.column_index]).GetIdentifierName(); } if (col_name.empty()) { return {"", ""}; } // Check against the unified protected_columns set - string table_lower = StringUtil::Lower(table_entry->name); + string table_lower = StringUtil::Lower(table_entry->name.GetIdentifierName()); string col_lower = StringUtil::Lower(col_name); auto it = protected_columns.find(table_lower); if (it != protected_columns.end() && it->second.count(col_lower) > 0) { - return {table_entry->name, col_name}; + return {table_entry->name.GetIdentifierName(), col_name}; } return {"", ""}; @@ -499,9 +505,9 @@ static void CheckOutputColumnsNotFromPU(LogicalOperator ¤t_op, LogicalOper continue; } ExpressionIterator::EnumerateExpression(expr, [&](Expression &e) { - if (e.type == ExpressionType::BOUND_COLUMN_REF) { + if (e.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e.Cast(); - TraceBindingToPUTable(plan_root, col_ref.binding, actual_pu_tables); + TraceBindingToPUTable(plan_root, col_ref.BindingMutable(), actual_pu_tables); } }); } @@ -520,7 +526,7 @@ static void CheckOutputColumnsNotFromPU(LogicalOperator ¤t_op, LogicalOper auto table_entry = get.GetTable(); if (table_entry) { for (auto &pu_table : actual_pu_tables) { - if (table_entry->name == pu_table) { + if (table_entry->name.GetIdentifierName() == pu_table) { throw InvalidInputException( "Privacy rewrite: columns from privacy unit tables can only be accessed inside aggregate " "functions (e.g., SUM, COUNT, AVG, MIN, MAX)"); @@ -571,9 +577,9 @@ CheckOutputColumnsNotProtected(LogicalOperator ¤t_op, LogicalOperator &pla continue; } ExpressionIterator::EnumerateExpression(expr, [&](Expression &e) { - if (e.type == ExpressionType::BOUND_COLUMN_REF) { + if (e.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e.Cast(); - TraceBindingForProtectedColumns(plan_root, col_ref.binding, protected_columns); + TraceBindingForProtectedColumns(plan_root, col_ref.BindingMutable(), protected_columns); } }); } @@ -590,18 +596,18 @@ CheckOutputColumnsNotProtected(LogicalOperator ¤t_op, LogicalOperator &pla auto &get = current_op.Cast(); auto table_entry = get.GetTable(); if (table_entry) { - string table_lower = StringUtil::Lower(table_entry->name); + string table_lower = StringUtil::Lower(table_entry->name.GetIdentifierName()); auto it = protected_columns.find(table_lower); if (it != protected_columns.end()) { const auto &column_ids = get.GetColumnIds(); for (const auto &col_idx : column_ids) { - const string &col_name = get.GetColumnName(col_idx); + const string &col_name = get.GetColumnName(col_idx).GetIdentifierName(); string col_lower = StringUtil::Lower(col_name); if (it->second.count(col_lower) > 0) { throw InvalidInputException( "Privacy rewrite: protected column '%s.%s' can only be accessed inside aggregate " "functions (e.g., SUM, COUNT, AVG, MIN, MAX)", - table_entry->name.c_str(), col_name.c_str()); + table_entry->name.GetIdentifierName().c_str(), col_name.c_str()); } } } @@ -645,9 +651,10 @@ CheckFiltersNotUsingProtectedColumns(LogicalOperator &op, LogicalOperator &plan_ continue; } ExpressionIterator::EnumerateExpression(expr, [&](Expression &e) { - if (e.type == ExpressionType::BOUND_COLUMN_REF) { + if (e.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e.Cast(); - auto protected_info = GetProtectedColumnInfo(plan_root, col_ref.binding, protected_columns); + auto protected_info = + GetProtectedColumnInfo(plan_root, col_ref.BindingMutable(), protected_columns); if (!protected_info.first.empty()) { throw InvalidInputException( "Privacy rewrite: protected column '%s.%s' cannot be used in WHERE/HAVING filters. " @@ -675,7 +682,7 @@ void CountScans(const LogicalOperator &op, std::unordered_map &co auto &scan = op.Cast(); auto table_entry = scan.GetTable(); if (table_entry) { - counts[table_entry->name]++; + counts[table_entry->name.GetIdentifierName()]++; } } // Handle CTEs: traverse into CTE definitions to find base table scans diff --git a/src/metadata/privacy_metadata_manager.cpp b/src/metadata/privacy_metadata_manager.cpp index 081ce493..f792af92 100644 --- a/src/metadata/privacy_metadata_manager.cpp +++ b/src/metadata/privacy_metadata_manager.cpp @@ -166,7 +166,7 @@ void PrivacyMetadataManager::Clear() { */ string PrivacyMetadataManager::GetMetadataFilePath(ClientContext &context) { // Get the database path from the default catalog - auto &db_name = DatabaseManager::GetDefaultDatabase(context); + auto db_name = DatabaseManager::GetDefaultDatabase(context); auto &catalog = Catalog::GetCatalog(context, db_name); string db_path = catalog.GetDBPath(); @@ -181,7 +181,7 @@ string PrivacyMetadataManager::GetMetadataFilePath(ClientContext &context) { CatalogSearchPath search_path(context); const auto &entries = search_path.Get(); if (!entries.empty()) { - schema_name = entries[0].schema; + schema_name = entries[0].GetSchema().GetIdentifierName(); } } catch (...) { // If we can't get search path, use default schema diff --git a/src/parser/privacy_parser.cpp b/src/parser/privacy_parser.cpp index 1e1c3324..8eff92d8 100644 --- a/src/parser/privacy_parser.cpp +++ b/src/parser/privacy_parser.cpp @@ -46,7 +46,7 @@ static vector FindMissingColumns(const TableCatalogEntry &table, const v for (const auto &col_name : column_names) { bool found = false; for (auto &col : table.GetColumns().Logical()) { - if (StringUtil::Lower(col.GetName()) == StringUtil::Lower(col_name)) { + if (StringUtil::Lower(col.GetName().GetIdentifierName()) == StringUtil::Lower(col_name)) { found = true; break; } @@ -210,7 +210,7 @@ static thread_local string g_pac_pending_table_name; * it persists across database sessions. For in-memory databases, no file is saved. */ static unique_ptr PACDDLBindFunction(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { // Get the pending SQL from thread-local storage string sql_to_execute = g_pac_pending_sql; string table_name = g_pac_pending_table_name; @@ -276,6 +276,30 @@ static void PACDDLExecuteFunction(ClientContext &context, TableFunctionInput &da * The order matters: DROP must be checked before ADD because DROP statements * also contain keywords like "protected" that might match ADD patterns. */ +ParserExtensionParseResult PrivacyParserExtension::PACParseTokens(ParserExtensionInfo *info, + const vector &tokens) { + string query; + idx_t consumed = 0; + for (auto &token : tokens) { + if (token.type == TokenType::END_OF_INPUT) { + break; + } + consumed++; + if (token.type == TokenType::TERMINATOR) { + break; + } + if (!query.empty()) { + query += ' '; + } + query += token.text; + } + auto result = PACParseFunction(info, query); + if (result.type == ParserExtensionResultType::PARSE_SUCCESSFUL) { + result.consumed_tokens = NumericCast(consumed); + } + return result; +} + ParserExtensionParseResult PrivacyParserExtension::PACParseFunction(ParserExtensionInfo *info, const string &query) { // Clean up query - preserve spaces but remove semicolons, comments, and newlines string clean_query = query; @@ -362,8 +386,9 @@ ParserExtensionPlanResult PrivacyParserExtension::PACPlanFunction(ParserExtensio // Check if table exists in the catalog try { auto &catalog = Catalog::GetCatalog(context, DatabaseManager::GetDefaultDatabase(context)); - auto table_entry = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, - pac_data.metadata.table_name, OnEntryNotFound::RETURN_NULL); + auto table_entry = + catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, + Identifier(pac_data.metadata.table_name), OnEntryNotFound::RETURN_NULL); if (!table_entry) { throw CatalogException("Table '" + pac_data.metadata.table_name + "' does not exist"); } @@ -411,8 +436,9 @@ ParserExtensionPlanResult PrivacyParserExtension::PACPlanFunction(ParserExtensio "table '" + pac_data.metadata.table_name + "'. PAC LINK was not added."); // Validate that referenced table exists - auto ref_table_entry = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, - link.referenced_table, OnEntryNotFound::RETURN_NULL); + auto ref_table_entry = + catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, + Identifier(link.referenced_table), OnEntryNotFound::RETURN_NULL); if (!ref_table_entry) { throw CatalogException("Referenced table '" + link.referenced_table + "' does not exist"); } @@ -444,8 +470,9 @@ ParserExtensionPlanResult PrivacyParserExtension::PACPlanFunction(ParserExtensio auto &catalog = Catalog::GetCatalog(context, DatabaseManager::GetDefaultDatabase(context)); for (const auto &link : pac_data.metadata.links) { // Check if referenced table exists and is a PAC table - auto ref_table_entry = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, - link.referenced_table, OnEntryNotFound::RETURN_NULL); + auto ref_table_entry = + catalog.GetEntry(context, CatalogType::TABLE_ENTRY, DEFAULT_SCHEMA, + Identifier(link.referenced_table), OnEntryNotFound::RETURN_NULL); if (ref_table_entry) { auto ref_metadata = PrivacyMetadataManager::Get().GetTableMetadata(link.referenced_table); if (ref_metadata && ref_metadata->is_privacy_unit) { diff --git a/src/parser/privacy_parser_helpers.cpp b/src/parser/privacy_parser_helpers.cpp index c46c3ca3..ce6348d6 100644 --- a/src/parser/privacy_parser_helpers.cpp +++ b/src/parser/privacy_parser_helpers.cpp @@ -12,6 +12,7 @@ // IMPORTANT: must be included BEFORE duckdb headers on Windows MSVC // because DuckDB defines its own std namespace that conflicts with #include +#include #include "parser/privacy_parser.hpp" #include "parser/privacy_parser_helpers.hpp" diff --git a/src/query_processing/pac_avg_rewriter.cpp b/src/query_processing/pac_avg_rewriter.cpp index f5f5cb75..a090df3a 100644 --- a/src/query_processing/pac_avg_rewriter.cpp +++ b/src/query_processing/pac_avg_rewriter.cpp @@ -4,6 +4,7 @@ // works for both compiler-generated and user-written as_avg() SQL. // +#include "duckdb/planner/expression_iterator.hpp" #include "query_processing/pac_avg_rewriter.hpp" #include "query_processing/privacy_expression_builder.hpp" #include "privacy_debug.hpp" @@ -21,10 +22,10 @@ namespace duckdb { // with a copy of the corresponding expression value. Recurses into children. static void ReplaceAvgRefs(unique_ptr &expr, const unordered_map> &replacements, idx_t table_index) { - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr->Cast(); - if (col_ref.binding.table_index == table_index) { - auto it = replacements.find(col_ref.binding.column_index); + if (col_ref.BindingMutable().table_index.index == table_index) { + auto it = replacements.find(col_ref.BindingMutable().column_index); if (it != replacements.end()) { expr = it->second->Copy(); return; @@ -61,7 +62,8 @@ static void RewritePacAvgAggregate(OptimizerExtensionInput &input, unique_ptrCast(); - if (aggr.function.name == "as_noised_avg" || aggr.function.name == "as_avg") { + if (aggr.FunctionMutable().GetName().GetIdentifierName() == "as_noised_avg" || + aggr.FunctionMutable().GetName().GetIdentifierName() == "as_avg") { avg_positions.push_back(i); } } @@ -69,7 +71,7 @@ static void RewritePacAvgAggregate(OptimizerExtensionInput &input, unique_ptrCast(); - bool is_noised = (old_aggr.function.name == "as_noised_avg"); + bool is_noised = (old_aggr.FunctionMutable().GetName().GetIdentifierName() == "as_noised_avg"); // Extract hash and value children: pac_[noised_]avg(hash, col[, correction]) - auto hash_child = old_aggr.children[0]->Copy(); - auto value_child = old_aggr.children[1]->Copy(); + auto hash_child = old_aggr.GetChildrenMutable()[0]->Copy(); + auto value_child = old_aggr.GetChildrenMutable()[1]->Copy(); // Extract optional correction factor (3rd arg), default to constant 1.0 unique_ptr corr_expr; - if (old_aggr.children.size() >= 3) { - corr_expr = old_aggr.children[2]->Copy(); + if (old_aggr.GetChildrenMutable().size() >= 3) { + corr_expr = old_aggr.GetChildrenMutable()[2]->Copy(); } string sum_name = is_noised ? "as_noised_sum" : "as_sum"; @@ -115,11 +117,13 @@ static void RewritePacAvgAggregate(OptimizerExtensionInput &input, unique_ptrreturn_type; - auto count_type = agg.expressions[info.count_col]->return_type; + auto sum_type = agg.expressions[info.sum_col]->GetReturnType(); + auto count_type = agg.expressions[info.count_col]->GetReturnType(); - auto sum_ref = make_uniq(sum_type, ColumnBinding(agg_index, info.sum_col)); - auto cnt_ref = make_uniq(count_type, ColumnBinding(agg_index, info.count_col)); + auto sum_ref = make_uniq( + sum_type, ColumnBinding(TableIndex(agg_index), ProjectionIndex(info.sum_col))); + auto cnt_ref = make_uniq( + count_type, ColumnBinding(TableIndex(agg_index), ProjectionIndex(info.count_col))); unique_ptr div_expr; if (info.is_noised) { diff --git a/src/query_processing/pac_derived_rewriter.cpp b/src/query_processing/pac_derived_rewriter.cpp index 1a94d8ab..16a287a1 100644 --- a/src/query_processing/pac_derived_rewriter.cpp +++ b/src/query_processing/pac_derived_rewriter.cpp @@ -62,8 +62,8 @@ static LogicalType CounterListType() { // Returns true if the expression tree contains a priv_finalize call. static bool ExpressionContainsPacFinalize(Expression &e) { - if (e.type == ExpressionType::BOUND_FUNCTION && - e.Cast().function.name == "priv_finalize") { + if (e.GetExpressionType() == ExpressionType::BOUND_FUNCTION && + e.Cast().FunctionMutable().GetName().GetIdentifierName() == "priv_finalize") { return true; } bool found = false; @@ -76,9 +76,9 @@ static bool ExpressionContainsPacFinalize(Expression &e) { } static bool ExpressionReferencesFinalizedBinding(Expression &e, const unordered_set &finalized_bindings) { - if (e.type == ExpressionType::BOUND_COLUMN_REF) { + if (e.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e.Cast(); - return finalized_bindings.count(HashBinding(col_ref.binding)) > 0; + return finalized_bindings.count(HashBinding(col_ref.BindingMutable())) > 0; } bool found = false; ExpressionIterator::EnumerateChildren(e, [&](Expression &child) { @@ -110,34 +110,35 @@ static void ConvertAggregatesRecursive(OptimizerExtensionInput &input, LogicalOp for (idx_t i = 0; i < agg.expressions.size(); i++) { auto &expr = agg.expressions[i]; - if (expr->type != ExpressionType::BOUND_AGGREGATE) { + if (expr->GetExpressionType() != ExpressionType::BOUND_AGGREGATE) { continue; } auto &bound_agg = expr->Cast(); - if (!IsPacAggregate(bound_agg.function.name)) { + if (!IsPacAggregate(bound_agg.FunctionMutable().GetName().GetIdentifierName())) { continue; } - string counters_name = GetCountersVariant(bound_agg.function.name); + string counters_name = GetCountersVariant(bound_agg.FunctionMutable().GetName().GetIdentifierName()); if (counters_name.empty()) { continue; } vector> children; - for (auto &child_expr : bound_agg.children) { + for (auto &child_expr : bound_agg.GetChildrenMutable()) { children.push_back(child_expr->Copy()); } auto new_aggr = RebindAggregate(input.context, counters_name, std::move(children), bound_agg.IsDistinct()); if (new_aggr) { - PRIVACY_DEBUG_PRINT("[PAC DERIVED WRITE] Rebound " + bound_agg.function.name + " → " + - new_aggr->Cast().function.name); + PRIVACY_DEBUG_PRINT("[PAC DERIVED WRITE] Rebound " + bound_agg.Function().GetName().GetIdentifierName() + + " → " + + new_aggr->Cast().Function().GetName().GetIdentifierName()); agg.expressions[i] = std::move(new_aggr); } else { - bound_agg.function.name = counters_name; - bound_agg.function.return_type = list_type; - expr->return_type = list_type; + bound_agg.FunctionMutable().SetName(Identifier(counters_name)); + bound_agg.FunctionMutable().SetReturnType(list_type); + expr->SetReturnType(list_type); } idx_t types_index = agg.groups.size() + i; @@ -145,7 +146,7 @@ static void ConvertAggregatesRecursive(OptimizerExtensionInput &input, LogicalOp agg.types[types_index] = list_type; } - converted.insert(HashBinding(ColumnBinding(agg.aggregate_index, i))); + converted.insert(HashBinding(ColumnBinding(TableIndex(agg.aggregate_index.index), ProjectionIndex(i)))); } } @@ -156,10 +157,10 @@ static void FixStaleColumnRefTypes(LogicalOperator *op, const unordered_set *expr_ptr) { std::function &)> Fix = [&](unique_ptr &e) { - if (e->type == ExpressionType::BOUND_COLUMN_REF) { + if (e->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e->Cast(); - if (converted.count(HashBinding(col_ref.binding))) { - col_ref.return_type = list_type; + if (converted.count(HashBinding(col_ref.BindingMutable()))) { + col_ref.SetReturnType(list_type); } } ExpressionIterator::EnumerateChildren(*e, Fix); @@ -180,10 +181,10 @@ static void StripRedundantCasts(LogicalOperator *op) { } std::function &)> Strip = [&](unique_ptr &e) { ExpressionIterator::EnumerateChildren(*e, Strip); - if (e->GetExpressionClass() == ExpressionClass::BOUND_CAST) { - auto &cast = e->Cast(); - if (cast.child->return_type == cast.return_type) { - e = std::move(cast.child); + if (e->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + auto &cast = e->Cast(); + if (cast.GetChildrenMutable()[0]->GetReturnType() == cast.GetReturnType()) { + e = std::move(cast.GetChildrenMutable()[0]); } } }; @@ -220,14 +221,14 @@ static void FindDerivedPuGets(LogicalOperator *op, unordered_set &derived return; } auto &mgr = PrivacyMetadataManager::Get(); - auto *meta = mgr.GetTableMetadata(entry->name); + auto *meta = mgr.GetTableMetadata(entry->name.GetIdentifierName()); if (!meta || !meta->derived_pu) { return; } auto list_type = CounterListType(); for (idx_t i = 0; i < get.returned_types.size(); i++) { if (get.returned_types[i] == list_type) { - derived_table_indices.insert(get.table_index); + derived_table_indices.insert(get.table_index.index); return; } } @@ -235,25 +236,25 @@ static void FindDerivedPuGets(LogicalOperator *op, unordered_set &derived // Returns true if the expression is (or contains at its root) a counter column ref. static bool IsCounterColumnRef(Expression &e, const LogicalType &list_type) { - if (e.type == ExpressionType::BOUND_COLUMN_REF && e.return_type == list_type) { + if (e.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF && e.GetReturnType() == list_type) { return true; } // Also catch CAST(counter_col → scalar) inserted by the binder - if (e.GetExpressionClass() == ExpressionClass::BOUND_CAST) { - auto &cast = e.Cast(); - return IsCounterColumnRef(*cast.child, list_type); + if (e.GetExpressionType() == ExpressionType::OPERATOR_CAST) { + auto &cast = e.Cast(); + return IsCounterColumnRef(*cast.GetChildrenMutable()[0], list_type); } return false; } // Extract the raw counter column ref (stripping binder-inserted casts). static unique_ptr ExtractCounterRef(unique_ptr e, const LogicalType &list_type) { - if (e->type == ExpressionType::BOUND_COLUMN_REF && e->return_type == list_type) { + if (e->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF && e->GetReturnType() == list_type) { return e; } - if (e->GetExpressionClass() == ExpressionClass::BOUND_CAST) { - auto &cast = e->Cast(); - return ExtractCounterRef(std::move(cast.child), list_type); + if (e->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + auto &cast = e->Cast(); + return ExtractCounterRef(std::move(cast.GetChildrenMutable()[0]), list_type); } return e; } @@ -262,28 +263,28 @@ static unique_ptr ExtractCounterRef(unique_ptr e, const // Strips binder-inserted CAST(scalar→FLOAT[]) and FLOAT[] constants back to scalars. static unique_ptr CastScalarToPacFloat(unique_ptr scalar, const LogicalType &pac_float_type, const LogicalType &list_type) { - if (scalar->return_type == pac_float_type) { + if (scalar->GetReturnType() == pac_float_type) { return scalar; } // Strip any binder-inserted CAST to FLOAT[] (scalar→LIST) - if (scalar->GetExpressionClass() == ExpressionClass::BOUND_CAST) { - auto &cast = scalar->Cast(); - if (cast.return_type == list_type && cast.child->return_type != list_type) { - scalar = std::move(cast.child); + if (scalar->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + auto &cast = scalar->Cast(); + if (cast.GetReturnType() == list_type && cast.GetChildrenMutable()[0]->GetReturnType() != list_type) { + scalar = std::move(cast.GetChildrenMutable()[0]); } } // Also unwrap FLOAT[] constants back to scalar - if (scalar->type == ExpressionType::VALUE_CONSTANT) { + if (scalar->GetExpressionType() == ExpressionType::VALUE_CONSTANT) { auto &constant = scalar->Cast(); - if (constant.return_type == list_type && !constant.value.IsNull()) { - auto &list_children = ListValue::GetChildren(constant.value); + if (constant.GetReturnType() == list_type && !constant.GetValue().IsNull()) { + auto &list_children = ListValue::GetChildren(constant.GetValue()); if (!list_children.empty()) { auto scalar_val = list_children[0].DefaultCastAs(pac_float_type); return make_uniq(std::move(scalar_val)); } } } - if (scalar->return_type != pac_float_type) { + if (scalar->GetReturnType() != pac_float_type) { return BoundCastExpression::AddDefaultCastToType(std::move(scalar), pac_float_type); } return scalar; @@ -297,7 +298,7 @@ static void SyncMaterializedCTERefTypes(LogicalOperator *op, idx_t cte_index, co } if (op->type == LogicalOperatorType::LOGICAL_CTE_REF) { auto &cte_ref = op->Cast(); - if (cte_ref.cte_index == cte_index) { + if (cte_ref.cte_index.index == cte_index) { auto bindings = cte_ref.GetColumnBindings(); for (idx_t i = 0; i < cte_types.size() && i < cte_ref.chunk_types.size(); i++) { if (cte_ref.chunk_types[i] == list_type && cte_types[i] == finalized_type && i < bindings.size()) { @@ -317,11 +318,11 @@ static void SyncMaterializedCTERefTypes(LogicalOperator *op, idx_t cte_index, co // Rewrite a comparison expression on a counter column to a priv_filter_ call. // Returns the rewritten expression, or nullptr if the comparison doesn't involve a counter column. -static unique_ptr RewriteComparisonToFilterCmp(OptimizerExtensionInput &input, - BoundComparisonExpression &cmp, const LogicalType &list_type, +static unique_ptr RewriteComparisonToFilterCmp(OptimizerExtensionInput &input, BoundFunctionExpression &cmp, + const LogicalType &list_type, const LogicalType &finalized_type) { - bool left_is_counter = IsCounterColumnRef(*cmp.left, list_type); - bool right_is_counter = IsCounterColumnRef(*cmp.right, list_type); + bool left_is_counter = IsCounterColumnRef(*cmp.GetChildrenMutable()[0], list_type); + bool right_is_counter = IsCounterColumnRef(*cmp.GetChildrenMutable()[1], list_type); if (!left_is_counter && !right_is_counter) { return nullptr; } @@ -330,17 +331,19 @@ static unique_ptr RewriteComparisonToFilterCmp(OptimizerExtensionInp return nullptr; } bool counter_on_left = left_is_counter; - ExpressionType effective = counter_on_left ? FlipComparison(cmp.type) : cmp.type; + ExpressionType effective = counter_on_left ? FlipComparison(cmp.GetExpressionType()) : cmp.GetExpressionType(); const char *resolved = GetPacFilterCmpName(effective, GetPrivacyMode(input.context)); if (!resolved) { return nullptr; } string func_name = resolved; - auto counter_ref = ExtractCounterRef(std::move(counter_on_left ? cmp.left : cmp.right), list_type); + auto counter_ref = ExtractCounterRef( + std::move(counter_on_left ? cmp.GetChildrenMutable()[0] : cmp.GetChildrenMutable()[1]), list_type); auto scalar_expr = - CastScalarToPacFloat(std::move(counter_on_left ? cmp.right : cmp.left), finalized_type, list_type); + CastScalarToPacFloat(std::move(counter_on_left ? cmp.GetChildrenMutable()[1] : cmp.GetChildrenMutable()[0]), + finalized_type, list_type); PRIVACY_DEBUG_PRINT("[PAC DERIVED READ] Rewriting comparison to " + func_name); - return input.optimizer.BindScalarFunction(func_name, std::move(scalar_expr), std::move(counter_ref)); + return input.optimizer.BindScalarFunction(Identifier(func_name), std::move(scalar_expr), std::move(counter_ref)); } // Rewrite a BETWEEN expression on a counter column to priv_filter_ AND conjunction. @@ -360,8 +363,10 @@ RewriteBetweenToFilterCmp(OptimizerExtensionInput &input, unique_ptr string upper_func = GetPacFilterCmpName(upper_inclusive ? ExpressionType::COMPARE_GREATERTHANOREQUALTO : ExpressionType::COMPARE_GREATERTHAN, privacy_mode); - auto lower_call = input.optimizer.BindScalarFunction(lower_func, std::move(lower_scalar), std::move(counter_ref1)); - auto upper_call = input.optimizer.BindScalarFunction(upper_func, std::move(upper_scalar), std::move(counter_ref2)); + auto lower_call = + input.optimizer.BindScalarFunction(Identifier(lower_func), std::move(lower_scalar), std::move(counter_ref1)); + auto upper_call = + input.optimizer.BindScalarFunction(Identifier(upper_func), std::move(upper_scalar), std::move(counter_ref2)); PRIVACY_DEBUG_PRINT("[PAC DERIVED READ] Rewriting BETWEEN to " + lower_func + " AND " + upper_func); return make_uniq(ExpressionType::CONJUNCTION_AND, std::move(lower_call), std::move(upper_call)); @@ -394,8 +399,8 @@ static void WrapCounterRefsWithFinalize(OptimizerExtensionInput &input, LogicalO auto &cte = op->Cast(); WrapCounterRefsWithFinalize(input, op->children[0].get(), derived_table_indices, finalized_bindings, child_suppress); - SyncMaterializedCTERefTypes(op->children[1].get(), cte.table_index, op->children[0]->types, CounterListType(), - PacFloatLogicalType(), finalized_bindings); + SyncMaterializedCTERefTypes(op->children[1].get(), cte.table_index.index, op->children[0]->types, + CounterListType(), PacFloatLogicalType(), finalized_bindings); WrapCounterRefsWithFinalize(input, op->children[1].get(), derived_table_indices, finalized_bindings, child_suppress); } else { @@ -415,21 +420,21 @@ static void WrapCounterRefsWithFinalize(OptimizerExtensionInput &input, LogicalO // Fix stale column refs that were finalized by a child operator std::function &)> FixExpr = [&](unique_ptr &e) { - if (e->type == ExpressionType::BOUND_COLUMN_REF) { + if (e->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e->Cast(); - if (finalized_bindings.count(HashBinding(col_ref.binding))) { - col_ref.return_type = finalized_type; + if (finalized_bindings.count(HashBinding(col_ref.BindingMutable()))) { + col_ref.SetReturnType(finalized_type); } } ExpressionIterator::EnumerateChildren(*e, FixExpr); - if (e->return_type == list_type && ExpressionReferencesFinalizedBinding(*e, finalized_bindings)) { - e->return_type = finalized_type; + if (e->GetReturnType() == list_type && ExpressionReferencesFinalizedBinding(*e, finalized_bindings)) { + e->SetReturnType(finalized_type); } - if (e->GetExpressionClass() == ExpressionClass::BOUND_BETWEEN) { - auto &between = e->Cast(); - FixExpr(between.input); - FixExpr(between.lower); - FixExpr(between.upper); + if (e->GetExpressionType() == ExpressionType::COMPARE_BETWEEN) { + auto &between = e->Cast(); + FixExpr(between.GetChildrenMutable()[0]); + FixExpr(between.GetChildrenMutable()[1]); + FixExpr(between.GetChildrenMutable()[2]); } }; LogicalOperatorVisitor::EnumerateExpressions(*op, [&](unique_ptr *expr_ptr) { FixExpr(*expr_ptr); }); @@ -438,30 +443,30 @@ static void WrapCounterRefsWithFinalize(OptimizerExtensionInput &input, LogicalO // and wrap remaining counter column refs with priv_finalize. std::function &)> WrapExpr = [&](unique_ptr &e) { // Skip already-wrapped expressions - if (e->type == ExpressionType::BOUND_FUNCTION) { - auto &func_name = e->Cast().function.name; + if (e->GetExpressionType() == ExpressionType::BOUND_FUNCTION) { + auto &func_name = e->Cast().FunctionMutable().GetName().GetIdentifierName(); if (func_name == "priv_finalize" || StringUtil::StartsWith(func_name, "priv_filter_")) { return; } } // Skip binder-inserted CAST(FLOAT[] → scalar) — the registered cast handles finalization - if (e->GetExpressionClass() == ExpressionClass::BOUND_CAST) { - auto &cast = e->Cast(); - if (cast.child->return_type == list_type && cast.return_type != list_type) { + if (e->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + auto &cast = e->Cast(); + if (cast.GetChildrenMutable()[0]->GetReturnType() == list_type && cast.GetReturnType() != list_type) { return; } } // Rewrite BoundComparisonExpression where one side is a counter column - if (e->GetExpressionClass() == ExpressionClass::BOUND_COMPARISON) { - auto &cmp = e->Cast(); + if (BoundComparisonExpression::IsComparison(*e)) { + auto &cmp = e->Cast(); // Recurse into the scalar side first (it may contain nested expressions) - bool left_is_counter = IsCounterColumnRef(*cmp.left, list_type); - bool right_is_counter = IsCounterColumnRef(*cmp.right, list_type); + bool left_is_counter = IsCounterColumnRef(*cmp.GetChildrenMutable()[0], list_type); + bool right_is_counter = IsCounterColumnRef(*cmp.GetChildrenMutable()[1], list_type); if (left_is_counter && !right_is_counter) { - WrapExpr(cmp.right); + WrapExpr(cmp.GetChildrenMutable()[1]); } else if (right_is_counter && !left_is_counter) { - WrapExpr(cmp.left); + WrapExpr(cmp.GetChildrenMutable()[0]); } auto rewritten = RewriteComparisonToFilterCmp(input, cmp, list_type, finalized_type); if (rewritten) { @@ -472,50 +477,50 @@ static void WrapCounterRefsWithFinalize(OptimizerExtensionInput &input, LogicalO } // Rewrite BoundBetweenExpression where input is a counter column - if (e->GetExpressionClass() == ExpressionClass::BOUND_BETWEEN) { - auto &between = e->Cast(); - if (IsCounterColumnRef(*between.input, list_type)) { - WrapExpr(between.lower); - WrapExpr(between.upper); - auto counter_ref1 = ExtractCounterRef(between.input->Copy(), list_type); - auto counter_ref2 = ExtractCounterRef(std::move(between.input), list_type); - e = RewriteBetweenToFilterCmp(input, std::move(counter_ref1), std::move(counter_ref2), - std::move(between.lower), std::move(between.upper), - between.lower_inclusive, between.upper_inclusive, finalized_type, - list_type); + if (e->GetExpressionType() == ExpressionType::COMPARE_BETWEEN) { + auto &between = e->Cast(); + if (IsCounterColumnRef(*between.GetChildrenMutable()[0], list_type)) { + WrapExpr(between.GetChildrenMutable()[1]); + WrapExpr(between.GetChildrenMutable()[2]); + auto counter_ref1 = ExtractCounterRef(between.GetChildrenMutable()[0]->Copy(), list_type); + auto counter_ref2 = ExtractCounterRef(std::move(between.GetChildrenMutable()[0]), list_type); + e = RewriteBetweenToFilterCmp( + input, std::move(counter_ref1), std::move(counter_ref2), std::move(between.GetChildrenMutable()[1]), + std::move(between.GetChildrenMutable()[2]), BoundBetweenExpression::LowerInclusive(between), + BoundBetweenExpression::UpperInclusive(between), finalized_type, list_type); return; } // Non-counter BETWEEN: recurse into fields manually (not visited by EnumerateChildren) - WrapExpr(between.input); - WrapExpr(between.lower); - WrapExpr(between.upper); + WrapExpr(between.GetChildrenMutable()[0]); + WrapExpr(between.GetChildrenMutable()[1]); + WrapExpr(between.GetChildrenMutable()[2]); } ExpressionIterator::EnumerateChildren(*e, WrapExpr); - if (e->type == ExpressionType::BOUND_COLUMN_REF) { + if (e->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e->Cast(); - if (col_ref.return_type == list_type) { + if (col_ref.GetReturnType() == list_type) { PRIVACY_DEBUG_PRINT("[PAC DERIVED READ] Wrapping binding (" + - std::to_string(col_ref.binding.table_index) + "," + - std::to_string(col_ref.binding.column_index) + ") with priv_finalize"); + std::to_string(col_ref.Binding().table_index.index) + "," + + std::to_string(col_ref.Binding().column_index) + ") with priv_finalize"); e = input.optimizer.BindScalarFunction("priv_finalize", std::move(e)); } } // Retarget CAST(scalar→FLOAT[]) to CAST(scalar→FLOAT), strip no-op casts - if (e->GetExpressionClass() == ExpressionClass::BOUND_CAST) { - auto &cast = e->Cast(); - if (cast.return_type == list_type && cast.child->return_type != list_type) { - e = BoundCastExpression::AddDefaultCastToType(std::move(cast.child), finalized_type); - } else if (cast.child->return_type == cast.return_type) { - e = std::move(cast.child); + if (e->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + auto &cast = e->Cast(); + if (cast.GetReturnType() == list_type && cast.GetChildrenMutable()[0]->GetReturnType() != list_type) { + e = BoundCastExpression::AddDefaultCastToType(std::move(cast.GetChildrenMutable()[0]), finalized_type); + } else if (cast.GetChildrenMutable()[0]->GetReturnType() == cast.GetReturnType()) { + e = std::move(cast.GetChildrenMutable()[0]); } } // Convert FLOAT[] constants back to scalar - if (e->type == ExpressionType::VALUE_CONSTANT) { + if (e->GetExpressionType() == ExpressionType::VALUE_CONSTANT) { auto &constant = e->Cast(); - if (constant.return_type == list_type && !constant.value.IsNull()) { - auto &list_children = ListValue::GetChildren(constant.value); + if (constant.GetReturnType() == list_type && !constant.GetValue().IsNull()) { + auto &list_children = ListValue::GetChildren(constant.GetValue()); if (!list_children.empty()) { auto scalar_val = list_children[0].DefaultCastAs(finalized_type); e = make_uniq(std::move(scalar_val)); @@ -530,21 +535,22 @@ static void WrapCounterRefsWithFinalize(OptimizerExtensionInput &input, LogicalO auto &proj = op->Cast(); for (idx_t i = 0; i < proj.expressions.size(); i++) { if (i < proj.types.size()) { - proj.types[i] = proj.expressions[i]->return_type; + proj.types[i] = proj.expressions[i]->GetReturnType(); } // Only mark as finalized if the expression actually produces a counter type. // Expressions that CONTAIN priv_finalize in sub-expressions (e.g. CASE WHEN priv_finalize(x) > 100 ...) // may return a different type (e.g. VARCHAR) and should NOT be marked finalized. - bool is_counter_output = - (proj.expressions[i]->return_type == finalized_type) || - (proj.expressions[i]->return_type == list_type && ExpressionContainsPacFinalize(*proj.expressions[i])); + bool is_counter_output = (proj.expressions[i]->GetReturnType() == finalized_type) || + (proj.expressions[i]->GetReturnType() == list_type && + ExpressionContainsPacFinalize(*proj.expressions[i])); if (is_counter_output) { - finalized_bindings.insert(HashBinding(ColumnBinding(proj.table_index, i))); + finalized_bindings.insert( + HashBinding(ColumnBinding(TableIndex(proj.table_index.index), ProjectionIndex(i)))); if (i < proj.types.size() && proj.types[i] == list_type) { proj.types[i] = finalized_type; } - if (proj.expressions[i]->return_type == list_type) { - proj.expressions[i]->return_type = finalized_type; + if (proj.expressions[i]->GetReturnType() == list_type) { + proj.expressions[i]->SetReturnType(finalized_type); } } } @@ -582,15 +588,17 @@ static void RewriteCounterFiltersFromGets(OptimizerExtensionInput &input, unique return; } auto &get = op->Cast(); - if (!derived_table_indices.count(get.table_index)) { + if (!derived_table_indices.count(get.table_index.index)) { return; } auto list_type = CounterListType(); auto finalized_type = PacFloatLogicalType(); vector to_pull; - for (auto &entry : get.table_filters.filters) { - auto col_idx = entry.first; - if (col_idx < get.returned_types.size() && get.returned_types[col_idx] == list_type) { + const auto &col_ids = get.GetColumnIds(); + for (auto &entry : get.table_filters) { + auto col_idx = entry.GetIndex(); + // DuckDB 2.0 keys filters by scan projection, not physical table column. + if (col_idx < col_ids.size() && get.GetColumnType(col_ids[col_idx]) == list_type) { to_pull.push_back(col_idx); } } @@ -598,43 +606,47 @@ static void RewriteCounterFiltersFromGets(OptimizerExtensionInput &input, unique return; } auto out_bindings = op->GetColumnBindings(); - auto &col_ids = get.GetColumnIds(); vector> filter_exprs; for (auto filter_col_idx : to_pull) { - auto &filter = get.table_filters.filters[filter_col_idx]; - auto col_type = get.returned_types[filter_col_idx]; + auto &filter = get.table_filters.GetFilterByColumnIndexMutable(ProjectionIndex(filter_col_idx)); + auto col_type = get.GetColumnType(col_ids[filter_col_idx]); // Find the output binding for this column ColumnBinding binding; bool found = false; for (idx_t i = 0; i < col_ids.size() && i < out_bindings.size(); i++) { - if (col_ids[i].GetPrimaryIndex() == filter_col_idx) { + if ((get.projection_ids.empty() ? i : idx_t(get.projection_ids[i])) == filter_col_idx) { binding = out_bindings[i]; found = true; break; } } if (!found) { - get.table_filters.filters.erase(filter_col_idx); - continue; + // Filter-only columns may have been pruned from the scan output. Expose + // them again so the pulled-up privacy predicate can evaluate them. + D_ASSERT(!get.projection_ids.empty()); + get.projection_ids.emplace_back(filter_col_idx); + binding = ColumnBinding(get.table_index, ProjectionIndex(filter_col_idx)); + get.ResolveOperatorTypes(); } auto col_ref = make_uniq(col_type, binding); - auto expr = filter->ToExpression(*col_ref); + auto expr = filter.ToExpression(*col_ref); // Rewrite the expression to priv_filter_ calls - if (expr->GetExpressionClass() == ExpressionClass::BOUND_BETWEEN) { - auto &between = expr->Cast(); + if (expr->GetExpressionType() == ExpressionType::COMPARE_BETWEEN) { + auto &between = expr->Cast(); auto counter1 = make_uniq(col_type, binding); auto counter2 = make_uniq(col_type, binding); filter_exprs.push_back(RewriteBetweenToFilterCmp( - input, std::move(counter1), std::move(counter2), std::move(between.lower), std::move(between.upper), - between.lower_inclusive, between.upper_inclusive, finalized_type, list_type)); - } else if (expr->GetExpressionClass() == ExpressionClass::BOUND_COMPARISON) { - auto &cmp = expr->Cast(); + input, std::move(counter1), std::move(counter2), std::move(between.GetChildrenMutable()[1]), + std::move(between.GetChildrenMutable()[2]), BoundBetweenExpression::LowerInclusive(between), + BoundBetweenExpression::UpperInclusive(between), finalized_type, list_type)); + } else if (BoundComparisonExpression::IsComparison(*expr)) { + auto &cmp = expr->Cast(); auto rewritten = RewriteComparisonToFilterCmp(input, cmp, list_type, finalized_type); filter_exprs.push_back(rewritten ? std::move(rewritten) : std::move(expr)); } else { filter_exprs.push_back(std::move(expr)); } - get.table_filters.filters.erase(filter_col_idx); + get.table_filters.RemoveFilterByColumnIndex(ProjectionIndex(filter_col_idx)); PRIVACY_DEBUG_PRINT("[PAC DERIVED READ] Rewrote table_filter to priv_filter_ for counter column " + std::to_string(filter_col_idx)); } diff --git a/src/query_processing/pac_projection_propagation.cpp b/src/query_processing/pac_projection_propagation.cpp index 4148f95c..91048589 100644 --- a/src/query_processing/pac_projection_propagation.cpp +++ b/src/query_processing/pac_projection_propagation.cpp @@ -13,6 +13,8 @@ #include "duckdb/planner/operator/logical_filter.hpp" #include "duckdb/planner/operator/logical_comparison_join.hpp" #include "duckdb/planner/operator/logical_delim_get.hpp" +#include "duckdb/planner/operator/logical_materialized_cte.hpp" +#include "duckdb/planner/operator/logical_cteref.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/expression_iterator.hpp" @@ -41,7 +43,7 @@ static bool FindDirectPathToSource(LogicalOperator *current, idx_t target_table_ // Check if this operator matches the target table_index if (current->type == LogicalOperatorType::LOGICAL_GET) { auto &get = current->Cast(); - if (get.table_index == target_table_index) { + if (get.table_index.index == target_table_index) { return true; } } else if (current->type == LogicalOperatorType::LOGICAL_CTE_REF) { @@ -50,17 +52,22 @@ static bool FindDirectPathToSource(LogicalOperator *current, idx_t target_table_ } } else if (current->type == LogicalOperatorType::LOGICAL_PROJECTION) { auto &proj = current->Cast(); - if (proj.table_index == target_table_index) { + if (proj.table_index.index == target_table_index) { return true; } } + else if (current->type == LogicalOperatorType::LOGICAL_CTE_REF && + current->Cast().table_index.index == target_table_index) { + return true; + } + // If this is a nested aggregate, check if it IS the target or if the target is below it. // Nested aggregates don't pass through bindings, but PropagateSingleBinding handles // them by adding the binding as a group column — so we continue the search. if (!is_start && current->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { auto &agg = current->Cast(); - if (agg.group_index == target_table_index || agg.aggregate_index == target_table_index) { + if (agg.group_index.index == target_table_index || agg.aggregate_index.index == target_table_index) { return true; } // Continue searching through the aggregate's children — PropagateSingleBinding @@ -81,7 +88,7 @@ static bool FindDirectPathToSource(LogicalOperator *current, idx_t target_table_ } // Ensure a single column_index is present in a projection map. Returns true if it was already there. -static bool EnsureInProjectionMap(vector &proj_map, idx_t column_index) { +static bool EnsureInProjectionMap(vector &proj_map, ProjectionIndex column_index) { if (proj_map.empty()) { return true; // Empty map means all columns pass through } @@ -98,20 +105,67 @@ static bool EnsureInProjectionMap(vector &proj_map, idx_t column_index) { static bool BindingInOutput(LogicalOperator &op, const ColumnBinding &binding) { auto bindings = op.GetColumnBindings(); for (auto &b : bindings) { - if (b.table_index == binding.table_index && b.column_index == binding.column_index) { + if (b.table_index.index == binding.table_index.index && b.column_index == binding.column_index) { return true; } } return false; } -ColumnBinding PropagateSingleBindingToOperator(LogicalOperator &plan_root, idx_t source_table_index, - ColumnBinding source_binding, const LogicalType &source_type, - LogicalOperator *target) { - ColumnBinding invalid(DConstants::INVALID_INDEX, DConstants::INVALID_INDEX); +// Find a CTE reader on the output tuple stream, without crossing an aggregation +// or the definition side of another CTE. Correlation-key deduplication must not +// acquire the entity mask as an extra grouping key. +static LogicalCTERef *FindOutputCTERef(LogicalOperator &op, TableIndex cte_index) { + if (op.type == LogicalOperatorType::LOGICAL_CTE_REF) { + auto &ref = op.Cast(); + return ref.cte_index == cte_index ? &ref : nullptr; + } + if (op.type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { + return nullptr; + } + if (op.type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE) { + return FindOutputCTERef(*op.children[1], cte_index); + } + for (idx_t i = 0; i < op.children.size(); i++) { + if (IsJoinWithProjectionMap(op.type)) { + auto join_type = op.Cast().join_type; + if ((i == 1 && + (join_type == JoinType::SEMI || join_type == JoinType::ANTI || join_type == JoinType::MARK)) || + (i == 0 && (join_type == JoinType::RIGHT_SEMI || join_type == JoinType::RIGHT_ANTI))) { + continue; + } + } + auto *ref = FindOutputCTERef(*op.children[i], cte_index); + if (ref) { + return ref; + } + } + return nullptr; +} + +static void SyncCTERefColumns(LogicalOperator &op, TableIndex cte_index, const vector &types) { + if (op.type == LogicalOperatorType::LOGICAL_CTE_REF) { + auto &ref = op.Cast(); + if (ref.cte_index == cte_index) { + while (ref.chunk_types.size() < types.size()) { + ref.bound_columns.emplace_back("__privacy_mask_" + std::to_string(ref.chunk_types.size())); + ref.chunk_types.push_back(types[ref.chunk_types.size()]); + } + ref.ResolveOperatorTypes(); + } + } + for (auto &child : op.children) { + SyncCTERefColumns(*child, cte_index, types); + } +} + +static ColumnBinding PropagateBinding(LogicalOperator &plan_root, idx_t source_table_index, + ColumnBinding source_binding, const LogicalType &source_type, + LogicalOperator *target, bool include_target) { + ColumnBinding invalid {TableIndex(DConstants::INVALID_INDEX), ProjectionIndex(DConstants::INVALID_INDEX)}; vector path_ops; - if (!FindDirectPathToSource(target, source_table_index, path_ops, true)) { + if (!FindDirectPathToSource(target, source_table_index, path_ops, !include_target)) { #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("PropagateSingleBinding: No direct path from aggregate to source #" + std::to_string(source_table_index)); @@ -137,9 +191,9 @@ ColumnBinding PropagateSingleBindingToOperator(LogicalOperator &plan_root, idx_t // Check if this binding is already in the projection's expressions idx_t existing_idx = DConstants::INVALID_INDEX; for (idx_t i = 0; i < proj.expressions.size(); i++) { - if (proj.expressions[i]->type == ExpressionType::BOUND_COLUMN_REF) { + if (proj.expressions[i]->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &expr_ref = proj.expressions[i]->Cast(); - if (expr_ref.binding == current) { + if (expr_ref.BindingMutable() == current) { existing_idx = i; break; } @@ -147,15 +201,41 @@ ColumnBinding PropagateSingleBindingToOperator(LogicalOperator &plan_root, idx_t } if (existing_idx != DConstants::INVALID_INDEX) { - current = ColumnBinding(proj.table_index, existing_idx); + current = ColumnBinding(TableIndex(proj.table_index.index), ProjectionIndex(existing_idx)); } else { // Add a passthrough column ref for this binding auto col_ref = make_uniq(source_type, current); proj.expressions.push_back(std::move(col_ref)); - current = ColumnBinding(proj.table_index, proj.expressions.size() - 1); + current = + ColumnBinding(TableIndex(proj.table_index.index), ProjectionIndex(proj.expressions.size() - 1)); } proj.ResolveOperatorTypes(); + } else if (op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE && child_idx == 0) { + auto &cte = op->Cast(); + auto *ref = FindOutputCTERef(*cte.children[1], cte.table_index); + if (!ref) { + return invalid; + } + cte.children[0]->ResolveOperatorTypes(); + auto bindings = cte.children[0]->GetColumnBindings(); + auto column = std::find(bindings.begin(), bindings.end(), current); + if (column == bindings.end()) { + return invalid; + } + auto position = idx_t(column - bindings.begin()); + cte.column_count = bindings.size(); + SyncCTERefColumns(*cte.children[1], cte.table_index, cte.children[0]->types); + current = PropagateBinding(plan_root, ref->table_index.index, + ColumnBinding(ref->table_index, ProjectionIndex(position)), source_type, + cte.children[1].get(), true); + if (current.table_index.index == DConstants::INVALID_INDEX) { + return invalid; + } + PRIVACY_DEBUG_PRINT("PropagateSingleBinding: propagated mask through CTE #" + + std::to_string(cte.table_index.index)); + cte.ResolveOperatorTypes(); + } else if (IsJoinWithProjectionMap(op->type)) { auto &join = op->Cast(); bool is_delim_join = (op->type == LogicalOperatorType::LOGICAL_DELIM_JOIN); @@ -190,8 +270,8 @@ ColumnBinding PropagateSingleBindingToOperator(LogicalOperator &plan_root, idx_t if (!BindingInOutput(join, current)) { #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("PropagateSingleBinding: WARNING - binding [" + - std::to_string(current.table_index) + "." + std::to_string(current.column_index) + - "] not found in join output"); + std::to_string(current.table_index.index) + "." + + std::to_string(current.column_index) + "] not found in join output"); #endif } // Binding identity preserved through joins @@ -207,10 +287,10 @@ ColumnBinding PropagateSingleBindingToOperator(LogicalOperator &plan_root, idx_t // Check if the binding already passes through as a group column bool found_in_groups = false; for (idx_t i = 0; i < agg.groups.size(); i++) { - if (agg.groups[i]->type == ExpressionType::BOUND_COLUMN_REF) { + if (agg.groups[i]->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = agg.groups[i]->Cast(); - if (col_ref.binding == current) { - current = ColumnBinding(agg.group_index, i); + if (col_ref.BindingMutable() == current) { + current = ColumnBinding(TableIndex(agg.group_index.index), ProjectionIndex(i)); found_in_groups = true; break; } @@ -229,6 +309,12 @@ ColumnBinding PropagateSingleBindingToOperator(LogicalOperator &plan_root, idx_t return current; } +ColumnBinding PropagateSingleBindingToOperator(LogicalOperator &plan_root, idx_t source_table_index, + ColumnBinding source_binding, const LogicalType &source_type, + LogicalOperator *target) { + return PropagateBinding(plan_root, source_table_index, source_binding, source_type, target, false); +} + ColumnBinding PropagateSingleBinding(LogicalOperator &plan_root, idx_t source_table_index, ColumnBinding source_binding, const LogicalType &source_type, LogicalAggregate *target_agg) { return PropagateSingleBindingToOperator(plan_root, source_table_index, source_binding, source_type, target_agg); diff --git a/src/query_processing/pac_subquery_handler.cpp b/src/query_processing/pac_subquery_handler.cpp index f8a15743..14c2875a 100644 --- a/src/query_processing/pac_subquery_handler.cpp +++ b/src/query_processing/pac_subquery_handler.cpp @@ -26,15 +26,15 @@ namespace duckdb { // Traces a binding from the source operator up through the tree, modifying operators as needed. static ColumnBinding EnsureBindingFlowsThroughRecursive(LogicalOperator *op, idx_t source_table_index, ColumnBinding source_binding, const LogicalType &source_type) { - ColumnBinding invalid(DConstants::INVALID_INDEX, DConstants::INVALID_INDEX); + ColumnBinding invalid {TableIndex(DConstants::INVALID_INDEX), ProjectionIndex(DConstants::INVALID_INDEX)}; // Base case: match any operator by table_index if (op->type == LogicalOperatorType::LOGICAL_GET) { - if (op->Cast().table_index == source_table_index) { + if (op->Cast().table_index.index == source_table_index) { return source_binding; } } else if (op->type == LogicalOperatorType::LOGICAL_PROJECTION) { - if (op->Cast().table_index == source_table_index) { + if (op->Cast().table_index.index == source_table_index) { return source_binding; } } @@ -43,12 +43,12 @@ static ColumnBinding EnsureBindingFlowsThroughRecursive(LogicalOperator *op, idx ColumnBinding child_result = invalid; for (auto &child : op->children) { child_result = EnsureBindingFlowsThroughRecursive(child.get(), source_table_index, source_binding, source_type); - if (child_result.table_index != DConstants::INVALID_INDEX) { + if (child_result.table_index.index != DConstants::INVALID_INDEX) { break; } } - if (child_result.table_index == DConstants::INVALID_INDEX) { + if (child_result.table_index.index == DConstants::INVALID_INDEX) { return child_result; } @@ -59,10 +59,10 @@ static ColumnBinding EnsureBindingFlowsThroughRecursive(LogicalOperator *op, idx case LogicalOperatorType::LOGICAL_PROJECTION: { auto &proj = op->Cast(); for (idx_t i = 0; i < proj.expressions.size(); i++) { - if (proj.expressions[i]->type == ExpressionType::BOUND_COLUMN_REF) { + if (proj.expressions[i]->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = proj.expressions[i]->Cast(); - if (col_ref.binding == child_result) { - return ColumnBinding(proj.table_index, i); + if (col_ref.BindingMutable() == child_result) { + return ColumnBinding(TableIndex(proj.table_index.index), ProjectionIndex(i)); } } } @@ -72,10 +72,10 @@ static ColumnBinding EnsureBindingFlowsThroughRecursive(LogicalOperator *op, idx auto &agg = op->Cast(); // Check if binding is already in groups for (idx_t i = 0; i < agg.groups.size(); i++) { - if (agg.groups[i]->type == ExpressionType::BOUND_COLUMN_REF) { + if (agg.groups[i]->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = agg.groups[i]->Cast(); - if (col_ref.binding == child_result) { - return ColumnBinding(agg.group_index, i); + if (col_ref.BindingMutable() == child_result) { + return ColumnBinding(TableIndex(agg.group_index.index), ProjectionIndex(i)); } } } @@ -84,7 +84,7 @@ static ColumnBinding EnsureBindingFlowsThroughRecursive(LogicalOperator *op, idx idx_t new_group_idx = agg.groups.size(); agg.groups.push_back(std::move(group_col_ref)); agg.ResolveOperatorTypes(); - return ColumnBinding(agg.group_index, new_group_idx); + return ColumnBinding(TableIndex(agg.group_index.index), ProjectionIndex(new_group_idx)); } default: return child_result; @@ -105,11 +105,11 @@ static bool HasSourceTableInSubtree(LogicalOperator *op, idx_t source_table_inde return false; } if (op->type == LogicalOperatorType::LOGICAL_GET) { - if (op->Cast().table_index == source_table_index) { + if (op->Cast().table_index.index == source_table_index) { return true; } } else if (op->type == LogicalOperatorType::LOGICAL_PROJECTION) { - if (op->Cast().table_index == source_table_index) { + if (op->Cast().table_index.index == source_table_index) { return true; } } @@ -183,13 +183,13 @@ static void UpdateDelimGetsInSubtree(LogicalOperator *op, const LogicalType &sou if (op->type == LogicalOperatorType::LOGICAL_DELIM_GET) { auto &dg = op->Cast(); #if PRIVACY_DEBUG - PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: Updating DELIM_GET #" + std::to_string(dg.table_index) + + PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: Updating DELIM_GET #" + std::to_string(dg.table_index.index) + " chunk_types before=" + std::to_string(dg.chunk_types.size())); #endif dg.chunk_types.push_back(source_type); op->ResolveOperatorTypes(); #if PRIVACY_DEBUG - PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: DELIM_GET #" + std::to_string(dg.table_index) + + PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: DELIM_GET #" + std::to_string(dg.table_index.index) + " chunk_types after=" + std::to_string(dg.chunk_types.size()) + " types after=" + std::to_string(dg.types.size())); #endif @@ -204,7 +204,8 @@ DelimColumnResult AddBindingToDelimJoin(unique_ptr &plan, idx_t ColumnBinding source_binding, const LogicalType &source_type, LogicalAggregate *target_agg) { DelimColumnResult invalid_result; - invalid_result.binding = ColumnBinding(DConstants::INVALID_INDEX, DConstants::INVALID_INDEX); + invalid_result.binding = + ColumnBinding(TableIndex(DConstants::INVALID_INDEX), ProjectionIndex(DConstants::INVALID_INDEX)); invalid_result.type = LogicalType::INVALID; auto *delim_join = FindDelimJoinForSource(plan.get(), source_table_index, target_agg); @@ -217,9 +218,10 @@ DelimColumnResult AddBindingToDelimJoin(unique_ptr &plan, idx_t } #if PRIVACY_DEBUG - PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: Found DELIM_JOIN, source_table_index=" + - std::to_string(source_table_index) + " binding=[" + std::to_string(source_binding.table_index) + - "." + std::to_string(source_binding.column_index) + "]"); + PRIVACY_DEBUG_PRINT( + "AddBindingToDelimJoin: Found DELIM_JOIN, source_table_index=" + std::to_string(source_table_index) + + " binding=[" + std::to_string(source_binding.table_index.index) + "." + + std::to_string(source_binding.column_index) + "]"); PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: Left child type=" + std::to_string(static_cast(delim_join->children[0]->type))); #endif @@ -229,7 +231,7 @@ DelimColumnResult AddBindingToDelimJoin(unique_ptr &plan, idx_t ColumnBinding output_binding = EnsureBindingFlowsThrough(left_child, source_table_index, source_binding, source_type); - if (output_binding.table_index == DConstants::INVALID_INDEX) { + if (output_binding.table_index.index == DConstants::INVALID_INDEX) { #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: EnsureBindingFlowsThrough failed"); #endif @@ -237,20 +239,21 @@ DelimColumnResult AddBindingToDelimJoin(unique_ptr &plan, idx_t } #if PRIVACY_DEBUG - PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: output_binding=[" + std::to_string(output_binding.table_index) + "." + - std::to_string(output_binding.column_index) + "]"); + PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: output_binding=[" + std::to_string(output_binding.table_index.index) + + "." + std::to_string(output_binding.column_index) + "]"); PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: duplicate_eliminated_columns.size()=" + std::to_string(delim_join->duplicate_eliminated_columns.size())); for (idx_t i = 0; i < delim_join->duplicate_eliminated_columns.size(); i++) { auto &dec = delim_join->duplicate_eliminated_columns[i]; PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: existing dup_elim[" + std::to_string(i) + "] = " + dec->ToString() + - " type=" + dec->return_type.ToString()); + " type=" + dec->GetReturnType().ToString()); } // Dump DELIM_JOIN conditions auto &dj = delim_join->Cast(); for (idx_t i = 0; i < dj.conditions.size(); i++) { - PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: condition[" + std::to_string(i) + "] left=" + - dj.conditions[i].left->ToString() + " right=" + dj.conditions[i].right->ToString()); + PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: condition[" + std::to_string(i) + + "] left=" + dj.conditions[i].LeftReference()->ToString() + + " right=" + dj.conditions[i].RightReference()->ToString()); } #endif @@ -271,13 +274,13 @@ DelimColumnResult AddBindingToDelimJoin(unique_ptr &plan, idx_t } #if PRIVACY_DEBUG - PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: DELIM_GET #" + std::to_string(delim_get->table_index) + " now has " + - std::to_string(delim_get->chunk_types.size()) + " chunk_types, " + + PRIVACY_DEBUG_PRINT("AddBindingToDelimJoin: DELIM_GET #" + std::to_string(delim_get->table_index.index) + + " now has " + std::to_string(delim_get->chunk_types.size()) + " chunk_types, " + std::to_string(delim_get->types.size()) + " types"); #endif DelimColumnResult result; - result.binding = ColumnBinding(delim_get->table_index, new_col_idx); + result.binding = ColumnBinding(TableIndex(delim_get->table_index.index), ProjectionIndex(new_col_idx)); result.type = source_type; return result; } @@ -286,7 +289,8 @@ DelimColumnResult AddBindingToDelimJoin(unique_ptr &plan, idx_t DelimColumnResult AddColumnToDelimJoin(unique_ptr &plan, LogicalGet &source_get, const string &column_name, LogicalAggregate *target_agg) { DelimColumnResult invalid_result; - invalid_result.binding = ColumnBinding(DConstants::INVALID_INDEX, DConstants::INVALID_INDEX); + invalid_result.binding = + ColumnBinding(TableIndex(DConstants::INVALID_INDEX), ProjectionIndex(DConstants::INVALID_INDEX)); invalid_result.type = LogicalType::INVALID; // Ensure the column is projected in source_get @@ -312,14 +316,14 @@ DelimColumnResult AddColumnToDelimJoin(unique_ptr &plan, Logica } } if (output_col_idx == DConstants::INVALID_INDEX) { - source_get.projection_ids.push_back(col_proj_idx); + source_get.projection_ids.push_back(ProjectionIndex(col_proj_idx)); output_col_idx = col_proj_idx; source_get.ResolveOperatorTypes(); } } - ColumnBinding source_binding(source_get.table_index, output_col_idx); - return AddBindingToDelimJoin(plan, source_get.table_index, source_binding, col_type, target_agg); + ColumnBinding source_binding {TableIndex(source_get.table_index.index), ProjectionIndex(output_col_idx)}; + return AddBindingToDelimJoin(plan, source_get.table_index.index, source_binding, col_type, target_agg); } } // namespace duckdb diff --git a/src/query_processing/pac_topk_rewriter.cpp b/src/query_processing/pac_topk_rewriter.cpp index 13e1085f..34c33cb4 100644 --- a/src/query_processing/pac_topk_rewriter.cpp +++ b/src/query_processing/pac_topk_rewriter.cpp @@ -5,6 +5,7 @@ // See pac_topk_rewriter.hpp for design documentation. // +#include "duckdb/common/vector/list_vector.hpp" #include "query_processing/pac_topk_rewriter.hpp" #include "privacy_debug.hpp" #include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" @@ -41,8 +42,8 @@ static void PacMeanFunction(DataChunk &args, ExpressionState &state, Vector &res UnifiedVectorFormat list_data; list_vec.ToUnifiedFormat(count, list_data); - auto result_data = FlatVector::GetData(result); - auto &result_validity = FlatVector::Validity(result); + auto result_data = FlatVector::GetDataMutable(result); + auto &result_validity = FlatVector::ValidityMutable(result); auto &child_vec = ListVector::GetEntry(list_vec); UnifiedVectorFormat child_data; @@ -108,11 +109,11 @@ struct PacTopKAggInfo { }; static void RemapExpressionTableIndices(Expression &expr, const std::unordered_map &index_map) { - if (expr.type == ExpressionType::BOUND_COLUMN_REF) { + if (expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr.Cast(); - auto it = index_map.find(col_ref.binding.table_index); + auto it = index_map.find(col_ref.BindingMutable().table_index.index); if (it != index_map.end()) { - col_ref.binding.table_index = it->second; + col_ref.BindingMutable().table_index.index = it->second; } } ExpressionIterator::EnumerateChildren(expr, @@ -127,49 +128,49 @@ static unique_ptr CopyExpressionWithTableMap(const Expression &expr, } static Expression *StripCastsMutable(Expression *expr) { - while (expr && expr->type == ExpressionType::OPERATOR_CAST) { - expr = expr->Cast().child.get(); + while (expr && expr->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + expr = expr->Cast().GetChildrenMutable()[0].get(); } return expr; } static const Expression *StripCastsConst(const Expression *expr) { - while (expr && expr->type == ExpressionType::OPERATOR_CAST) { - expr = expr->Cast().child.get(); + while (expr && expr->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + expr = expr->Cast().GetChildren()[0].get(); } return expr; } static bool GetDirectColumnBinding(const Expression &expr, ColumnBinding &binding) { auto stripped = StripCastsConst(&expr); - if (!stripped || stripped->type != ExpressionType::BOUND_COLUMN_REF) { + if (!stripped || stripped->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) { return false; } - binding = stripped->Cast().binding; + binding = stripped->Cast().Binding(); return true; } static bool RemapDirectOrderExpression(unique_ptr &expr, const std::unordered_map &map) { auto *stripped = StripCastsMutable(expr.get()); - if (!stripped || stripped->type != ExpressionType::BOUND_COLUMN_REF) { + if (!stripped || stripped->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) { return false; } auto &ref = stripped->Cast(); - auto it = map.find(HashBinding(ref.binding)); + auto it = map.find(HashBinding(ref.BindingMutable())); if (it == map.end()) { return false; } - ref.binding = it->second; + ref.BindingMutable() = it->second; return true; } static void RemapExpressionBindingsExact(Expression &expr, const std::unordered_map &map) { - if (expr.type == ExpressionType::BOUND_COLUMN_REF) { + if (expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr.Cast(); - auto it = map.find(HashBinding(col_ref.binding)); + auto it = map.find(HashBinding(col_ref.BindingMutable())); if (it != map.end()) { - col_ref.binding = it->second; + col_ref.BindingMutable() = it->second; } } ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { RemapExpressionBindingsExact(child, map); }); @@ -262,11 +263,12 @@ static TopKContext FindTopKContext(const unique_ptr &plan) { // tracing through any number of intermediate projections. static bool OrderExprReferencesPacAgg(Expression &expr, const vector &pac_aggs, const vector &projections, idx_t proj_depth) { - if (expr.type == ExpressionType::BOUND_COLUMN_REF) { + if (expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr.Cast(); // If there are more projections to trace through, resolve the column ref - if (proj_depth < projections.size() && col_ref.binding.table_index == projections[proj_depth]->table_index) { - auto col_idx = col_ref.binding.column_index; + if (proj_depth < projections.size() && + col_ref.BindingMutable().table_index.index == projections[proj_depth]->table_index.index) { + auto col_idx = col_ref.BindingMutable().column_index; if (col_idx < projections[proj_depth]->expressions.size()) { auto &proj_expr = projections[proj_depth]->expressions[col_idx]; return OrderExprReferencesPacAgg(*proj_expr, pac_aggs, projections, proj_depth + 1); @@ -274,15 +276,15 @@ static bool OrderExprReferencesPacAgg(Expression &expr, const vector(); - return OrderExprReferencesPacAgg(*cast.child, pac_aggs, projections, proj_depth); + if (expr.GetExpressionType() == ExpressionType::OPERATOR_CAST) { + auto &cast = expr.Cast(); + return OrderExprReferencesPacAgg(*cast.GetChildrenMutable()[0], pac_aggs, projections, proj_depth); } return false; } @@ -292,16 +294,16 @@ static vector FindPacAggregates(LogicalAggregate &agg) { vector result; for (idx_t i = 0; i < agg.expressions.size(); i++) { auto &expr = agg.expressions[i]; - if (expr->type != ExpressionType::BOUND_AGGREGATE) { + if (expr->GetExpressionType() != ExpressionType::BOUND_AGGREGATE) { continue; } auto &bound_agg = expr->Cast(); - if (IsPacAggregate(bound_agg.function.name)) { + if (IsPacAggregate(bound_agg.FunctionMutable().GetName().GetIdentifierName())) { PacTopKAggInfo info; info.agg_index = i; - info.original_name = bound_agg.function.name; - info.original_type = expr->return_type; - info.agg_binding = ColumnBinding(agg.aggregate_index, i); + info.original_name = bound_agg.FunctionMutable().GetName().GetIdentifierName(); + info.original_type = expr->GetReturnType(); + info.agg_binding = ColumnBinding(TableIndex(agg.aggregate_index.index), ProjectionIndex(i)); result.push_back(std::move(info)); } } @@ -322,7 +324,7 @@ static bool TraceBindingThroughProjections(const ColumnBinding &binding, const v return true; } auto *proj = projections[depth]; - if (binding.table_index != proj->table_index) { + if (binding.table_index.index != proj->table_index.index) { out_binding = binding; return true; } @@ -331,11 +333,11 @@ static bool TraceBindingThroughProjections(const ColumnBinding &binding, const v return false; } auto &proj_expr = proj->expressions[col_idx]; - if (proj_expr->type != ExpressionType::BOUND_COLUMN_REF) { + if (proj_expr->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) { return false; } auto &inner_ref = proj_expr->Cast(); - return TraceBindingThroughProjections(inner_ref.binding, projections, depth + 1, out_binding); + return TraceBindingThroughProjections(inner_ref.BindingMutable(), projections, depth + 1, out_binding); } // Remap column bindings in an operator's expressions after a child was rewritten. @@ -359,11 +361,11 @@ static void RemapBindingsInOperator(LogicalOperator &op, const vector &)> Rewrite = [&](unique_ptr &e) { - if (e->type == ExpressionType::BOUND_COLUMN_REF) { + if (e->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e->Cast(); - auto it = remap.find(HashBinding(col_ref.binding)); + auto it = remap.find(HashBinding(col_ref.BindingMutable())); if (it != remap.end()) { - col_ref.binding = it->second; + col_ref.BindingMutable() = it->second; } } ExpressionIterator::EnumerateChildren(*e, Rewrite); @@ -385,8 +387,8 @@ static void RemapBindingsInOperator(LogicalOperator &op, const vector(); for (auto &cond : join.conditions) { - Rewrite(cond.left); - Rewrite(cond.right); + Rewrite(cond.LeftReference()); + Rewrite(cond.RightReference()); } } @@ -415,7 +417,7 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq std::unordered_set pac_agg_indices; for (auto &info : pac_aggs) { if (info.agg_index >= agg.expressions.size() || - agg.expressions[info.agg_index]->type != ExpressionType::BOUND_AGGREGATE) { + agg.expressions[info.agg_index]->GetExpressionType() != ExpressionType::BOUND_AGGREGATE) { return false; } auto &bound_agg = agg.expressions[info.agg_index]->Cast(); @@ -430,17 +432,19 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq for (auto &order : topn.orders) { ColumnBinding proj_binding; if (!GetDirectColumnBinding(*order.expression, proj_binding) || - proj_binding.table_index != proj.table_index || proj_binding.column_index >= proj.expressions.size()) { + proj_binding.table_index.index != proj.table_index.index || + proj_binding.column_index >= proj.expressions.size()) { return false; } ColumnBinding agg_binding; if (!GetDirectColumnBinding(*proj.expressions[proj_binding.column_index], agg_binding)) { return false; } - if (agg_binding.table_index == agg.group_index) { + if (agg_binding.table_index.index == agg.group_index.index) { continue; } - if (agg_binding.table_index != agg.aggregate_index || agg_binding.column_index >= agg.expressions.size() || + if (agg_binding.table_index.index != agg.aggregate_index.index || + agg_binding.column_index >= agg.expressions.size() || pac_agg_indices.find(agg_binding.column_index) == pac_agg_indices.end()) { return false; } @@ -460,7 +464,7 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq for (idx_t pi = 0; pi < proj.expressions.size(); pi++) { ColumnBinding binding; if (GetDirectColumnBinding(*proj.expressions[pi], binding) && - binding == ColumnBinding(agg.group_index, gi)) { + binding == ColumnBinding(TableIndex(agg.group_index.index), ProjectionIndex(gi))) { group_proj_cols.push_back(pi); found = true; break; @@ -495,9 +499,9 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq #endif auto &binder = input.optimizer.binder; - auto original_group_index = agg.group_index; - auto original_agg_index = agg.aggregate_index; - auto original_proj_index = proj.table_index; + auto original_group_index = agg.group_index.index; + auto original_agg_index = agg.aggregate_index.index; + auto original_proj_index = proj.table_index.index; vector original_agg_types = agg.types; vector original_proj_types = proj.types; @@ -533,18 +537,18 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq std::unordered_map agg_to_top_remap; for (idx_t gi = 0; gi < agg.groups.size(); gi++) { - agg_to_top_remap[HashBinding(ColumnBinding(original_group_index, gi))] = - ColumnBinding(original_group_index, gi); + agg_to_top_remap[HashBinding(ColumnBinding(TableIndex(original_group_index), ProjectionIndex(gi)))] = + ColumnBinding(TableIndex(original_group_index), ProjectionIndex(gi)); } for (auto &kv : order_old_to_new) { - agg_to_top_remap[HashBinding(ColumnBinding(original_agg_index, kv.first))] = - ColumnBinding(original_agg_index, kv.second); + agg_to_top_remap[HashBinding(ColumnBinding(TableIndex(original_agg_index), ProjectionIndex(kv.first)))] = + ColumnBinding(TableIndex(original_agg_index), ProjectionIndex(kv.second)); } agg.expressions = std::move(top_expressions); agg.types.clear(); for (auto &group : agg.groups) { - agg.types.push_back(group->return_type); + agg.types.push_back(group->GetReturnType()); } for (auto old_idx : order_agg_indices) { agg.types.push_back(original_agg_types[agg.groups.size() + old_idx]); @@ -566,8 +570,8 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq } std::unordered_map proj_to_top_remap; for (auto &kv : proj_old_to_new) { - proj_to_top_remap[HashBinding(ColumnBinding(original_proj_index, kv.first))] = - ColumnBinding(original_proj_index, kv.second); + proj_to_top_remap[HashBinding(ColumnBinding(TableIndex(original_proj_index), ProjectionIndex(kv.first)))] = + ColumnBinding(TableIndex(original_proj_index), ProjectionIndex(kv.second)); } for (auto &order : topn.orders) { if (!RemapDirectOrderExpression(order.expression, proj_to_top_remap)) { @@ -581,12 +585,13 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq auto left_topn = std::move(plan); auto join = make_uniq(JoinType::INNER); for (idx_t gi = 0; gi < group_proj_cols.size(); gi++) { - JoinCondition cond; + JoinCondition cond(nullptr, nullptr, ExpressionType::COMPARE_NOT_DISTINCT_FROM); auto left_col = proj_old_to_new[group_proj_cols[gi]]; - cond.left = make_uniq(original_proj_types[group_proj_cols[gi]], - ColumnBinding(original_proj_index, left_col)); - cond.right = CopyExpressionWithTableMap(*original_groups[gi], right_index_map); - cond.comparison = ExpressionType::COMPARE_NOT_DISTINCT_FROM; + cond.LeftReference() = make_uniq( + original_proj_types[group_proj_cols[gi]], + ColumnBinding(TableIndex(original_proj_index), ProjectionIndex(left_col))); + cond.RightReference() = CopyExpressionWithTableMap(*original_groups[gi], right_index_map); + join->conditions.push_back(std::move(cond)); } join->children.push_back(std::move(left_topn)); @@ -601,30 +606,32 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq payload_expressions.push_back(CopyExpressionWithTableMap(*original_aggs[old_idx], right_index_map)); } - auto payload_group_index = binder.GenerateTableIndex(); - auto payload_agg_index = binder.GenerateTableIndex(); + auto payload_group_index = binder.GenerateTableIndex().index; + auto payload_agg_index = binder.GenerateTableIndex().index; vector> payload_groups; for (auto old_col : group_proj_cols) { auto new_col = proj_old_to_new[old_col]; - payload_groups.push_back(make_uniq(original_proj_types[old_col], - ColumnBinding(original_proj_index, new_col))); + payload_groups.push_back(make_uniq( + original_proj_types[old_col], + ColumnBinding(TableIndex(original_proj_index), ProjectionIndex(new_col)))); } for (auto old_col : order_proj_cols) { auto new_col = proj_old_to_new[old_col]; - payload_groups.push_back(make_uniq(original_proj_types[old_col], - ColumnBinding(original_proj_index, new_col))); + payload_groups.push_back(make_uniq( + original_proj_types[old_col], + ColumnBinding(TableIndex(original_proj_index), ProjectionIndex(new_col)))); } - auto payload_agg = - make_uniq(payload_group_index, payload_agg_index, std::move(payload_expressions)); + auto payload_agg = make_uniq(TableIndex(payload_group_index), TableIndex(payload_agg_index), + std::move(payload_expressions)); payload_agg->groups = std::move(payload_groups); payload_agg->children.push_back(std::move(join)); payload_agg->ResolveOperatorTypes(); std::unordered_map final_expr_remap; for (idx_t gi = 0; gi < group_proj_cols.size(); gi++) { - final_expr_remap[HashBinding(ColumnBinding(original_group_index, gi))] = - ColumnBinding(payload_group_index, gi); + final_expr_remap[HashBinding(ColumnBinding(TableIndex(original_group_index), ProjectionIndex(gi)))] = + ColumnBinding(TableIndex(payload_group_index), ProjectionIndex(gi)); } for (idx_t oi = 0; oi < order_proj_cols.size(); oi++) { ColumnBinding agg_binding; @@ -632,22 +639,22 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq return false; } final_expr_remap[HashBinding(agg_binding)] = - ColumnBinding(payload_group_index, group_proj_cols.size() + oi); + ColumnBinding(TableIndex(payload_group_index), ProjectionIndex(group_proj_cols.size() + oi)); } for (auto &kv : payload_old_to_new) { - final_expr_remap[HashBinding(ColumnBinding(original_agg_index, kv.first))] = - ColumnBinding(payload_agg_index, kv.second); + final_expr_remap[HashBinding(ColumnBinding(TableIndex(original_agg_index), ProjectionIndex(kv.first)))] = + ColumnBinding(TableIndex(payload_agg_index), ProjectionIndex(kv.second)); } - auto final_proj_index = binder.GenerateTableIndex(); + auto final_proj_index = binder.GenerateTableIndex().index; vector> final_exprs; std::unordered_map final_order_remap; for (idx_t pi = 0; pi < original_proj_exprs.size(); pi++) { - final_order_remap[HashBinding(ColumnBinding(original_proj_index, pi))] = - ColumnBinding(final_proj_index, final_exprs.size()); + final_order_remap[HashBinding(ColumnBinding(TableIndex(original_proj_index), ProjectionIndex(pi)))] = + ColumnBinding(TableIndex(final_proj_index), ProjectionIndex(final_exprs.size())); final_exprs.push_back(CopyExpressionWithBindingMap(*original_proj_exprs[pi], final_expr_remap)); } - auto final_proj = make_uniq(final_proj_index, std::move(final_exprs)); + auto final_proj = make_uniq(TableIndex(final_proj_index), std::move(final_exprs)); final_proj->children.push_back(std::move(payload_agg)); final_proj->ResolveOperatorTypes(); @@ -684,7 +691,7 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq return false; } auto &expr = agg.expressions[info.agg_index]; - if (expr->type != ExpressionType::BOUND_AGGREGATE) { + if (expr->GetExpressionType() != ExpressionType::BOUND_AGGREGATE) { return false; } auto &bound_agg = expr->Cast(); @@ -700,13 +707,13 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq if (!GetDirectColumnBinding(*order.expression, binding)) { return false; } - if (binding.table_index == agg.group_index) { + if (binding.table_index.index == agg.group_index.index) { if (binding.column_index >= agg.groups.size()) { return false; } continue; } - if (binding.table_index != agg.aggregate_index || binding.column_index >= agg.expressions.size()) { + if (binding.table_index.index != agg.aggregate_index.index || binding.column_index >= agg.expressions.size()) { return false; } if (pac_agg_indices.find(binding.column_index) == pac_agg_indices.end()) { @@ -744,8 +751,8 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq #endif auto &binder = input.optimizer.binder; - auto original_group_index = agg.group_index; - auto original_agg_index = agg.aggregate_index; + auto original_group_index = agg.group_index.index; + auto original_agg_index = agg.aggregate_index.index; vector original_types = agg.types; vector> original_groups; @@ -779,11 +786,12 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq std::unordered_map top_order_remap; for (idx_t gi = 0; gi < agg.groups.size(); gi++) { - top_order_remap[HashBinding(ColumnBinding(original_group_index, gi))] = ColumnBinding(original_group_index, gi); + top_order_remap[HashBinding(ColumnBinding(TableIndex(original_group_index), ProjectionIndex(gi)))] = + ColumnBinding(TableIndex(original_group_index), ProjectionIndex(gi)); } for (auto &kv : order_old_to_new) { - top_order_remap[HashBinding(ColumnBinding(original_agg_index, kv.first))] = - ColumnBinding(original_agg_index, kv.second); + top_order_remap[HashBinding(ColumnBinding(TableIndex(original_agg_index), ProjectionIndex(kv.first)))] = + ColumnBinding(TableIndex(original_agg_index), ProjectionIndex(kv.second)); } for (auto &order : topn.orders) { @@ -795,7 +803,7 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq agg.expressions = std::move(top_expressions); agg.types.clear(); for (auto &group : agg.groups) { - agg.types.push_back(group->return_type); + agg.types.push_back(group->GetReturnType()); } for (auto old_idx : order_agg_indices) { agg.types.push_back(original_types[agg.groups.size() + old_idx]); @@ -806,11 +814,11 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq auto left_topn = std::move(plan); auto join = make_uniq(JoinType::INNER); for (idx_t gi = 0; gi < original_groups.size(); gi++) { - JoinCondition cond; - cond.left = make_uniq(original_groups[gi]->return_type, - ColumnBinding(original_group_index, gi)); - cond.right = CopyExpressionWithTableMap(*original_groups[gi], right_index_map); - cond.comparison = ExpressionType::COMPARE_NOT_DISTINCT_FROM; + JoinCondition cond(nullptr, nullptr, ExpressionType::COMPARE_NOT_DISTINCT_FROM); + cond.LeftReference() = make_uniq( + original_groups[gi]->GetReturnType(), ColumnBinding(TableIndex(original_group_index), ProjectionIndex(gi))); + cond.RightReference() = CopyExpressionWithTableMap(*original_groups[gi], right_index_map); + join->conditions.push_back(std::move(cond)); } join->children.push_back(std::move(left_topn)); @@ -826,56 +834,60 @@ static bool TryRewriteLatePayloadAggregates(OptimizerExtensionInput &input, uniq payload_expressions.push_back(CopyExpressionWithTableMap(*original_aggs[old_idx], right_index_map)); } - auto payload_group_index = binder.GenerateTableIndex(); - auto payload_agg_index = binder.GenerateTableIndex(); + auto payload_group_index = binder.GenerateTableIndex().index; + auto payload_agg_index = binder.GenerateTableIndex().index; vector> payload_groups; payload_groups.reserve(original_groups.size() + order_agg_indices.size()); for (idx_t gi = 0; gi < original_groups.size(); gi++) { - payload_groups.push_back(make_uniq(original_groups[gi]->return_type, - ColumnBinding(original_group_index, gi))); + payload_groups.push_back( + make_uniq(original_groups[gi]->GetReturnType(), + ColumnBinding(TableIndex(original_group_index), ProjectionIndex(gi)))); } for (auto old_idx : order_agg_indices) { auto new_idx = order_old_to_new[old_idx]; - payload_groups.push_back(make_uniq(original_types[original_groups.size() + old_idx], - ColumnBinding(original_agg_index, new_idx))); + payload_groups.push_back(make_uniq( + original_types[original_groups.size() + old_idx], + ColumnBinding(TableIndex(original_agg_index), ProjectionIndex(new_idx)))); } - auto payload_agg = - make_uniq(payload_group_index, payload_agg_index, std::move(payload_expressions)); + auto payload_agg = make_uniq(TableIndex(payload_group_index), TableIndex(payload_agg_index), + std::move(payload_expressions)); payload_agg->groups = std::move(payload_groups); payload_agg->children.push_back(std::move(join)); payload_agg->ResolveOperatorTypes(); - auto final_proj_index = binder.GenerateTableIndex(); + auto final_proj_index = binder.GenerateTableIndex().index; vector> final_exprs; final_exprs.reserve(original_types.size()); std::unordered_map final_order_remap; for (idx_t gi = 0; gi < original_groups.size(); gi++) { - auto final_binding = ColumnBinding(final_proj_index, final_exprs.size()); - final_order_remap[HashBinding(ColumnBinding(original_group_index, gi))] = final_binding; - final_exprs.push_back( - make_uniq(original_types[gi], ColumnBinding(payload_group_index, gi))); + auto final_binding = ColumnBinding(TableIndex(final_proj_index), ProjectionIndex(final_exprs.size())); + final_order_remap[HashBinding(ColumnBinding(TableIndex(original_group_index), ProjectionIndex(gi)))] = + final_binding; + final_exprs.push_back(make_uniq( + original_types[gi], ColumnBinding(TableIndex(payload_group_index), ProjectionIndex(gi)))); } for (idx_t ai = 0; ai < original_aggs.size(); ai++) { - auto final_binding = ColumnBinding(final_proj_index, final_exprs.size()); - final_order_remap[HashBinding(ColumnBinding(original_agg_index, ai))] = final_binding; + auto final_binding = ColumnBinding(TableIndex(final_proj_index), ProjectionIndex(final_exprs.size())); + final_order_remap[HashBinding(ColumnBinding(TableIndex(original_agg_index), ProjectionIndex(ai)))] = + final_binding; auto original_type = original_types[original_groups.size() + ai]; auto order_it = order_old_to_new.find(ai); if (order_it != order_old_to_new.end()) { auto group_pos = original_groups.size() + order_it->second; - final_exprs.push_back( - make_uniq(original_type, ColumnBinding(payload_group_index, group_pos))); + final_exprs.push_back(make_uniq( + original_type, ColumnBinding(TableIndex(payload_group_index), ProjectionIndex(group_pos)))); continue; } auto payload_it = payload_old_to_new.find(ai); if (payload_it == payload_old_to_new.end()) { return false; } - final_exprs.push_back( - make_uniq(original_type, ColumnBinding(payload_agg_index, payload_it->second))); + final_exprs.push_back(make_uniq( + original_type, ColumnBinding(TableIndex(payload_agg_index), ProjectionIndex(payload_it->second)))); } - auto final_proj = make_uniq(final_proj_index, std::move(final_exprs)); + auto final_proj = make_uniq(TableIndex(final_proj_index), std::move(final_exprs)); final_proj->children.push_back(std::move(payload_agg)); final_proj->ResolveOperatorTypes(); @@ -976,7 +988,7 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique for (auto &info : pac_aggs) { PRIVACY_DEBUG_PRINT("PACTopKRule: Found PAC aggregate: " + info.original_name + " at index " + std::to_string(info.agg_index) + " binding=[" + - std::to_string(info.agg_binding.table_index) + "." + + std::to_string(info.agg_binding.table_index.index) + "." + std::to_string(info.agg_binding.column_index) + "]"); } #endif @@ -1027,7 +1039,7 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique double expansion = 1.0; { Value exp_val; - if (input.context.TryGetCurrentSetting("pac_topk_expansion", exp_val) && !exp_val.IsNull()) { + if (input.context.TryGetCurrentSetting(Identifier("pac_topk_expansion"), exp_val) && !exp_val.IsNull()) { expansion = exp_val.GetValue(); if (expansion < 1.0) { expansion = 1.0; @@ -1054,11 +1066,12 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique auto &expr = agg.expressions[info.agg_index]; auto &bound_agg = expr->Cast(); vector> children; - for (auto &child_expr : bound_agg.children) { + for (auto &child_expr : bound_agg.GetChildrenMutable()) { children.push_back(child_expr->Copy()); } - auto new_aggr = RebindAggregate(input.context, GetCountersVariant(bound_agg.function.name), std::move(children), - bound_agg.IsDistinct()); + auto new_aggr = RebindAggregate(input.context, + GetCountersVariant(bound_agg.FunctionMutable().GetName().GetIdentifierName()), + std::move(children), bound_agg.IsDistinct()); if (new_aggr) { #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("PACTopKRule: Rebound " + info.original_name + " -> " + @@ -1069,11 +1082,12 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique // Fallback: rename in place #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("PACTopKRule: Fallback rename " + info.original_name + " -> " + - GetCountersVariant(bound_agg.function.name)); + GetCountersVariant(bound_agg.Function().GetName().GetIdentifierName())); #endif - bound_agg.function.name = GetCountersVariant(bound_agg.function.name); - bound_agg.function.return_type = list_type; - expr->return_type = list_type; + bound_agg.FunctionMutable().SetName( + Identifier(GetCountersVariant(bound_agg.FunctionMutable().GetName().GetIdentifierName()))); + bound_agg.FunctionMutable().SetReturnType(list_type); + expr->SetReturnType(list_type); } // Update types in the aggregate idx_t types_index = agg.groups.size() + info.agg_index; @@ -1090,7 +1104,7 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("PACTopKRule: Step 2 - Building mean projection"); #endif - idx_t mean_proj_idx = binder.GenerateTableIndex(); + idx_t mean_proj_idx = binder.GenerateTableIndex().index; vector> mean_proj_exprs; // Get all column bindings from the aggregate @@ -1115,7 +1129,7 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique pac_mean_column_map[HashBinding(info.agg_binding)] = mean_col_idx; } - auto mean_proj = make_uniq(mean_proj_idx, std::move(mean_proj_exprs)); + auto mean_proj = make_uniq(TableIndex(mean_proj_idx), std::move(mean_proj_exprs)); if (ctx.has_intermediate_projection) { // ================================================================== @@ -1144,14 +1158,14 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique unordered_map order_pac_refs; for (idx_t oi = 0; oi < ctx.topn->orders.size(); oi++) { auto *stripped = StripCasts(ctx.topn->orders[oi].expression.get()); - auto *col_ref = stripped->type == ExpressionType::BOUND_COLUMN_REF + auto *col_ref = stripped->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF ? &stripped->Cast() : nullptr; if (!col_ref) { continue; } ColumnBinding resolved; - if (TraceBindingThroughProjections(col_ref->binding, ctx.intermediate_projections, 0, resolved)) { + if (TraceBindingThroughProjections(col_ref->BindingMutable(), ctx.intermediate_projections, 0, resolved)) { for (idx_t j = 0; j < pac_aggs.size(); j++) { if (resolved == pac_aggs[j].agg_binding) { order_pac_refs[oi] = j; @@ -1164,12 +1178,12 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique // Map: outermost projection column index -> pac_aggs index (for PAC aggregate columns) unordered_map outermost_pac_col_map; for (idx_t i = 0; i < outermost->expressions.size(); i++) { - if (outermost->expressions[i]->type != ExpressionType::BOUND_COLUMN_REF) { + if (outermost->expressions[i]->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) { continue; // function calls (e.g. decompress) are non-PAC group columns } auto &ref = outermost->expressions[i]->Cast(); ColumnBinding resolved; - if (TraceBindingThroughProjections(ref.binding, ctx.intermediate_projections, 1, resolved)) { + if (TraceBindingThroughProjections(ref.BindingMutable(), ctx.intermediate_projections, 1, resolved)) { for (idx_t j = 0; j < pac_aggs.size(); j++) { if (resolved == pac_aggs[j].agg_binding) { outermost_pac_col_map[i] = j; @@ -1194,12 +1208,12 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique // Rewrite all ColRefs in innermost projection: aggregate bindings -> MeanProj bindings. // Also update return_type since PAC columns changed from e.g. BIGINT to LIST. std::function &)> UpdateRefs = [&](unique_ptr &e) { - if (e->type == ExpressionType::BOUND_COLUMN_REF) { + if (e->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e->Cast(); for (idx_t i = 0; i < agg_bindings.size(); i++) { - if (col_ref.binding == agg_bindings[i]) { - col_ref.binding = ColumnBinding(mean_proj_idx, i); - col_ref.return_type = agg.types[i]; + if (col_ref.BindingMutable() == agg_bindings[i]) { + col_ref.BindingMutable() = ColumnBinding(TableIndex(mean_proj_idx), ProjectionIndex(i)); + col_ref.SetReturnType(agg.types[i]); break; } } @@ -1228,13 +1242,14 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique // Innermost: reference MeanProj's priv_mean column directly idx_t mean_col = pac_mean_column_map[HashBinding(pac_aggs[j].agg_binding)]; proj->expressions.push_back(make_uniq( - PacFloatLogicalType(), ColumnBinding(mean_proj_idx, mean_col))); + PacFloatLogicalType(), ColumnBinding(TableIndex(mean_proj_idx), ProjectionIndex(mean_col)))); } else { // Reference the projection below's priv_mean passthrough column auto *below = ctx.intermediate_projections[pi + 1]; idx_t below_col = pac_mean_col_indices[pi + 1][j]; proj->expressions.push_back(make_uniq( - PacFloatLogicalType(), ColumnBinding(below->table_index, below_col))); + PacFloatLogicalType(), + ColumnBinding(TableIndex(below->table_index.index), ProjectionIndex(below_col)))); } } } @@ -1248,11 +1263,11 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique auto &child_types = proj->children[0]->types; // Update ColRef return_types based on the child operator's resolved types std::function &)> FixTypes = [&](unique_ptr &e) { - if (e->type == ExpressionType::BOUND_COLUMN_REF) { + if (e->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e->Cast(); for (idx_t ci = 0; ci < child_bindings.size(); ci++) { - if (col_ref.binding == child_bindings[ci] && ci < child_types.size()) { - col_ref.return_type = child_types[ci]; + if (col_ref.BindingMutable() == child_bindings[ci] && ci < child_types.size()) { + col_ref.SetReturnType(child_types[ci]); break; } } @@ -1281,15 +1296,16 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique auto &expr = proj->expressions[i]; #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("PACTopKRule: A3b proj expr[" + std::to_string(i) + - "] type=" + std::to_string((int)expr->type) + - " return_type=" + expr->return_type.ToString() + " expr=" + expr->ToString()); + "] type=" + std::to_string((int)expr->GetExpressionType()) + + " return_type=" + expr->GetReturnType().ToString() + " expr=" + expr->ToString()); #endif - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { continue; } std::function &)> WrapListRefs = [&](unique_ptr &e) { - if (e->type == ExpressionType::BOUND_COLUMN_REF && e->return_type.id() == LogicalTypeId::LIST) { - auto binding = e->Cast().binding; + if (e->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF && + e->GetReturnType().id() == LogicalTypeId::LIST) { + auto binding = e->Cast().BindingMutable(); #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("PACTopKRule: A3b WRAPPING LIST colref: " + e->ToString()); #endif @@ -1301,7 +1317,7 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique bool traced = TraceBindingThroughProjections(binding, ctx.intermediate_projections, pi + 1, resolved); for (auto &info : pac_aggs) { - if (info.original_type != e->return_type) { + if (info.original_type != e->GetReturnType()) { if (traced && resolved == info.agg_binding) { e = BoundCastExpression::AddCastToType(input.context, std::move(e), info.original_type); @@ -1324,16 +1340,18 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique // If this is a CAST whose child is a LIST colref, replace the // entire CAST so the bound cast function matches the new source // type (FLOAT from as_noised instead of original INT64/DECIMAL). - if (e->GetExpressionClass() == ExpressionClass::BOUND_CAST) { - auto &cast_expr = e->Cast(); - if (cast_expr.child->type == ExpressionType::BOUND_COLUMN_REF && - cast_expr.child->return_type.id() == LogicalTypeId::LIST) { - auto target_type = cast_expr.return_type; + if (e->GetExpressionType() == ExpressionType::OPERATOR_CAST) { + auto &cast_expr = e->Cast(); + if (cast_expr.GetChildrenMutable()[0]->GetExpressionType() == + ExpressionType::BOUND_COLUMN_REF && + cast_expr.GetChildrenMutable()[0]->GetReturnType().id() == LogicalTypeId::LIST) { + auto target_type = cast_expr.GetReturnType(); #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("PACTopKRule: A3b REPLACING CAST(LIST->" + target_type.ToString() + ") with CAST(as_noised()->" + target_type.ToString() + ")"); #endif - auto noised = input.optimizer.BindScalarFunction("as_noised", std::move(cast_expr.child)); + auto noised = input.optimizer.BindScalarFunction( + "as_noised", std::move(cast_expr.GetChildrenMutable()[0])); e = BoundCastExpression::AddCastToType(input.context, std::move(noised), target_type); return; } @@ -1354,12 +1372,13 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique // ------------------------------------------------------------------ for (auto &kv : order_pac_refs) { auto *stripped = StripCasts(ctx.topn->orders[kv.first].expression.get()); - auto *col_ref = stripped->type == ExpressionType::BOUND_COLUMN_REF + auto *col_ref = stripped->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF ? &stripped->Cast() : nullptr; if (col_ref) { - col_ref->binding = ColumnBinding(outermost->table_index, pac_mean_col_indices[0][kv.second]); - col_ref->return_type = PacFloatLogicalType(); + col_ref->BindingMutable() = ColumnBinding(TableIndex(outermost->table_index.index), + ProjectionIndex(pac_mean_col_indices[0][kv.second])); + col_ref->SetReturnType(PacFloatLogicalType()); } } @@ -1373,34 +1392,35 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique // priv_mean() applied then cast back to the original return type. // Non-PAC columns are passed through unchanged. // ------------------------------------------------------------------ - idx_t noised_proj_idx = binder.GenerateTableIndex(); + idx_t noised_proj_idx = binder.GenerateTableIndex().index; vector> noised_proj_exprs; for (idx_t i = 0; i < original_outermost_col_count; i++) { auto it_pac = outermost_pac_col_map.find(i); if (it_pac != outermost_pac_col_map.end()) { // PAC aggregate column: apply as_noised to counter LIST, cast back to original type - auto counters_ref = - make_uniq(list_type, ColumnBinding(outermost->table_index, i)); + auto counters_ref = make_uniq( + list_type, ColumnBinding(TableIndex(outermost->table_index.index), ProjectionIndex(i))); auto noised = input.optimizer.BindScalarFunction("as_noised", std::move(counters_ref)); auto &orig_type = pac_aggs[it_pac->second].original_type; - if (orig_type != noised->return_type) { + if (orig_type != noised->GetReturnType()) { noised = BoundCastExpression::AddCastToType(input.context, std::move(noised), orig_type); } noised_proj_exprs.push_back(std::move(noised)); } else { // Non-PAC column: passthrough (preserves decompress result, etc.) - auto ref = - make_uniq(outermost->types[i], ColumnBinding(outermost->table_index, i)); + auto ref = make_uniq( + outermost->types[i], ColumnBinding(TableIndex(outermost->table_index.index), ProjectionIndex(i))); noised_proj_exprs.push_back(std::move(ref)); } } - auto noised_proj = make_uniq(noised_proj_idx, std::move(noised_proj_exprs)); + auto noised_proj = make_uniq(TableIndex(noised_proj_idx), std::move(noised_proj_exprs)); // Build order remap: original outermost projection columns -> NoisedProj columns for (idx_t i = 0; i < original_outermost_col_count; i++) { - order_remap[HashBinding(ColumnBinding(outermost->table_index, i))] = ColumnBinding(noised_proj_idx, i); + order_remap[HashBinding(ColumnBinding(TableIndex(outermost->table_index.index), ProjectionIndex(i)))] = + ColumnBinding(TableIndex(noised_proj_idx), ProjectionIndex(i)); } // ------------------------------------------------------------------ @@ -1430,16 +1450,16 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique // Step B1: Rewrite TopN ORDER BY to reference MeanProj std::function &)> RewriteOrderExpr = [&](unique_ptr &expr) { - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr->Cast(); - auto it = pac_mean_column_map.find(HashBinding(col_ref.binding)); + auto it = pac_mean_column_map.find(HashBinding(col_ref.BindingMutable())); if (it != pac_mean_column_map.end()) { - col_ref.binding = ColumnBinding(mean_proj_idx, it->second); - col_ref.return_type = PacFloatLogicalType(); + col_ref.BindingMutable() = ColumnBinding(TableIndex(mean_proj_idx), ProjectionIndex(it->second)); + col_ref.SetReturnType(PacFloatLogicalType()); } else { for (idx_t i = 0; i < agg_bindings.size(); i++) { - if (agg_bindings[i] == col_ref.binding) { - col_ref.binding = ColumnBinding(mean_proj_idx, i); + if (agg_bindings[i] == col_ref.BindingMutable()) { + col_ref.BindingMutable() = ColumnBinding(TableIndex(mean_proj_idx), ProjectionIndex(i)); break; } } @@ -1455,18 +1475,19 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique } // Step B2: Build NoisedProj - idx_t noised_proj_idx = binder.GenerateTableIndex(); + idx_t noised_proj_idx = binder.GenerateTableIndex().index; vector> noised_proj_exprs; for (idx_t i = 0; i < agg_bindings.size(); i++) { auto &binding = agg_bindings[i]; auto it = pac_mean_column_map.find(HashBinding(binding)); if (it != pac_mean_column_map.end()) { - auto counters_ref = make_uniq(list_type, ColumnBinding(mean_proj_idx, i)); + auto counters_ref = make_uniq( + list_type, ColumnBinding(TableIndex(mean_proj_idx), ProjectionIndex(i))); auto noised = input.optimizer.BindScalarFunction("as_noised", std::move(counters_ref)); // Cast back to the original aggregate return type (e.g. BIGINT for count) for (auto &info : pac_aggs) { - if (info.agg_binding == binding && info.original_type != noised->return_type) { + if (info.agg_binding == binding && info.original_type != noised->GetReturnType()) { noised = BoundCastExpression::AddCastToType(input.context, std::move(noised), info.original_type); break; @@ -1474,18 +1495,20 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique } noised_proj_exprs.push_back(std::move(noised)); } else { - auto ref = make_uniq(agg.types[i], ColumnBinding(mean_proj_idx, i)); + auto ref = make_uniq( + agg.types[i], ColumnBinding(TableIndex(mean_proj_idx), ProjectionIndex(i))); noised_proj_exprs.push_back(std::move(ref)); } } - auto noised_proj = make_uniq(noised_proj_idx, std::move(noised_proj_exprs)); + auto noised_proj = make_uniq(TableIndex(noised_proj_idx), std::move(noised_proj_exprs)); // Build order remap: original aggregate bindings -> NoisedProj columns { idx_t noised_col = 0; for (idx_t i = 0; i < agg_bindings.size(); i++) { - order_remap[HashBinding(agg_bindings[i])] = ColumnBinding(noised_proj_idx, noised_col); + order_remap[HashBinding(agg_bindings[i])] = + ColumnBinding(TableIndex(noised_proj_idx), ProjectionIndex(noised_col)); noised_col++; } } @@ -1518,11 +1541,11 @@ void PACTopKRule::PACTopKOptimizeFunction(OptimizerExtensionInput &input, unique for (auto &saved : saved_orders) { auto expr = saved.expression->Copy(); std::function &)> RemapToNoised = [&](unique_ptr &e) { - if (e->type == ExpressionType::BOUND_COLUMN_REF) { + if (e->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &ref = e->Cast(); - auto it = order_remap.find(HashBinding(ref.binding)); + auto it = order_remap.find(HashBinding(ref.BindingMutable())); if (it != order_remap.end()) { - ref.binding = it->second; + ref.BindingMutable() = it->second; } } ExpressionIterator::EnumerateChildren(*e, RemapToNoised); diff --git a/src/query_processing/privacy_expression_builder.cpp b/src/query_processing/privacy_expression_builder.cpp index 41d42229..0cc3ec9c 100644 --- a/src/query_processing/privacy_expression_builder.cpp +++ b/src/query_processing/privacy_expression_builder.cpp @@ -43,7 +43,7 @@ static void EnsureProjectionIdsPopulated(LogicalGet &get) { } if (get.projection_ids.empty()) { for (idx_t i = 0; i < column_ids.size(); i++) { - get.projection_ids.push_back(i); + get.projection_ids.push_back(ProjectionIndex(i)); } } } @@ -53,7 +53,7 @@ static void EnsureProjectionIdsPopulated(LogicalGet &get) { static LogicalGet *FindLogicalGetByTableIndex(LogicalOperator &op, idx_t table_index) { if (op.type == LogicalOperatorType::LOGICAL_GET) { auto &get = op.Cast(); - if (get.table_index == table_index) { + if (get.table_index.index == table_index) { return &get; } } @@ -68,19 +68,19 @@ static LogicalGet *FindLogicalGetByTableIndex(LogicalOperator &op, idx_t table_i // Helper function to resolve a column name from a binding by finding the corresponding LogicalGet static string ResolveColumnNameFromBinding(LogicalOperator &root, const ColumnBinding &binding) { - auto get = FindLogicalGetByTableIndex(root, binding.table_index); + auto get = FindLogicalGetByTableIndex(root, binding.table_index.index); if (!get) { - return "[" + std::to_string(binding.table_index) + "." + std::to_string(binding.column_index) + "]"; + return "[" + std::to_string(binding.table_index.index) + "." + std::to_string(binding.column_index) + "]"; } const auto &column_ids = get->GetColumnIds(); if (binding.column_index >= column_ids.size()) { - return "[" + std::to_string(binding.table_index) + "." + std::to_string(binding.column_index) + "]"; + return "[" + std::to_string(binding.table_index.index) + "." + std::to_string(binding.column_index) + "]"; } - auto col_name = get->GetColumnName(column_ids[binding.column_index]); + auto col_name = get->GetColumnName(column_ids[binding.column_index]).GetIdentifierName(); if (col_name.empty()) { - return "[" + std::to_string(binding.table_index) + "." + std::to_string(binding.column_index) + "]"; + return "[" + std::to_string(binding.table_index.index) + "." + std::to_string(binding.column_index) + "]"; } return col_name; @@ -93,7 +93,7 @@ idx_t EnsureProjectedColumn(LogicalGet &g, const string &col_name) { for (idx_t cid = 0; cid < g.GetColumnIds().size(); ++cid) { auto col_idx = g.GetColumnIds()[cid]; if (!col_idx.IsVirtualColumn()) { - const auto &existing_name = g.GetColumnName(col_idx); + const auto &existing_name = g.GetColumnName(col_idx).GetIdentifierName(); if (!existing_name.empty() && StringUtil::CIEquals(existing_name, col_name)) { return cid; } @@ -108,7 +108,7 @@ idx_t EnsureProjectedColumn(LogicalGet &g, const string &col_name) { LogicalType logical_type = LogicalType::INVALID; idx_t ti = 0; for (auto &col : table_entry->GetColumns().Logical()) { - if (StringUtil::CIEquals(col.Name(), col_name)) { + if (StringUtil::CIEquals(col.Name().GetIdentifierName(), col_name)) { logical_idx = ti; logical_type = col.Type(); break; @@ -130,7 +130,7 @@ idx_t EnsureProjectedColumn(LogicalGet &g, const string &col_name) { // The projection index is the position in the output, which equals the new size - 1 idx_t new_proj_idx = g.GetColumnIds().size() - 1; - g.projection_ids.push_back(new_proj_idx); + g.projection_ids.push_back(ProjectionIndex(new_proj_idx)); // ResolveOperatorTypes() calls the protected ResolveTypes() which rebuilds // the types vector from column_ids/projection_ids @@ -162,7 +162,7 @@ void AddRowIDColumn(LogicalGet &get) { EnsureProjectionIdsPopulated(get); get.AddColumnId(COLUMN_IDENTIFIER_ROW_ID); - get.projection_ids.push_back(get.GetColumnIds().size() - 1); + get.projection_ids.push_back(ProjectionIndex(get.GetColumnIds().size() - 1)); // ResolveOperatorTypes() calls the protected ResolveTypes() which rebuilds // the types vector from column_ids/projection_ids @@ -191,7 +191,7 @@ unique_ptr BuildXorHashFromPKs(OptimizerExtensionInput &input, Logic if (proj_idx == DConstants::INVALID_INDEX) { throw InternalException("Privacy compiler: failed to find PK column " + pk); } - auto col_binding = ColumnBinding(get.table_index, proj_idx); + auto col_binding = ColumnBinding(TableIndex(get.table_index.index), ProjectionIndex(proj_idx)); auto col_index_obj = get.GetColumnIds()[proj_idx]; auto &col_type = get.GetColumnType(col_index_obj); pk_cols.push_back(make_uniq(col_type, col_binding)); @@ -205,7 +205,8 @@ ColumnBinding InsertHashProjectionAboveGet(OptimizerExtensionInput &input, uniqu unique_ptr hash_expr; if (use_rowid) { AddRowIDColumn(get); - auto rowid_binding = ColumnBinding(get.table_index, get.GetColumnIds().size() - 1); + auto rowid_binding = + ColumnBinding(TableIndex(get.table_index.index), ProjectionIndex(get.GetColumnIds().size() - 1)); auto rowid_col = make_uniq(LogicalType::BIGINT, rowid_binding); hash_expr = input.optimizer.BindScalarFunction("hash", std::move(rowid_col)); } else { @@ -215,14 +216,14 @@ ColumnBinding InsertHashProjectionAboveGet(OptimizerExtensionInput &input, uniqu // 1b. Wrap in priv_hash() to XOR with query_hash and optionally repair to 32 bits set hash_expr = input.optimizer.BindScalarFunction("priv_hash", std::move(hash_expr)); - hash_expr->alias = "pac_pu"; + hash_expr->SetAlias(Identifier("pac_pu")); // 2. Find the unique_ptr slot holding this get in the plan tree vector *> get_nodes; - FindAllNodesByTableIndex(&plan, get.table_index, get_nodes); + FindAllNodesByTableIndex(&plan, get.table_index.index, get_nodes); if (get_nodes.empty()) { throw InternalException("Privacy compiler: InsertHashProjectionAboveGet could not find get #" + - std::to_string(get.table_index)); + std::to_string(get.table_index.index)); } auto &get_slot = *get_nodes[0]; @@ -232,20 +233,20 @@ ColumnBinding InsertHashProjectionAboveGet(OptimizerExtensionInput &input, uniqu // 4. Create projection with passthrough + hash column auto &binder = input.optimizer.binder; - idx_t proj_table_index = binder.GenerateTableIndex(); + idx_t proj_table_index = binder.GenerateTableIndex().index; vector> proj_expressions; // Passthrough expressions for each existing get output column for (idx_t i = 0; i < num_get_cols; i++) { auto col_type = get_slot->types[i]; - auto binding = ColumnBinding(get.table_index, i); + auto binding = ColumnBinding(TableIndex(get.table_index.index), ProjectionIndex(i)); proj_expressions.push_back(make_uniq(col_type, binding)); } // Hash expression as the last column proj_expressions.push_back(std::move(hash_expr)); - auto projection = make_uniq(proj_table_index, std::move(proj_expressions)); + auto projection = make_uniq(TableIndex(proj_table_index), std::move(proj_expressions)); // 5. Move get into projection as child, place projection in slot projection->children.push_back(std::move(get_slot)); @@ -255,29 +256,29 @@ ColumnBinding InsertHashProjectionAboveGet(OptimizerExtensionInput &input, uniqu LogicalOperator *proj_ptr = projection.get(); get_slot = std::move(projection); - // 6. Use ColumnBindingReplacer to remap [get.table_index, i] → [proj.table_index, i] + // 6. Use ColumnBindingReplacer to remap [get.table_index.index, i] → [proj.table_index.index, i] ColumnBindingReplacer replacer; for (idx_t i = 0; i < num_get_cols; i++) { ColumnBinding old_b = old_bindings[i]; - ColumnBinding new_b(proj_table_index, i); + ColumnBinding new_b {TableIndex(proj_table_index), ProjectionIndex(i)}; replacer.replacement_bindings.emplace_back(old_b, new_b); } replacer.stop_operator = proj_ptr; replacer.VisitOperator(*plan); // 7. Return binding for the hash column (last expression in projection) - return ColumnBinding(proj_table_index, num_get_cols); + return ColumnBinding(TableIndex(proj_table_index), ProjectionIndex(num_get_cols)); } ColumnBinding GetOrInsertHashProjection(OptimizerExtensionInput &input, unique_ptr &plan, LogicalGet &get, const vector &key_columns, bool use_rowid, std::unordered_map &cache) { - auto it = cache.find(get.table_index); + auto it = cache.find(get.table_index.index); if (it != cache.end()) { return it->second; } auto binding = InsertHashProjectionAboveGet(input, plan, get, key_columns, use_rowid); - cache[get.table_index] = binding; + cache[get.table_index.index] = binding; return binding; } @@ -291,7 +292,7 @@ static unique_ptr *FindCTERefSlot(unique_ptr & } if (root->type == LogicalOperatorType::LOGICAL_CTE_REF) { auto &ref = root->Cast(); - if (ref.table_index == table_index) { + if (ref.table_index.index == table_index) { return &root; } } @@ -307,8 +308,9 @@ static unique_ptr *FindCTERefSlot(unique_ptr & ColumnBinding InsertHashProjectionAboveCTERef(OptimizerExtensionInput &input, unique_ptr &plan, LogicalCTERef &cte_ref, const vector &key_columns) { #if PRIVACY_DEBUG - PRIVACY_DEBUG_PRINT("InsertHashProjectionAboveCTERef: CTE_SCAN table_index=" + std::to_string(cte_ref.table_index) + - " cte_index=" + std::to_string(cte_ref.cte_index)); + PRIVACY_DEBUG_PRINT( + "InsertHashProjectionAboveCTERef: CTE_SCAN table_index=" + std::to_string(cte_ref.table_index.index) + + " cte_index=" + std::to_string(cte_ref.cte_index.index)); #endif // 1. Resolve key column indices in the CTE_SCAN's bound_columns @@ -317,7 +319,7 @@ ColumnBinding InsertHashProjectionAboveCTERef(OptimizerExtensionInput &input, un bool found = false; for (idx_t i = 0; i < cte_ref.bound_columns.size(); i++) { if (cte_ref.bound_columns[i] == key) { - auto binding = ColumnBinding(cte_ref.table_index, i); + auto binding = ColumnBinding(TableIndex(cte_ref.table_index.index), ProjectionIndex(i)); key_col_exprs.push_back(make_uniq(cte_ref.chunk_types[i], binding)); found = true; break; @@ -327,7 +329,7 @@ ColumnBinding InsertHashProjectionAboveCTERef(OptimizerExtensionInput &input, un #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("InsertHashProjectionAboveCTERef: key column '" + key + "' not found in bound_columns"); #endif - return ColumnBinding(DConstants::INVALID_INDEX, 0); + return ColumnBinding(TableIndex(DConstants::INVALID_INDEX), ProjectionIndex(0)); } } @@ -336,13 +338,13 @@ ColumnBinding InsertHashProjectionAboveCTERef(OptimizerExtensionInput &input, un // Wrap in priv_hash() to XOR with query_hash and optionally repair to 32 bits set hash_expr = input.optimizer.BindScalarFunction("priv_hash", std::move(hash_expr)); - hash_expr->alias = "pac_pu"; + hash_expr->SetAlias(Identifier("pac_pu")); // 3. Find the unique_ptr slot holding this CTE_SCAN in the plan tree - auto *cte_slot_ptr = FindCTERefSlot(plan, cte_ref.table_index); + auto *cte_slot_ptr = FindCTERefSlot(plan, cte_ref.table_index.index); if (!cte_slot_ptr) { throw InternalException("Privacy compiler: InsertHashProjectionAboveCTERef could not find CTE_SCAN #" + - std::to_string(cte_ref.table_index)); + std::to_string(cte_ref.table_index.index)); } auto &cte_slot = *cte_slot_ptr; @@ -351,14 +353,14 @@ ColumnBinding InsertHashProjectionAboveCTERef(OptimizerExtensionInput &input, un idx_t num_cte_cols = old_bindings.size(); auto &binder = input.optimizer.binder; - idx_t proj_table_index = binder.GenerateTableIndex(); + idx_t proj_table_index = binder.GenerateTableIndex().index; vector> proj_expressions; for (idx_t i = 0; i < num_cte_cols; i++) { proj_expressions.push_back(make_uniq(cte_slot->types[i], old_bindings[i])); } proj_expressions.push_back(std::move(hash_expr)); - auto projection = make_uniq(proj_table_index, std::move(proj_expressions)); + auto projection = make_uniq(TableIndex(proj_table_index), std::move(proj_expressions)); projection->children.push_back(std::move(cte_slot)); projection->ResolveOperatorTypes(); @@ -368,18 +370,19 @@ ColumnBinding InsertHashProjectionAboveCTERef(OptimizerExtensionInput &input, un ColumnBindingReplacer replacer; for (idx_t i = 0; i < num_cte_cols; i++) { - replacer.replacement_bindings.emplace_back(old_bindings[i], ColumnBinding(proj_table_index, i)); + replacer.replacement_bindings.emplace_back(old_bindings[i], + ColumnBinding(TableIndex(proj_table_index), ProjectionIndex(i))); } replacer.stop_operator = proj_ptr; replacer.VisitOperator(*plan); #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("InsertHashProjectionAboveCTERef: inserted hash projection (table_index=" + - std::to_string(proj_table_index) + ") above CTE_SCAN #" + std::to_string(cte_ref.table_index) + - ", hash at column " + std::to_string(num_cte_cols)); + std::to_string(proj_table_index) + ") above CTE_SCAN #" + + std::to_string(cte_ref.table_index.index) + ", hash at column " + std::to_string(num_cte_cols)); #endif - return ColumnBinding(proj_table_index, num_cte_cols); + return ColumnBinding(TableIndex(proj_table_index), ProjectionIndex(num_cte_cols)); } // Build AND expression from multiple hash expressions (for multiple PUs) @@ -409,14 +412,15 @@ unique_ptr BindPacAggregate(OptimizerExtensionInput &input, const st unique_ptr correction_expr) { FunctionBinder function_binder(input.context); ErrorData error; - vector arg_types = {hash_expr->return_type, value_expr->return_type}; + vector arg_types = {hash_expr->GetReturnType(), value_expr->GetReturnType()}; if (correction_expr) { - arg_types.push_back(correction_expr->return_type); + arg_types.push_back(correction_expr->GetReturnType()); } - auto &entry = Catalog::GetSystemCatalog(input.context) - .GetEntry(input.context, DEFAULT_SCHEMA, pac_func_name); - auto best = function_binder.BindFunction(entry.name, entry.functions, arg_types, error); + auto &entry = + Catalog::GetSystemCatalog(input.context) + .GetEntry(input.context, DEFAULT_SCHEMA, Identifier(pac_func_name)); + auto best = function_binder.BindFunction(Identifier(entry.name), entry.functions, arg_types, error); if (!best.IsValid()) { throw InternalException("Privacy compiler: failed to bind " + pac_func_name); } @@ -435,11 +439,12 @@ PuPreAggregationInfo InsertPuPreAggregation(OptimizerExtensionInput &input, Logi vector> lower_expressions, unique_ptr pu_hash_expr) { auto &binder = input.optimizer.binder; - idx_t lower_group_index = binder.GenerateTableIndex(); - idx_t lower_agg_index = binder.GenerateTableIndex(); + idx_t lower_group_index = binder.GenerateTableIndex().index; + idx_t lower_agg_index = binder.GenerateTableIndex().index; idx_t num_original_groups = agg->groups.size(); - auto lower_agg = make_uniq(lower_group_index, lower_agg_index, std::move(lower_expressions)); + auto lower_agg = make_uniq(TableIndex(lower_group_index), TableIndex(lower_agg_index), + std::move(lower_expressions)); for (auto &g : agg->groups) { lower_agg->groups.push_back(g->Copy()); } @@ -448,11 +453,13 @@ PuPreAggregationInfo InsertPuPreAggregation(OptimizerExtensionInput &input, Logi lower_agg->ResolveOperatorTypes(); for (idx_t i = 0; i < num_original_groups; i++) { - agg->groups[i] = make_uniq(lower_agg->types[i], ColumnBinding(lower_group_index, i)); + agg->groups[i] = make_uniq( + lower_agg->types[i], ColumnBinding(TableIndex(lower_group_index), ProjectionIndex(i))); } - auto pu_hash_ref = make_uniq(lower_agg->types[num_original_groups], - ColumnBinding(lower_group_index, num_original_groups)); + auto pu_hash_ref = make_uniq( + lower_agg->types[num_original_groups], + ColumnBinding(TableIndex(lower_group_index), ProjectionIndex(num_original_groups))); auto *lower_agg_ptr = lower_agg.get(); agg->children[0] = std::move(lower_agg); @@ -464,10 +471,11 @@ unique_ptr BindBitOrAggregate(OptimizerExtensionInput &input, unique unique_ptr filter_col_expr) { FunctionBinder function_binder(input.context); ErrorData error; - vector bit_or_types = {hash_expr->return_type}; + vector bit_or_types = {hash_expr->GetReturnType()}; auto &bit_or_entry = Catalog::GetSystemCatalog(input.context) .GetEntry(input.context, DEFAULT_SCHEMA, "bit_or"); - auto bit_or_best = function_binder.BindFunction(bit_or_entry.name, bit_or_entry.functions, bit_or_types, error); + auto bit_or_best = + function_binder.BindFunction(Identifier(bit_or_entry.name), bit_or_entry.functions, bit_or_types, error); if (!bit_or_best.IsValid()) { throw InternalException("Privacy compiler: failed to bind bit_or"); } @@ -481,8 +489,8 @@ unique_ptr BindBitOrAggregate(OptimizerExtensionInput &input, unique if (filter_col_expr) { auto not_null_filter = make_uniq(ExpressionType::OPERATOR_IS_NOT_NULL, LogicalType::BOOLEAN); - not_null_filter->children.push_back(std::move(filter_col_expr)); - bit_or_expr->Cast().filter = std::move(not_null_filter); + not_null_filter->GetChildrenMutable().push_back(std::move(filter_col_expr)); + bit_or_expr->Cast().GetFilterMutable() = std::move(not_null_filter); } return bit_or_expr; @@ -524,8 +532,8 @@ static void InsertDistinctPreAggregation(OptimizerExtensionInput &input, Logical auto &binder = input.optimizer.binder; // 1. Create inner aggregate with new table indices - idx_t inner_group_index = binder.GenerateTableIndex(); - idx_t inner_agg_index = binder.GenerateTableIndex(); + idx_t inner_group_index = binder.GenerateTableIndex().index; + idx_t inner_agg_index = binder.GenerateTableIndex().index; // 2. Bind bit_or aggregate on the hash expression with IS NOT NULL filter auto bit_or_expr = BindBitOrAggregate(input, hash_input_expr->Copy(), distinct_value_expr->Copy()); @@ -533,7 +541,8 @@ static void InsertDistinctPreAggregation(OptimizerExtensionInput &input, Logical // 3. Build the inner aggregate node vector> inner_expressions; inner_expressions.push_back(std::move(bit_or_expr)); - auto inner_agg_node = make_uniq(inner_group_index, inner_agg_index, std::move(inner_expressions)); + auto inner_agg_node = make_uniq(TableIndex(inner_group_index), TableIndex(inner_agg_index), + std::move(inner_expressions)); // Copy outer groups into inner aggregate (passthrough group keys) idx_t num_original_groups = agg->groups.size(); @@ -552,20 +561,21 @@ static void InsertDistinctPreAggregation(OptimizerExtensionInput &input, Logical // 5. Remap outer aggregate's groups to reference inner aggregate's group output for (idx_t i = 0; i < num_original_groups; i++) { - auto gtype = agg->groups[i]->return_type; - agg->groups[i] = make_uniq(gtype, ColumnBinding(inner_group_index, i)); + auto gtype = agg->groups[i]->GetReturnType(); + agg->groups[i] = make_uniq( + gtype, ColumnBinding(TableIndex(inner_group_index), ProjectionIndex(i))); } // Inner aggregate output layout: // groups: [g1, ..., gN, distinct_col] at inner_group_index // aggregates: [bit_or(hash)] at inner_agg_index - ColumnBinding combined_hash_binding(inner_agg_index, 0); - ColumnBinding distinct_val_binding(inner_group_index, num_original_groups); + ColumnBinding combined_hash_binding {TableIndex(inner_agg_index), ProjectionIndex(0)}; + ColumnBinding distinct_val_binding {TableIndex(inner_group_index), ProjectionIndex(num_original_groups)}; // 6. Replace each aggregate expression with the corresponding PAC function for (idx_t i = 0; i < agg->expressions.size(); i++) { auto &old_aggr = agg->expressions[i]->Cast(); - string function_name = old_aggr.function.name; + string function_name = old_aggr.FunctionMutable().GetName().GetIdentifierName(); string pac_name = GetPacAggregateFunctionName(function_name, &input.context); auto hash_ref = make_uniq(LogicalType::UBIGINT, combined_hash_binding); @@ -574,7 +584,7 @@ static void InsertDistinctPreAggregation(OptimizerExtensionInput &input, Logical value_ref = make_uniq_base(Value::BIGINT(1)); } else { // SUM/AVG/MIN/MAX DISTINCT: value comes from the distinct column (now a group key in inner agg) - value_ref = make_uniq(distinct_value_expr->return_type, distinct_val_binding); + value_ref = make_uniq(distinct_value_expr->GetReturnType(), distinct_val_binding); } agg->expressions[i] = BindPacAggregate(input, pac_name, std::move(hash_ref), std::move(value_ref)); @@ -599,14 +609,15 @@ BuildDistinctBranch(OptimizerExtensionInput &input, unique_ptr const vector> &agg_specs, // (original_agg_idx, original_func_name) idx_t &out_group_index, idx_t &out_agg_index) { auto &binder = input.optimizer.binder; - idx_t inner_group_index = binder.GenerateTableIndex(); - idx_t inner_agg_index = binder.GenerateTableIndex(); + idx_t inner_group_index = binder.GenerateTableIndex().index; + idx_t inner_agg_index = binder.GenerateTableIndex().index; // Inner aggregate: bit_or(hash) FILTER (distinct_col IS NOT NULL) GROUP BY [groups..., distinct_col] auto bit_or_expr = BindBitOrAggregate(input, hash_expr->Copy(), distinct_value_expr->Copy()); vector> inner_expressions; inner_expressions.push_back(std::move(bit_or_expr)); - auto inner_agg = make_uniq(inner_group_index, inner_agg_index, std::move(inner_expressions)); + auto inner_agg = make_uniq(TableIndex(inner_group_index), TableIndex(inner_agg_index), + std::move(inner_expressions)); idx_t num_groups = groups.size(); for (auto &g : groups) { @@ -617,11 +628,11 @@ BuildDistinctBranch(OptimizerExtensionInput &input, unique_ptr inner_agg->ResolveOperatorTypes(); // Outer aggregate: pac_*(combined_hash, value) GROUP BY [groups...] - idx_t outer_group_index = binder.GenerateTableIndex(); - idx_t outer_agg_index = binder.GenerateTableIndex(); + idx_t outer_group_index = binder.GenerateTableIndex().index; + idx_t outer_agg_index = binder.GenerateTableIndex().index; - ColumnBinding combined_hash_binding(inner_agg_index, 0); - ColumnBinding distinct_val_binding(inner_group_index, num_groups); + ColumnBinding combined_hash_binding {TableIndex(inner_agg_index), ProjectionIndex(0)}; + ColumnBinding distinct_val_binding {TableIndex(inner_group_index), ProjectionIndex(num_groups)}; vector> outer_expressions; for (auto &spec : agg_specs) { @@ -631,15 +642,17 @@ BuildDistinctBranch(OptimizerExtensionInput &input, unique_ptr if (spec.second == "count" || spec.second == "count_star") { value_ref = make_uniq_base(Value::BIGINT(1)); } else { - value_ref = make_uniq(distinct_value_expr->return_type, distinct_val_binding); + value_ref = make_uniq(distinct_value_expr->GetReturnType(), distinct_val_binding); } outer_expressions.push_back(BindPacAggregate(input, pac_name, std::move(hash_ref), std::move(value_ref))); } - auto outer_agg = make_uniq(outer_group_index, outer_agg_index, std::move(outer_expressions)); + auto outer_agg = make_uniq(TableIndex(outer_group_index), TableIndex(outer_agg_index), + std::move(outer_expressions)); for (idx_t i = 0; i < num_groups; i++) { - auto gtype = groups[i]->return_type; - outer_agg->groups.push_back(make_uniq(gtype, ColumnBinding(inner_group_index, i))); + auto gtype = groups[i]->GetReturnType(); + outer_agg->groups.push_back(make_uniq( + gtype, ColumnBinding(TableIndex(inner_group_index), ProjectionIndex(i)))); } outer_agg->children.push_back(std::move(inner_agg)); outer_agg->ResolveOperatorTypes(); @@ -657,26 +670,26 @@ static unique_ptr BuildNonDistinctBranch( const vector> &agg_specs, // (original_agg_idx, original_expr) idx_t &out_group_index, idx_t &out_agg_index) { auto &binder = input.optimizer.binder; - idx_t group_index = binder.GenerateTableIndex(); - idx_t agg_index = binder.GenerateTableIndex(); + idx_t group_index = binder.GenerateTableIndex().index; + idx_t agg_index = binder.GenerateTableIndex().index; vector> expressions; for (auto &spec : agg_specs) { auto &old_aggr = *spec.second; - string function_name = old_aggr.function.name; + string function_name = old_aggr.Function().GetName().GetIdentifierName(); string pac_name = GetPacAggregateFunctionName(function_name, &input.context); unique_ptr value_child; - if (old_aggr.children.empty()) { + if (old_aggr.GetChildren().empty()) { value_child = make_uniq_base(Value::BIGINT(1)); } else { - value_child = old_aggr.children[0]->Copy(); + value_child = old_aggr.GetChildren()[0]->Copy(); } expressions.push_back(BindPacAggregate(input, pac_name, hash_expr->Copy(), std::move(value_child))); } - auto agg_node = make_uniq(group_index, agg_index, std::move(expressions)); + auto agg_node = make_uniq(TableIndex(group_index), TableIndex(agg_index), std::move(expressions)); for (auto &g : groups) { agg_node->groups.push_back(g->Copy()); } @@ -690,11 +703,11 @@ static unique_ptr BuildNonDistinctBranch( // Helper: remap expression bindings using an index map static void RemapExpressionBindings(Expression &e, const std::unordered_map &map) { - if (e.type == ExpressionType::BOUND_COLUMN_REF) { + if (e.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e.Cast(); - auto it = map.find(col_ref.binding.table_index); + auto it = map.find(col_ref.BindingMutable().table_index.index); if (it != map.end()) { - col_ref.binding.table_index = it->second; + col_ref.BindingMutable().table_index.index = it->second; } } ExpressionIterator::EnumerateChildren(e, [&](Expression &child) { RemapExpressionBindings(child, map); }); @@ -729,20 +742,21 @@ JoinedDistinctAggregateBranches BuildJoinedDistinctAggregateBranches(OptimizerEx std::unordered_map distinct_col_to_branch; for (idx_t i = 0; i < agg->expressions.size(); i++) { auto &aggr = agg->expressions[i]->Cast(); - bool is_distinct = aggr.IsDistinct() && !aggr.children.empty(); + bool is_distinct = aggr.IsDistinct() && !aggr.GetChildrenMutable().empty(); if (is_distinct) { - string col_str = aggr.children[0]->ToString(); + string col_str = aggr.GetChildrenMutable()[0]->ToString(); auto it = distinct_col_to_branch.find(col_str); if (it != distinct_col_to_branch.end()) { - branches[it->second].distinct_agg_specs.emplace_back(i, aggr.function.name); + branches[it->second].distinct_agg_specs.emplace_back( + i, aggr.FunctionMutable().GetName().GetIdentifierName()); branches[it->second].original_agg_indices.push_back(i); } else { BranchInfo info; info.is_distinct = true; info.distinct_col_str = col_str; - info.distinct_col_expr = aggr.children[0]->Copy(); - info.distinct_agg_specs.emplace_back(i, aggr.function.name); + info.distinct_col_expr = aggr.GetChildrenMutable()[0]->Copy(); + info.distinct_agg_specs.emplace_back(i, aggr.FunctionMutable().GetName().GetIdentifierName()); info.original_agg_indices.push_back(i); distinct_col_to_branch[col_str] = branches.size(); branches.push_back(std::move(info)); @@ -810,7 +824,7 @@ JoinedDistinctAggregateBranches BuildJoinedDistinctAggregateBranches(OptimizerEx } auto &branch_agg = built.subtree->Cast(); for (auto &expr : branch_agg.expressions) { - built.aggregate_types.push_back(expr->return_type); + built.aggregate_types.push_back(expr->GetReturnType()); } built_branches.push_back(std::move(built)); } @@ -823,12 +837,13 @@ JoinedDistinctAggregateBranches BuildJoinedDistinctAggregateBranches(OptimizerEx if (has_groups) { auto join = make_uniq(JoinType::INNER); for (idx_t gi = 0; gi < agg->groups.size(); gi++) { - JoinCondition cond; - cond.left = make_uniq(agg->groups[gi]->return_type, - ColumnBinding(left_group_index, gi)); - cond.right = make_uniq(agg->groups[gi]->return_type, - ColumnBinding(right.group_index, gi)); - cond.comparison = ExpressionType::COMPARE_NOT_DISTINCT_FROM; + JoinCondition cond(nullptr, nullptr, ExpressionType::COMPARE_NOT_DISTINCT_FROM); + cond.LeftReference() = make_uniq( + agg->groups[gi]->GetReturnType(), ColumnBinding(TableIndex(left_group_index), ProjectionIndex(gi))); + cond.RightReference() = make_uniq( + agg->groups[gi]->GetReturnType(), + ColumnBinding(TableIndex(right.group_index), ProjectionIndex(gi))); + join->conditions.push_back(std::move(cond)); } join->children.push_back(std::move(joined)); @@ -884,8 +899,8 @@ static void InsertMultiBranchPreAggregation(OptimizerExtensionInput &input, uniq auto &branch_agg = subtree->Cast(); for (auto &expr : branch_agg.expressions) { auto &bound_agg = expr->Cast(); - if (bound_agg.children.size() > 1) { - RemapExpressionBindings(*bound_agg.children[1], *index_map); + if (bound_agg.GetChildrenMutable().size() > 1) { + RemapExpressionBindings(*bound_agg.GetChildrenMutable()[1], *index_map); } } subtree->ResolveOperatorTypes(); @@ -897,13 +912,14 @@ static void InsertMultiBranchPreAggregation(OptimizerExtensionInput &input, uniq PacBranchBuilder builder; auto branch_plan = BuildJoinedDistinctAggregateBranches(input, plan, agg, hash_input_expr->Copy(), builder); - idx_t proj_table_index = input.optimizer.binder.GenerateTableIndex(); + idx_t proj_table_index = input.optimizer.binder.GenerateTableIndex().index; vector> proj_expressions; idx_t num_groups = agg->groups.size(); for (idx_t gi = 0; gi < num_groups; gi++) { proj_expressions.push_back(make_uniq( - agg->groups[gi]->return_type, ColumnBinding(branch_plan.left_group_index, gi))); + agg->groups[gi]->GetReturnType(), + ColumnBinding(TableIndex(branch_plan.left_group_index), ProjectionIndex(gi)))); } for (idx_t ai = 0; ai < agg->expressions.size(); ai++) { @@ -915,22 +931,22 @@ static void InsertMultiBranchPreAggregation(OptimizerExtensionInput &input, uniq idx_t pos_in_branch = it->second.second; auto &branch = branch_plan.branches[branch_idx]; - auto original_type = agg->expressions[ai]->return_type; + auto original_type = agg->expressions[ai]->GetReturnType(); auto branch_type = branch.aggregate_types[pos_in_branch]; PRIVACY_DEBUG_PRINT("MultiBranch output: ai=" + std::to_string(ai) + " branch=" + std::to_string(branch_idx) + " pos=" + std::to_string(pos_in_branch) + " agg_index=" + std::to_string(branch.agg_index) + " original_type=" + original_type.ToString() + " branch_type=" + branch_type.ToString()); - unique_ptr ref = - make_uniq(branch_type, ColumnBinding(branch.agg_index, pos_in_branch)); + unique_ptr ref = make_uniq( + branch_type, ColumnBinding(TableIndex(branch.agg_index), ProjectionIndex(pos_in_branch))); if (branch_type != original_type) { ref = BoundCastExpression::AddCastToType(input.context, std::move(ref), original_type); } proj_expressions.push_back(std::move(ref)); } - auto projection = make_uniq(proj_table_index, std::move(proj_expressions)); + auto projection = make_uniq(TableIndex(proj_table_index), std::move(proj_expressions)); projection->children.push_back(std::move(branch_plan.joined)); projection->ResolveOperatorTypes(); @@ -939,18 +955,19 @@ static void InsertMultiBranchPreAggregation(OptimizerExtensionInput &input, uniq throw InternalException("Privacy compiler: InsertMultiBranchPreAggregation could not find aggregate in plan"); } - idx_t old_group_index = agg->group_index; - idx_t old_agg_index = agg->aggregate_index; + idx_t old_group_index = agg->group_index.index; + idx_t old_agg_index = agg->aggregate_index.index; idx_t num_agg_exprs = agg->expressions.size(); ColumnBindingReplacer replacer; for (idx_t gi = 0; gi < num_groups; gi++) { - replacer.replacement_bindings.emplace_back(ColumnBinding(old_group_index, gi), - ColumnBinding(proj_table_index, gi)); + replacer.replacement_bindings.emplace_back(ColumnBinding(TableIndex(old_group_index), ProjectionIndex(gi)), + ColumnBinding(TableIndex(proj_table_index), ProjectionIndex(gi))); } for (idx_t ai = 0; ai < num_agg_exprs; ai++) { - replacer.replacement_bindings.emplace_back(ColumnBinding(old_agg_index, ai), - ColumnBinding(proj_table_index, num_groups + ai)); + replacer.replacement_bindings.emplace_back( + ColumnBinding(TableIndex(old_agg_index), ProjectionIndex(ai)), + ColumnBinding(TableIndex(proj_table_index), ProjectionIndex(num_groups + ai))); } LogicalOperator *proj_ptr = projection.get(); @@ -1020,16 +1037,16 @@ void ModifyAggregatesWithPacFunctions(OptimizerExtensionInput &input, LogicalAgg // Debug: Print hash input expression details PRIVACY_DEBUG_PRINT("ModifyAggregatesWithPacFunctions: Hash input expression: " + hash_input_expr->ToString()); PRIVACY_DEBUG_PRINT("ModifyAggregatesWithPacFunctions: Hash input type: " + - hash_input_expr->return_type.ToString()); + hash_input_expr->GetReturnType().ToString()); // Debug: Print all column references in the hash expression ExpressionIterator::EnumerateExpression(hash_input_expr, [&](Expression &expr) { - if (expr.type == ExpressionType::BOUND_COLUMN_REF) { + if (expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr.Cast(); PRIVACY_DEBUG_PRINT("ModifyAggregatesWithPacFunctions: Hash expr references column [" + - std::to_string(col_ref.binding.table_index) + "." + - std::to_string(col_ref.binding.column_index) + - "] type=" + col_ref.return_type.ToString()); + std::to_string(col_ref.Binding().table_index.index) + "." + + std::to_string(col_ref.Binding().column_index) + + "] type=" + col_ref.GetReturnType().ToString()); } }); @@ -1041,8 +1058,8 @@ void ModifyAggregatesWithPacFunctions(OptimizerExtensionInput &input, LogicalAgg } // Debug: Print aggregate's group_index and aggregate_index - PRIVACY_DEBUG_PRINT("ModifyAggregatesWithPacFunctions: group_index=" + std::to_string(agg->group_index) + - ", aggregate_index=" + std::to_string(agg->aggregate_index)); + PRIVACY_DEBUG_PRINT("ModifyAggregatesWithPacFunctions: group_index=" + std::to_string(agg->group_index.index) + + ", aggregate_index=" + std::to_string(agg->aggregate_index.index)); // Debug: Print child operator info if (!agg->children.empty()) { @@ -1051,7 +1068,7 @@ void ModifyAggregatesWithPacFunctions(OptimizerExtensionInput &input, LogicalAgg if (agg->children[0]->type == LogicalOperatorType::LOGICAL_GET) { auto &child_get = agg->children[0]->Cast(); PRIVACY_DEBUG_PRINT("ModifyAggregatesWithPacFunctions: Child is GET with table_index=" + - std::to_string(child_get.table_index) + + std::to_string(child_get.table_index.index) + ", columns=" + std::to_string(child_get.GetColumnIds().size())); } } @@ -1064,7 +1081,8 @@ void ModifyAggregatesWithPacFunctions(OptimizerExtensionInput &input, LogicalAgg throw NotImplementedException("Not found expected aggregate expression in PAC compiler"); } auto &check_aggr = agg->expressions[i]->Cast(); - GetPacAggregateFunctionName(check_aggr.function.name); // throws if unsupported + GetPacAggregateFunctionName( + check_aggr.FunctionMutable().GetName().GetIdentifierName()); // throws if unsupported } // === DISTINCT pre-aggregation === @@ -1079,11 +1097,11 @@ void ModifyAggregatesWithPacFunctions(OptimizerExtensionInput &input, LogicalAgg for (idx_t i = 0; i < agg->expressions.size(); i++) { auto &check = agg->expressions[i]->Cast(); - if (check.IsDistinct() && !check.children.empty()) { + if (check.IsDistinct() && !check.GetChildrenMutable().empty()) { has_any_distinct = true; if (!common_distinct_value) { - common_distinct_value = check.children[0]->Copy(); - } else if (check.children[0]->ToString() != common_distinct_value->ToString()) { + common_distinct_value = check.GetChildrenMutable()[0]->Copy(); + } else if (check.GetChildrenMutable()[0]->ToString() != common_distinct_value->ToString()) { all_same_col = false; } } else { @@ -1110,7 +1128,7 @@ void ModifyAggregatesWithPacFunctions(OptimizerExtensionInput &input, LogicalAgg } auto &old_aggr = agg->expressions[i]->Cast(); - string function_name = old_aggr.function.name; + string function_name = old_aggr.FunctionMutable().GetName().GetIdentifierName(); #if PRIVACY_DEBUG PRIVACY_DEBUG_PRINT("ModifyAggregatesWithPacFunctions: Transforming " + function_name + " to PAC function"); @@ -1118,14 +1136,14 @@ void ModifyAggregatesWithPacFunctions(OptimizerExtensionInput &input, LogicalAgg // Extract the original aggregate's value child expression unique_ptr value_child; - if (old_aggr.children.empty()) { + if (old_aggr.GetChildrenMutable().empty()) { if (function_name == "count_star" || function_name == "count") { value_child = make_uniq_base(Value::BIGINT(1)); } else { throw InternalException("Privacy compiler: expected aggregate to have a child expression"); } } else { - value_child = old_aggr.children[0]->Copy(); + value_child = old_aggr.GetChildrenMutable()[0]->Copy(); } string pac_function_name = GetPacAggregateFunctionName(function_name, &input.context); @@ -1180,11 +1198,11 @@ unique_ptr BindPlainAggregate(OptimizerExtensionInput &input, const ErrorData error; vector arg_types; for (auto &child : children) { - arg_types.push_back(child->return_type); + arg_types.push_back(child->GetReturnType()); } auto &entry = Catalog::GetSystemCatalog(input.context) - .GetEntry(input.context, DEFAULT_SCHEMA, func_name); - auto best = function_binder.BindFunction(entry.name, entry.functions, arg_types, error); + .GetEntry(input.context, DEFAULT_SCHEMA, Identifier(func_name)); + auto best = function_binder.BindFunction(Identifier(entry.name), entry.functions, arg_types, error); if (!best.IsValid()) { throw InternalException("Privacy compiler: failed to bind aggregate '" + func_name + "'"); } @@ -1208,7 +1226,7 @@ static bool IsPacAggregate(LogicalAggregate *agg) { continue; } auto &aggr = expr->Cast(); - if (!GetClipVariant(aggr.function.name).empty()) { + if (!GetClipVariant(aggr.FunctionMutable().GetName().GetIdentifierName()).empty()) { return true; } } @@ -1241,13 +1259,13 @@ void RewriteClipAggregates(OptimizerExtensionInput &input, unique_ptrexpressions[i]->Cast(); - string clip_name = GetClipVariant(aggr.function.name); + string clip_name = GetClipVariant(aggr.FunctionMutable().GetName().GetIdentifierName()); if (clip_name.empty()) { continue; } // Rebind with the clip variant name vector> children; - for (auto &child : aggr.children) { + for (auto &child : aggr.GetChildrenMutable()) { children.push_back(child->Copy()); } agg->expressions[i] = RebindAggregate(input.context, clip_name, std::move(children), false); @@ -1263,8 +1281,9 @@ void RewriteClipAggregates(OptimizerExtensionInput &input, unique_ptrCast(); - if (!GetClipVariant(aggr.function.name).empty() && !aggr.children.empty()) { - pu_hash_expr = aggr.children[0]->Copy(); + if (!GetClipVariant(aggr.FunctionMutable().GetName().GetIdentifierName()).empty() && + !aggr.GetChildrenMutable().empty()) { + pu_hash_expr = aggr.GetChildrenMutable()[0]->Copy(); break; } } @@ -1281,21 +1300,22 @@ void RewriteClipAggregates(OptimizerExtensionInput &input, unique_ptr> lower_expressions; for (idx_t i = 0; i < agg->expressions.size(); i++) { auto &aggr = agg->expressions[i]->Cast(); - string orig_name = GetOriginalAggregate(aggr.function.name); + string orig_name = GetOriginalAggregate(aggr.FunctionMutable().GetName().GetIdentifierName()); if (orig_name.empty()) { - throw InternalException("PAC clip rewrite: unexpected aggregate " + aggr.function.name); + throw InternalException("PAC clip rewrite: unexpected aggregate " + + aggr.FunctionMutable().GetName().GetIdentifierName()); } vector> plain_children; - if (orig_name == "count" && (aggr.children.size() <= 1)) { + if (orig_name == "count" && (aggr.GetChildrenMutable().size() <= 1)) { // as_noised_count(hash) or as_count(hash) → count_star() // as_noised_count(hash, col) → count(col) — but children[1] might be constant 1 - if (aggr.children.size() >= 2) { - auto &val_child = aggr.children[1]; + if (aggr.GetChildrenMutable().size() >= 2) { + auto &val_child = aggr.GetChildrenMutable()[1]; // Check if it's a constant 1 (from count_star rewrite) - if (val_child->type == ExpressionType::VALUE_CONSTANT) { + if (val_child->GetExpressionType() == ExpressionType::VALUE_CONSTANT) { auto &const_expr = val_child->Cast(); - if (const_expr.value.IsNull() || const_expr.value == Value::BIGINT(1)) { + if (const_expr.GetValue().IsNull() || const_expr.GetValue() == Value::BIGINT(1)) { // count_star — no children } else { plain_children.push_back(val_child->Copy()); @@ -1305,14 +1325,14 @@ void RewriteClipAggregates(OptimizerExtensionInput &input, unique_ptr 1) { + } else if (orig_name == "count" && aggr.GetChildrenMutable().size() > 1) { // count with column reference - plain_children.push_back(aggr.children[1]->Copy()); + plain_children.push_back(aggr.GetChildrenMutable()[1]->Copy()); lower_expressions.push_back(BindPlainAggregate(input, "count", std::move(plain_children))); } else { // sum, min, max — extract the value child (children[1]) - if (aggr.children.size() >= 2) { - plain_children.push_back(aggr.children[1]->Copy()); + if (aggr.GetChildrenMutable().size() >= 2) { + plain_children.push_back(aggr.GetChildrenMutable()[1]->Copy()); } lower_expressions.push_back(BindPlainAggregate(input, orig_name, std::move(plain_children))); } @@ -1323,14 +1343,14 @@ void RewriteClipAggregates(OptimizerExtensionInput &input, unique_ptrexpressions.size(); i++) { auto &aggr = agg->expressions[i]->Cast(); - string pac_name = aggr.function.name; + string pac_name = aggr.FunctionMutable().GetName().GetIdentifierName(); bool noised = IsNoisedVariant(pac_name); string orig = GetOriginalAggregate(pac_name); // Reference to lower aggregate's result auto lower_type = pre_agg.lower_agg->types[pre_agg.num_original_groups + 1 + i]; - unique_ptr lower_ref = - make_uniq(lower_type, ColumnBinding(pre_agg.lower_agg_index, i)); + unique_ptr lower_ref = make_uniq( + lower_type, ColumnBinding(TableIndex(pre_agg.lower_agg_index), ProjectionIndex(i))); // as_clip_sum has integer + DECIMAL overloads but no FLOAT/DOUBLE. // Cast FLOAT/DOUBLE to BIGINT so binding succeeds. diff --git a/src/query_processing/privacy_fk_resolver.cpp b/src/query_processing/privacy_fk_resolver.cpp index 80e54cc5..6c23d0ab 100644 --- a/src/query_processing/privacy_fk_resolver.cpp +++ b/src/query_processing/privacy_fk_resolver.cpp @@ -45,7 +45,7 @@ AccessibleFKTableResult FindAccessibleFKTable(unique_ptr &plan, for (auto *node : table_nodes) { auto &node_get = node->get()->Cast(); - idx_t node_table_idx = node_get.table_index; + idx_t node_table_idx = node_get.table_index.index; // Check if this table is in the search root's subtree if (search_root && !HasTableIndexInSubtree(search_root, node_table_idx)) { @@ -55,7 +55,7 @@ AccessibleFKTableResult FindAccessibleFKTable(unique_ptr &plan, // Check if this table's columns are accessible if (AreTableColumnsAccessible(plan.get(), node_table_idx)) { result.get_node = &node_get; - result.table_index = node_table_idx; + result.table_index.index = node_table_idx; result.table_name = table_name; result.fk_columns = fk_result.fk_columns; result.found = true; diff --git a/src/query_processing/privacy_plan_traversal.cpp b/src/query_processing/privacy_plan_traversal.cpp index 46c4a377..2429f6dd 100644 --- a/src/query_processing/privacy_plan_traversal.cpp +++ b/src/query_processing/privacy_plan_traversal.cpp @@ -40,7 +40,7 @@ unique_ptr *FindPrivacyUnitGetNode(unique_ptr if (!pu_table_name.empty()) { auto &get = cur->Cast(); auto tblptr = get.GetTable(); - if (tblptr && tblptr->name == pu_table_name) { + if (tblptr && tblptr->name.GetIdentifierName() == pu_table_name) { found_ptr = cur_ptr; break; } @@ -76,7 +76,7 @@ LogicalGet *FindTableScanInSubtree(LogicalOperator *subtree, const string &table if (subtree->type == LogicalOperatorType::LOGICAL_GET) { auto &get = subtree->Cast(); auto tblptr = get.GetTable(); - if (tblptr && StringUtil::CIEquals(tblptr->name, table_name)) { + if (tblptr && StringUtil::CIEquals(tblptr->name.GetIdentifierName(), table_name)) { return &get; } } @@ -160,7 +160,7 @@ unique_ptr *FindNodeRefByTable(unique_ptr *roo if (cur->type == LogicalOperatorType::LOGICAL_GET) { auto &get = cur->Cast(); auto tblptr = get.GetTable(); - if (tblptr && tblptr->name == table_name) { + if (tblptr && tblptr->name.GetIdentifierName() == table_name) { if (parent_out) { *parent_out = entry.parent; } @@ -239,7 +239,7 @@ bool HasTableInSubtree(LogicalOperator *op, const string &table_name) { if (op->type == LogicalOperatorType::LOGICAL_GET) { auto &get = op->Cast(); auto tblptr = get.GetTable(); - if (tblptr && tblptr->name == table_name) { + if (tblptr && tblptr->name.GetIdentifierName() == table_name) { return true; } } @@ -265,7 +265,7 @@ static void CollectTableNamesInSubtree(LogicalOperator *op, std::unordered_setCast(); auto tblptr = get.GetTable(); if (tblptr) { - table_names.insert(tblptr->name); + table_names.insert(tblptr->name.GetIdentifierName()); } } for (auto &child : op->children) { @@ -282,7 +282,7 @@ void BuildCTETableMap(LogicalOperator *op, CTETableMap &cte_map) { if (op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE) { auto &cte = op->Cast(); - idx_t cte_idx = cte.table_index; + idx_t cte_idx = cte.table_index.index; // children[0] = CTE definition, children[1] = consumer if (!cte.children.empty()) { @@ -305,7 +305,7 @@ static void MergeCTERefsIntoSet(LogicalOperator *op, idx_t target_cte_idx, CTETa } if (op->type == LogicalOperatorType::LOGICAL_CTE_REF) { auto &ref = op->Cast(); - auto it = cte_map.find(ref.cte_index); + auto it = cte_map.find(ref.cte_index.index); if (it != cte_map.end()) { cte_map[target_cte_idx].insert(it->second.begin(), it->second.end()); } @@ -324,7 +324,7 @@ static void ResolveCTERefsInDefinitions(LogicalOperator *op, CTETableMap &cte_ma if (op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE) { auto &cte = op->Cast(); if (!cte.children.empty()) { - MergeCTERefsIntoSet(cte.children[0].get(), cte.table_index, cte_map); + MergeCTERefsIntoSet(cte.children[0].get(), cte.table_index.index, cte_map); } } for (auto &child : op->children) { @@ -370,7 +370,7 @@ bool HasTableInSubtreeCTE(LogicalOperator *op, const string &table_name, const C if (op->type == LogicalOperatorType::LOGICAL_GET) { auto &get = op->Cast(); auto tblptr = get.GetTable(); - if (tblptr && tblptr->name == table_name) { + if (tblptr && tblptr->name.GetIdentifierName() == table_name) { return true; } } @@ -378,7 +378,7 @@ bool HasTableInSubtreeCTE(LogicalOperator *op, const string &table_name, const C // Follow CTE references through the map if (op->type == LogicalOperatorType::LOGICAL_CTE_REF) { auto &ref = op->Cast(); - auto it = cte_map.find(ref.cte_index); + auto it = cte_map.find(ref.cte_index.index); if (it != cte_map.end() && it->second.count(table_name) > 0) { return true; } @@ -406,7 +406,7 @@ void FindAllNodesByTable(unique_ptr *root, const string &table_ if (cur->type == LogicalOperatorType::LOGICAL_GET) { auto &get = cur->Cast(); auto tblptr = get.GetTable(); - if (tblptr && tblptr->name == table_name) { + if (tblptr && tblptr->name.GetIdentifierName() == table_name) { results.push_back(root); } } @@ -438,7 +438,7 @@ bool HasTableIndexInSubtree(LogicalOperator *op, idx_t table_index) { // Check if this is a LogicalGet with the target table index if (op->type == LogicalOperatorType::LOGICAL_GET) { auto &get = op->Cast(); - if (get.table_index == table_index) { + if (get.table_index.index == table_index) { return true; } } @@ -461,22 +461,22 @@ LogicalOperator *FindOperatorByTableIndex(LogicalOperator *op, idx_t table_index } if (op->type == LogicalOperatorType::LOGICAL_GET) { auto &get = op->Cast(); - if (get.table_index == table_index) { + if (get.table_index.index == table_index) { return op; } } else if (op->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY) { auto &aggr = op->Cast(); - if (aggr.group_index == table_index || aggr.aggregate_index == table_index) { + if (aggr.group_index.index == table_index || aggr.aggregate_index.index == table_index) { return op; } } else if (op->type == LogicalOperatorType::LOGICAL_PROJECTION) { auto &proj = op->Cast(); - if (proj.table_index == table_index) { + if (proj.table_index.index == table_index) { return op; } } else if (op->type == LogicalOperatorType::LOGICAL_CTE_REF) { auto &cte_ref = op->Cast(); - if (cte_ref.table_index == table_index) { + if (cte_ref.table_index.index == table_index) { return op; } } @@ -496,7 +496,7 @@ LogicalMaterializedCTE *FindMaterializedCTE(LogicalOperator *op, idx_t cte_table } if (op->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE) { auto &cte = op->Cast(); - if (cte.table_index == cte_table_index) { + if (cte.table_index.index == cte_table_index) { return &cte; } } @@ -521,7 +521,7 @@ void FindAllNodesByTableIndex(unique_ptr *root, idx_t table_ind // Check if this is a LogicalGet with the target table index if (cur->type == LogicalOperatorType::LOGICAL_GET) { auto &get = cur->Cast(); - if (get.table_index == table_index) { + if (get.table_index.index == table_index) { results.push_back(root); } } @@ -631,7 +631,7 @@ static bool CheckColumnsAccessible(LogicalOperator *op, idx_t table_index) { // If this is the target table, it's accessible from here if (op->type == LogicalOperatorType::LOGICAL_GET) { auto &get = op->Cast(); - if (get.table_index == table_index) { + if (get.table_index.index == table_index) { return true; } } @@ -732,7 +732,7 @@ static LogicalGet *FindLogicalGetByTableIndex(LogicalOperator *op, idx_t table_i } if (op->type == LogicalOperatorType::LOGICAL_GET) { auto &get = op->Cast(); - if (get.table_index == table_index) { + if (get.table_index.index == table_index) { return &get; } } @@ -751,7 +751,7 @@ static std::pair GetColumnInfoFromBinding(LogicalOperator *subtr return {"", ""}; } - auto *get = FindLogicalGetByTableIndex(subtree, binding.table_index); + auto *get = FindLogicalGetByTableIndex(subtree, binding.table_index.index); if (!get) { return {"", ""}; } @@ -761,7 +761,7 @@ static std::pair GetColumnInfoFromBinding(LogicalOperator *subtr return {"", ""}; } - string table_name = table_entry->name; + string table_name = table_entry->name.GetIdentifierName(); // Get column name from the binding const auto &column_ids = get->GetColumnIds(); @@ -769,7 +769,7 @@ static std::pair GetColumnInfoFromBinding(LogicalOperator *subtr return {table_name, ""}; } - string col_name = get->GetColumnName(column_ids[binding.column_index]); + string col_name = get->GetColumnName(column_ids[binding.column_index]).GetIdentifierName(); return {table_name, col_name}; } @@ -788,9 +788,9 @@ bool AggregateGroupsByPUKey(LogicalAggregate *agg, const PrivacyCompatibilityRes continue; } ExpressionIterator::EnumerateExpression(group_expr, [&](Expression &expr) { - if (expr.type == ExpressionType::BOUND_COLUMN_REF) { + if (expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = expr.Cast(); - group_bindings.push_back(col_ref.binding); + group_bindings.push_back(col_ref.BindingMutable()); } }); } @@ -805,7 +805,7 @@ bool AggregateGroupsByPUKey(LogicalAggregate *agg, const PrivacyCompatibilityRes auto &table_name = col_info.first; auto &col_name = col_info.second; #if PRIVACY_DEBUG - PRIVACY_DEBUG_PRINT("AggregateGroupsByPUKey: binding [" + std::to_string(binding.table_index) + "." + + PRIVACY_DEBUG_PRINT("AggregateGroupsByPUKey: binding [" + std::to_string(binding.table_index.index) + "." + std::to_string(binding.column_index) + "] -> table='" + table_name + "' col='" + col_name + "'"); #endif @@ -890,6 +890,11 @@ vector FilterTargetAggregatesWithPUKeyCheck(const vector include_outer_aggregates; for (auto *agg : all_aggregates) { + // Correlation-domain deduplication is a GROUP BY without aggregate + // expressions. It must not move privacy responsibility to another node. + if (agg->expressions.empty()) { + continue; + } // Check if this aggregate has a child aggregate (nested aggregate — direct subtree) for (auto &child : agg->children) { auto *inner_agg = FindFirstChildAggregate(child.get()); @@ -919,7 +924,7 @@ vector FilterTargetAggregatesWithPUKeyCheck(const vectorexpressions.empty()) { continue; } bool other_has_direct = false; @@ -948,6 +953,11 @@ vector FilterTargetAggregatesWithPUKeyCheck(const vectorexpressions.empty()) { + continue; + } // Skip aggregates that are marked to be skipped (inner aggregates that group by PU key) if (skip_aggregates.count(agg) > 0) { #if PRIVACY_DEBUG @@ -1052,12 +1062,12 @@ LogicalAggregate *FindInnerAggregateWithPUKeyGroup(LogicalAggregate *target_agg, // Now find which group column is the PU key and get its output binding for (idx_t group_idx = 0; group_idx < inner_agg->groups.size(); group_idx++) { auto &group_expr = inner_agg->groups[group_idx]; - if (!group_expr || group_expr->type != ExpressionType::BOUND_COLUMN_REF) { + if (!group_expr || group_expr->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) { continue; } auto &col_ref = group_expr->Cast(); - auto col_info = GetColumnInfoFromBinding(inner_agg, col_ref.binding); + auto col_info = GetColumnInfoFromBinding(inner_agg, col_ref.BindingMutable()); auto &table_name = col_info.first; auto &col_name = col_info.second; if (table_name.empty() || col_name.empty()) { @@ -1076,7 +1086,7 @@ LogicalAggregate *FindInnerAggregateWithPUKeyGroup(LogicalAggregate *target_agg, // Found the PU key group column // The group columns are output as the first N columns of the aggregate // Their bindings use the aggregate's group_index - ColumnBinding current_binding(inner_agg->group_index, group_idx); + ColumnBinding current_binding {TableIndex(inner_agg->group_index.index), ProjectionIndex(group_idx)}; // Now propagate this binding through any projections between inner_agg and target_agg // The path_to_inner contains operators from outer's child down to (but not including) inner_agg @@ -1090,12 +1100,13 @@ LogicalAggregate *FindInnerAggregateWithPUKeyGroup(LogicalAggregate *target_agg, bool found = false; for (idx_t expr_idx = 0; expr_idx < proj.expressions.size(); expr_idx++) { auto &expr = proj.expressions[expr_idx]; - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { + if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &proj_col_ref = expr->Cast(); - if (proj_col_ref.binding == current_binding) { + if (proj_col_ref.BindingMutable() == current_binding) { // This projection expression references our binding // The new binding is the projection's output - current_binding = ColumnBinding(proj.table_index, expr_idx); + current_binding = + ColumnBinding(TableIndex(proj.table_index.index), ProjectionIndex(expr_idx)); found = true; break; } diff --git a/src/table_functions/dp_sass_stability_query.cpp b/src/table_functions/dp_sass_stability_query.cpp index 80a77210..b85d6486 100644 --- a/src/table_functions/dp_sass_stability_query.cpp +++ b/src/table_functions/dp_sass_stability_query.cpp @@ -32,13 +32,13 @@ struct DpSassStabilityQueryState : public GlobalTableFunctionState { idx_t offset = 0; }; -static void AddOutputColumn(vector &return_types, vector &names, const string &name, +static void AddOutputColumn(vector &return_types, vector &names, const string &name, const LogicalType &type) { - names.push_back(name); + names.emplace_back(name); return_types.push_back(type); } -static void AddStabilityOutputColumns(vector &return_types, vector &names) { +static void AddStabilityOutputColumns(vector &return_types, vector &names) { AddOutputColumn(return_types, names, "group_row", LogicalType::BIGINT); AddOutputColumn(return_types, names, "aggregate_index", LogicalType::INTEGER); AddOutputColumn(return_types, names, "aggregate_name", LogicalType::VARCHAR); @@ -59,7 +59,7 @@ static void AddStabilityOutputColumns(vector &return_types, vector< AddOutputColumn(return_types, names, "stable_mad_1", LogicalType::BOOLEAN); } -static void AddNoiseScaleOutputColumns(vector &return_types, vector &names) { +static void AddNoiseScaleOutputColumns(vector &return_types, vector &names) { AddOutputColumn(return_types, names, "group_row", LogicalType::BIGINT); AddOutputColumn(return_types, names, "aggregate_index", LogicalType::INTEGER); AddOutputColumn(return_types, names, "aggregate_name", LogicalType::VARCHAR); @@ -113,7 +113,7 @@ static void RunSetting(Connection &conn, const string &sql) { static void CopySetting(ClientContext &context, Connection &conn, const string &setting_name) { Value value; - if (!context.TryGetCurrentSetting(setting_name, value) || value.IsNull()) { + if (!context.TryGetCurrentSetting(Identifier(setting_name), value) || value.IsNull()) { return; } RunSetting(conn, "SET " + setting_name + " = " + value.ToSQLString()); @@ -179,7 +179,7 @@ static string BuildGroupKeyJson(QueryResult &result, const vector &row, i json += ","; } json += "\""; - json += EscapeJsonString(result.names[col]); + json += EscapeJsonString(result.GetNames()[col].GetIdentifierName()); json += "\":"; auto &value = row[col]; if (value.IsNull()) { @@ -266,8 +266,8 @@ ExecuteDiagnosticQuery(ClientContext &context, const string &sql, const string & values.push_back(Value::BIGINT(static_cast(group_row))); values.push_back(Value::INTEGER(record.aggregate_index)); idx_t result_name_idx = group_cols + aggregate_index; - if (result_columns_align && result_name_idx < result->names.size()) { - values.push_back(Value(result->names[result_name_idx])); + if (result_columns_align && result_name_idx < result->GetNames().size()) { + values.push_back(Value(result->GetNames()[result_name_idx].GetIdentifierName())); } else { values.push_back(Value("agg_" + std::to_string(record.aggregate_index))); } @@ -302,7 +302,7 @@ static vector ExecuteNoiseScaleQuery(ClientContext &con } static unique_ptr DpSassStabilityQueryBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, vector &names) { if (input.inputs.empty() || input.inputs[0].IsNull()) { throw BinderException("dp_sass_stability_query requires a non-NULL SQL string"); } @@ -312,7 +312,8 @@ static unique_ptr DpSassStabilityQueryBind(ClientContext &context, } static unique_ptr DpSassNoiseScaleQueryBind(ClientContext &context, TableFunctionBindInput &input, - vector &return_types, vector &names) { + vector &return_types, + vector &names) { if (input.inputs.empty() || input.inputs[0].IsNull()) { throw BinderException("dp_sass_noise_scale_query requires a non-NULL SQL string"); } diff --git a/src/test/test_compiler_functions.cpp b/src/test/test_compiler_functions.cpp index 250c241c..1fbe653b 100644 --- a/src/test/test_compiler_functions.cpp +++ b/src/test/test_compiler_functions.cpp @@ -48,7 +48,7 @@ static void CollectColumnBindingsRecursiveForTest(LogicalOperator *node, std::se } auto binds = node->GetColumnBindings(); for (auto &b : binds) { - out.insert({b.table_index, b.column_index}); + out.insert({b.table_index.index, b.column_index}); } for (auto &c : node->children) { CollectColumnBindingsRecursiveForTest(c.get(), out); @@ -63,11 +63,11 @@ static bool VerifyAllExprBindingsExist(LogicalOperator *node, const std::setexpressions) { ExpressionIterator::VisitExpression(*expr, [&](const BoundColumnRefExpression &bcr) { - auto key = std::make_pair(bcr.binding.table_index, bcr.binding.column_index); + auto key = std::make_pair(bcr.Binding().table_index.index, bcr.Binding().column_index); if (bindings.find(key) == bindings.end()) { std::cerr << "Verification failure: expression references missing binding (table=" - << bcr.binding.table_index << ", col=" << bcr.binding.column_index << ") in operator " - << node->GetName() << "\n"; + << bcr.Binding().table_index.index << ", col=" << bcr.Binding().column_index + << ") in operator " << node->GetName() << "\n"; ok = false; } }); @@ -98,7 +98,7 @@ static void CollectBindingProducers(LogicalOperator *node, } auto binds = node->GetColumnBindings(); for (auto &b : binds) { - out[{b.table_index, b.column_index}].push_back(node); + out[{b.table_index.index, b.column_index}].push_back(node); } for (auto &c : node->children) { CollectBindingProducers(c.get(), out); @@ -200,7 +200,7 @@ static bool VerifyExprTypesMatchProducers(LogicalOperator *root) { for (auto &expr : node->expressions) { ExpressionIterator::VisitExpression( *expr, [&](const BoundColumnRefExpression &bcr) { - auto key = std::make_pair(bcr.binding.table_index, bcr.binding.column_index); + auto key = std::make_pair(bcr.Binding().table_index.index, bcr.Binding().column_index); auto it = producers.find(key); if (it == producers.end() || it->second.empty()) { // missing producer already reported elsewhere; skip @@ -208,17 +208,18 @@ static bool VerifyExprTypesMatchProducers(LogicalOperator *root) { } auto *producer = it->second[0]; auto types = producer->types; - if (bcr.binding.column_index >= (idx_t)types.size()) { + if (bcr.Binding().column_index >= (idx_t)types.size()) { std::cerr << "Verification failure: expression references column index " - << bcr.binding.column_index << " beyond producer types size " << types.size() << "\n"; + << bcr.Binding().column_index << " beyond producer types size " << types.size() + << "\n"; ok = false; return; } - const auto &prod_type = types[bcr.binding.column_index]; - if (bcr.return_type.id() != prod_type.id()) { + const auto &prod_type = types[bcr.Binding().column_index]; + if (bcr.GetReturnType().id() != prod_type.id()) { std::cerr << "Verification failure: expression type mismatch for binding (table=" - << bcr.binding.table_index << ", col=" << bcr.binding.column_index - << "): expr_type=" << bcr.return_type.ToString() + << bcr.Binding().table_index.index << ", col=" << bcr.Binding().column_index + << "): expr_type=" << bcr.GetReturnType().ToString() << " producer_type=" << prod_type.ToString() << "\n"; ok = false; } @@ -261,7 +262,7 @@ static bool VerifyOperatorBindingsConsistentWithTypes(LogicalOperator *root) { for (auto &expr : node->expressions) { // Enumerate all expressions under this unique_ptr and check their return types ExpressionIterator::EnumerateExpression(expr, [&](const Expression &e) { - if (e.return_type.id() == LogicalTypeId::INVALID) { + if (e.GetReturnType().id() == LogicalTypeId::INVALID) { std::cerr << "Verification failure: expression in operator " << node->GetName() << " has INVALID return type\n"; ok = false; @@ -342,18 +343,20 @@ static int RunRobustPlanVerification(const unique_ptr &root) { // Test 1: basic ReplaceNode insertion and binding remap static int Test_BasicReplaceNode() { // Build a simple plan: root projection (table_index = 0) with a child DummyScan (table_index = 0) - auto child = make_uniq(0); + auto child = make_uniq(TableIndex(0)); vector> proj_exprs; // Projection expression referencing child's first column (binding {0,0}) - proj_exprs.emplace_back(make_uniq(LogicalType::INTEGER, ColumnBinding(0, 0))); + proj_exprs.emplace_back( + make_uniq(LogicalType::INTEGER, ColumnBinding(TableIndex(0), ProjectionIndex(0)))); // store root as unique_ptr so it can be passed to ReplaceNode - unique_ptr root = make_uniq(0, std::move(proj_exprs)); + unique_ptr root = make_uniq(TableIndex(0), std::move(proj_exprs)); root->children.emplace_back(std::move(child)); // Prepare a new parent projection that contains its own child (strict replace semantics) - unique_ptr new_parent = make_uniq(1, vector>()); - new_parent->children.emplace_back(make_uniq(5)); + unique_ptr new_parent = + make_uniq(TableIndex(1), vector>()); + new_parent->children.emplace_back(make_uniq(TableIndex(5))); ReplaceNode(root, root->children[0], new_parent); // After insertion, validate @@ -383,16 +386,18 @@ static int Test_BasicReplaceNode() { // Test 2: multiple consecutive ReplaceNode calls produce consistent remapping static int Test_ConsecutiveReplaceNode() { // Start with root projection (table_index = 0) and a DummyScan child (table_index = 0) - auto child = make_uniq(0); + auto child = make_uniq(TableIndex(0)); vector> proj_exprs; - proj_exprs.emplace_back(make_uniq(LogicalType::INTEGER, ColumnBinding(0, 0))); - unique_ptr root = make_uniq(0, std::move(proj_exprs)); + proj_exprs.emplace_back( + make_uniq(LogicalType::INTEGER, ColumnBinding(TableIndex(0), ProjectionIndex(0)))); + unique_ptr root = make_uniq(TableIndex(0), std::move(proj_exprs)); root->children.emplace_back(std::move(child)); // First insertion - unique_ptr new_parent1 = make_uniq(1, vector>()); - new_parent1->children.emplace_back(make_uniq(7)); + unique_ptr new_parent1 = + make_uniq(TableIndex(1), vector>()); + new_parent1->children.emplace_back(make_uniq(TableIndex(7))); ReplaceNode(root, root->children[0], new_parent1); // Second insertion above the same original child @@ -400,8 +405,9 @@ static int Test_ConsecutiveReplaceNode() { if (!root->children[0]) { return 7; } - unique_ptr new_parent2 = make_uniq(2, vector>()); - new_parent2->children.emplace_back(make_uniq(8)); + unique_ptr new_parent2 = + make_uniq(TableIndex(2), vector>()); + new_parent2->children.emplace_back(make_uniq(TableIndex(8))); ReplaceNode(root->children[0], root->children[0]->children[0], new_parent2); // Validate structure: there should be two projections between root and the DummyScan @@ -435,17 +441,19 @@ static int Test_ConsecutiveReplaceNode() { // Test 3: inserting parent when child has multiple columns and projection references later column static int Test_MultiColumnChildBinding() { // DummyScan with two columns: logical behavior is simulated via projections using column bindings - auto child = make_uniq(0); + auto child = make_uniq(TableIndex(0)); vector> proj_exprs; // Reference first column from child binding - proj_exprs.emplace_back(make_uniq(LogicalType::INTEGER, ColumnBinding(0, 0))); - unique_ptr root = make_uniq(0, std::move(proj_exprs)); + proj_exprs.emplace_back( + make_uniq(LogicalType::INTEGER, ColumnBinding(TableIndex(0), ProjectionIndex(0)))); + unique_ptr root = make_uniq(TableIndex(0), std::move(proj_exprs)); root->children.emplace_back(std::move(child)); // Create new_parent with a child scan - this matches the pattern used in other tests - unique_ptr new_parent = make_uniq(1, vector>()); - new_parent->children.emplace_back(make_uniq(17)); + unique_ptr new_parent = + make_uniq(TableIndex(1), vector>()); + new_parent->children.emplace_back(make_uniq(TableIndex(17))); ReplaceNode(root, root->children[0], new_parent); @@ -471,15 +479,17 @@ static int Test_MultiColumnChildBinding() { // Test 4: insert a projection between an aggregate and root static int Test_InsertProjectionBetweenAggregateAndRoot() { // Setup: root projection referencing an aggregate child - auto agg_child = make_uniq(0, 0, vector>()); - unique_ptr root = make_uniq(0, vector>()); + auto agg_child = make_uniq(TableIndex(0), TableIndex(0), vector>()); + unique_ptr root = make_uniq(TableIndex(0), vector>()); // projection referencing the aggregate's (would-be) column 0 - root->expressions.emplace_back(make_uniq(LogicalType::INTEGER, ColumnBinding(0, 0))); + root->expressions.emplace_back( + make_uniq(LogicalType::INTEGER, ColumnBinding(TableIndex(0), ProjectionIndex(0)))); root->children.emplace_back(std::move(agg_child)); // Insert a LogicalProjection as new parent above the aggregate - unique_ptr new_proj = make_uniq(12, vector>()); - new_proj->children.emplace_back(make_uniq(13)); + unique_ptr new_proj = + make_uniq(TableIndex(12), vector>()); + new_proj->children.emplace_back(make_uniq(TableIndex(13))); ReplaceNode(root, root->children[0], new_proj); @@ -505,14 +515,15 @@ static int Test_InsertProjectionBetweenAggregateAndRoot() { // Test 6: insert a filter parent above a scan and ensure child moved static int Test_InsertFilterParent() { - auto child = make_uniq(0); + auto child = make_uniq(TableIndex(0)); vector> proj_exprs; - proj_exprs.emplace_back(make_uniq(LogicalType::INTEGER, ColumnBinding(0, 0))); - unique_ptr root = make_uniq(0, std::move(proj_exprs)); + proj_exprs.emplace_back( + make_uniq(LogicalType::INTEGER, ColumnBinding(TableIndex(0), ProjectionIndex(0)))); + unique_ptr root = make_uniq(TableIndex(0), std::move(proj_exprs)); root->children.emplace_back(std::move(child)); unique_ptr new_filter = make_uniq(); - new_filter->children.emplace_back(make_uniq(16)); + new_filter->children.emplace_back(make_uniq(TableIndex(16))); ReplaceNode(root, root->children[0], new_filter); // Verify plan @@ -567,15 +578,18 @@ static int Test_InsertFilterParent() { // Test 7: ensure ReplaceNode preserves column positions when no remapping is required static int Test_PreserveColumnsNoRemap() { - auto child = make_uniq(0); + auto child = make_uniq(TableIndex(0)); vector> proj_exprs; - proj_exprs.emplace_back(make_uniq(LogicalType::INTEGER, ColumnBinding(0, 0))); - proj_exprs.emplace_back(make_uniq(LogicalType::INTEGER, ColumnBinding(0, 1))); - unique_ptr root = make_uniq(0, std::move(proj_exprs)); + proj_exprs.emplace_back( + make_uniq(LogicalType::INTEGER, ColumnBinding(TableIndex(0), ProjectionIndex(0)))); + proj_exprs.emplace_back( + make_uniq(LogicalType::INTEGER, ColumnBinding(TableIndex(0), ProjectionIndex(1)))); + unique_ptr root = make_uniq(TableIndex(0), std::move(proj_exprs)); root->children.emplace_back(std::move(child)); - unique_ptr new_parent = make_uniq(1, vector>()); - new_parent->children.emplace_back(make_uniq(17)); + unique_ptr new_parent = + make_uniq(TableIndex(1), vector>()); + new_parent->children.emplace_back(make_uniq(TableIndex(17))); ReplaceNode(root, root->children[0], new_parent); // Validate structure @@ -597,10 +611,10 @@ static int Test_PreserveColumnsNoRemap() { // Verify column indices are preserved (0 and 1) auto &bcr0 = root->expressions[0]->Cast(); auto &bcr1 = root->expressions[1]->Cast(); - if (bcr0.binding.column_index != 0) { + if (bcr0.Binding().column_index != 0) { return 31; } - if (bcr1.binding.column_index != 1) { + if (bcr1.Binding().column_index != 1) { return 32; } @@ -610,15 +624,16 @@ static int Test_PreserveColumnsNoRemap() { // New test: verifier should detect duplicate producers for the same ColumnBinding static int Test_VerifierDetectsDuplicateProducers() { // Create a join whose left and right are both DummyScans with the same table_index (0) - auto left = make_uniq(0); - auto right = make_uniq(0); + auto left = make_uniq(TableIndex(0)); + auto right = make_uniq(TableIndex(0)); auto base_join = make_uniq(JoinType::INNER); base_join->children.emplace_back(std::move(left)); base_join->children.emplace_back(std::move(right)); - unique_ptr root = make_uniq(0, vector>()); + unique_ptr root = make_uniq(TableIndex(0), vector>()); // Add a projection expression that references the duplicated binding (table=0,col=0) - root->expressions.emplace_back(make_uniq(LogicalType::INTEGER, ColumnBinding(0, 0))); + root->expressions.emplace_back( + make_uniq(LogicalType::INTEGER, ColumnBinding(TableIndex(0), ProjectionIndex(0)))); root->children.emplace_back(std::move(base_join)); int res = RunRobustPlanVerification(root); @@ -678,7 +693,7 @@ int RunCompilerFunctionTests() { // Test 1: Simple scan - check that columns from t1 belong to t1 { - Parser parser; + Parser parser(*con.context); parser.ParseQuery("SELECT a, b FROM t1;"); Planner planner(*con.context); @@ -744,7 +759,7 @@ int RunCompilerFunctionTests() { // Join key columns (t1.a, t2.x) belong to BOTH tables via equivalence. // Non-join-key columns (t1.b, t2.y) belong ONLY to their own table. { - Parser parser; + Parser parser(*con.context); parser.ParseQuery("SELECT t1.a, t1.b, t2.x, t2.y FROM t1 INNER JOIN t2 ON t1.a = t2.x;"); Planner planner(*con.context); planner.CreatePlan(std::move(parser.statements[0])); @@ -841,7 +856,7 @@ int RunCompilerFunctionTests() { // Join conditions: t1.a = t2.x AND t2.x = t3.m → transitive: t1.a = t2.x = t3.m // Join key columns belong to all three tables; non-join-key columns stay isolated. { - Parser parser; + Parser parser(*con.context); parser.ParseQuery("SELECT * FROM t1 INNER JOIN t2 ON t1.a = t2.x INNER JOIN t3 ON t2.x = t3.m;"); Planner planner(*con.context); planner.CreatePlan(std::move(parser.statements[0])); @@ -943,7 +958,7 @@ int RunCompilerFunctionTests() { // Test 4: Non-existent table { - Parser parser; + Parser parser(*con.context); parser.ParseQuery("SELECT a FROM t1;"); Planner planner(*con.context); planner.CreatePlan(std::move(parser.statements[0])); @@ -991,7 +1006,7 @@ int RunCompilerFunctionTests() { // Test 5: Subquery with multiple tables { - Parser parser; + Parser parser(*con.context); parser.ParseQuery("SELECT * FROM (SELECT t1.a, t2.x FROM t1, t2) AS sub;"); Planner planner(*con.context); planner.CreatePlan(std::move(parser.statements[0])); @@ -1070,7 +1085,7 @@ int RunCompilerFunctionTests() { "SELECT t2.x, SUM(t3.n) FROM t1 INNER JOIN t2 ON t1.a = t2.x INNER JOIN t3 ON t2.x = t3.m GROUP BY t2.x;"; // Create plan using parser, planner and optimizer - Parser parser; + Parser parser(*con.context); parser.ParseQuery(query); Planner planner(*con.context); planner.CreatePlan(std::move(parser.statements[0])); @@ -1110,7 +1125,7 @@ int RunCompilerFunctionTests() { } auto &group_expr = agg_node->groups[0]; - if (group_expr->type != ExpressionType::BOUND_COLUMN_REF) { + if (group_expr->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) { std::cerr << "FAIL: ColumnBelongsToTable test 6: group expression is not a column ref" << "\n"; return 1; @@ -1148,8 +1163,8 @@ int RunCompilerFunctionTests() { // The GROUP BY column could belong to either t1 or t2 since we're joining on the key // (t1.a = t2.x), making them equivalent - bool belongs_to_t1 = ColumnBelongsToTable(*plan, "t1", col_ref.binding); - bool belongs_to_t2 = ColumnBelongsToTable(*plan, "t2", col_ref.binding); + bool belongs_to_t1 = ColumnBelongsToTable(*plan, "t1", col_ref.Binding()); + bool belongs_to_t2 = ColumnBelongsToTable(*plan, "t2", col_ref.Binding()); // The column should belong to at least one of the tables (t1 or t2) if (!belongs_to_t1 && !belongs_to_t2) { @@ -1176,7 +1191,7 @@ int RunCompilerFunctionTests() { const string query = "SELECT a, SUM(b) FROM t1 GROUP BY a ORDER BY a;"; // Create plan using parser, planner and optimizer - Parser parser; + Parser parser(*con.context); parser.ParseQuery(query); Planner planner(*con.context); planner.CreatePlan(std::move(parser.statements[0])); @@ -1239,7 +1254,7 @@ int RunCompilerFunctionTests() { } auto &group_expr = agg_node->groups[0]; - if (group_expr->type != ExpressionType::BOUND_COLUMN_REF) { + if (group_expr->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) { std::cerr << "FAIL: ColumnBelongsToTable test 7: group expression is not a column ref" << "\n"; return 1; @@ -1248,7 +1263,7 @@ int RunCompilerFunctionTests() { auto &group_col_ref = group_expr->Cast(); // Verify that the GROUP BY column 'a' belongs to t1 - if (!ColumnBelongsToTable(*plan, "t1", group_col_ref.binding)) { + if (!ColumnBelongsToTable(*plan, "t1", group_col_ref.Binding())) { std::cerr << "FAIL: ColumnBelongsToTable test 7: GROUP BY column 'a' should belong to t1" << "\n"; return 1; @@ -1267,7 +1282,7 @@ int RunCompilerFunctionTests() { ExpressionIterator::VisitExpression( *agg_expr, [&](const BoundColumnRefExpression &col_ref) { // Verify that column 'b' belongs to t1 - if (!ColumnBelongsToTable(*plan, "t1", col_ref.binding)) { + if (!ColumnBelongsToTable(*plan, "t1", col_ref.Binding())) { std::cerr << "FAIL: ColumnBelongsToTable test 7: SUM(b) column 'b' should belong to t1" << "\n"; found_b_column = false; @@ -1321,7 +1336,7 @@ int RunCompilerFunctionTests() { "pu_table.a = non_pu_table.x GROUP BY non_pu_table.y;"; // Create plan using parser, planner and optimizer - Parser parser; + Parser parser(*con.context); parser.ParseQuery(query); Planner planner(*con.context); planner.CreatePlan(std::move(parser.statements[0])); @@ -1361,7 +1376,7 @@ int RunCompilerFunctionTests() { } auto &group_expr = agg_node->groups[0]; - if (group_expr->type != ExpressionType::BOUND_COLUMN_REF) { + if (group_expr->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) { std::cerr << "FAIL: ColumnBelongsToTable test 8: group expression is not a column ref" << "\n"; return 1; @@ -1370,14 +1385,14 @@ int RunCompilerFunctionTests() { auto &col_ref = group_expr->Cast(); // The GROUP BY column should belong to non_pu_table - if (!ColumnBelongsToTable(*plan, "non_pu_table", col_ref.binding)) { + if (!ColumnBelongsToTable(*plan, "non_pu_table", col_ref.Binding())) { std::cerr << "FAIL: ColumnBelongsToTable test 8: GROUP BY column should belong to non_pu_table" << "\n"; return 1; } // Should NOT belong to pu_table - if (ColumnBelongsToTable(*plan, "pu_table", col_ref.binding)) { + if (ColumnBelongsToTable(*plan, "pu_table", col_ref.Binding())) { std::cerr << "FAIL: ColumnBelongsToTable test 8: GROUP BY column should not belong to pu_table" << "\n"; return 1; @@ -1407,7 +1422,7 @@ int RunCompilerFunctionTests() { "pu_table.a = non_pu_table.x GROUP BY non_pu_table.x;"; // Create plan using parser, planner and optimizer - Parser parser; + Parser parser(*con.context); parser.ParseQuery(query); Planner planner(*con.context); planner.CreatePlan(std::move(parser.statements[0])); @@ -1447,7 +1462,7 @@ int RunCompilerFunctionTests() { } auto &group_expr = agg_node->groups[0]; - if (group_expr->type != ExpressionType::BOUND_COLUMN_REF) { + if (group_expr->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) { std::cerr << "FAIL: ColumnBelongsToTable test 9: group expression is not a column ref" << "\n"; return 1; @@ -1456,7 +1471,7 @@ int RunCompilerFunctionTests() { auto &col_ref = group_expr->Cast(); // The GROUP BY column physically comes from non_pu_table - if (!ColumnBelongsToTable(*plan, "non_pu_table", col_ref.binding)) { + if (!ColumnBelongsToTable(*plan, "non_pu_table", col_ref.Binding())) { std::cerr << "FAIL: ColumnBelongsToTable test 9: GROUP BY column should belong to non_pu_table" << "\n"; return 1; @@ -1464,7 +1479,7 @@ int RunCompilerFunctionTests() { // BUT because it's a join key (pu_table.a = non_pu_table.x), it should ALSO be detected // as belonging to pu_table due to join key equivalence - if (!ColumnBelongsToTable(*plan, "pu_table", col_ref.binding)) { + if (!ColumnBelongsToTable(*plan, "pu_table", col_ref.Binding())) { std::cerr << "FAIL: ColumnBelongsToTable test 9: JOIN KEY column should ALSO belong to pu_table " "(due to join key equivalence)" << "\n"; diff --git a/src/test/test_plan_traversal.cpp b/src/test/test_plan_traversal.cpp index 345f4cb3..e19246e8 100644 --- a/src/test/test_plan_traversal.cpp +++ b/src/test/test_plan_traversal.cpp @@ -28,7 +28,7 @@ int RunPlanTraversalTests() { // Test 1: FindPrivacyUnitGetNode - find specific table std::cerr << "\n--- Test FindPrivacyUnitGetNode ---\n"; - Parser parser; + Parser parser(*con.context); parser.ParseQuery("SELECT SUM(val) FROM t_test;"); Planner planner(*con.context); planner.CreatePlan(std::move(parser.statements[0])); @@ -75,7 +75,7 @@ int RunPlanTraversalTests() { // Test 4: FindTopAggregate - no aggregate should throw std::cerr << "\n--- Test FindTopAggregate (no aggregate) ---\n"; - Parser parser2; + Parser parser2(*con.context); parser2.ParseQuery("SELECT val FROM t_test;"); Planner planner2(*con.context); planner2.CreatePlan(std::move(parser2.statements[0])); diff --git a/src/utils/privacy_helpers.cpp b/src/utils/privacy_helpers.cpp index 38310eb0..215170e5 100644 --- a/src/utils/privacy_helpers.cpp +++ b/src/utils/privacy_helpers.cpp @@ -1,3 +1,7 @@ +#include "duckdb/main/database_manager.hpp" +#include "duckdb/catalog/catalog_entry/window_function_catalog_entry.hpp" +#include "duckdb/function/function_binder.hpp" +#include "duckdb/planner/expression/bound_window_expression.hpp" #include "utils/privacy_helpers.hpp" #include @@ -23,6 +27,7 @@ #include "duckdb/planner/operator/logical_distinct.hpp" #include "duckdb/planner/operator/logical_order.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" #include "duckdb/planner/operator/logical_comparison_join.hpp" #include "duckdb/planner/expression_iterator.hpp" #include @@ -39,6 +44,26 @@ using idx_set = std::unordered_set; namespace duckdb { +unique_ptr BindRankingWindow(ClientContext &context, ExpressionType type) { + Identifier name; + switch (type) { + case ExpressionType::WINDOW_ROW_NUMBER: + name = "row_number"; + break; + case ExpressionType::WINDOW_RANK_DENSE: + name = "dense_rank"; + break; + default: + throw InternalException("Unsupported privacy ranking window"); + } + auto &entry = + Catalog::GetSystemCatalog(context).GetEntry(context, DEFAULT_SCHEMA, name); + FunctionBinder binder(context); + vector order_types; + vector arg_order_types; + return binder.BindWindowFunction(entry.functions.GetFunctionByOffset(0), {}, order_types, arg_order_types); +} + string Sanitize(const string &in) { string out; for (char c : in) { @@ -95,7 +120,8 @@ idx_t GetNextTableIndex(unique_ptr &plan) { continue; } auto tbls = cur->GetTableIndex(); - for (auto t : tbls) { + for (auto typed_index : tbls) { + auto t = typed_index.index; if (t != DConstants::INVALID_INDEX && (max_index == DConstants::INVALID_INDEX || t > max_index)) { max_index = t; } @@ -112,7 +138,8 @@ void CollectTableIndicesRecursive(LogicalOperator *node, idx_set &out) { return; } auto tbls = node->GetTableIndex(); - for (auto t : tbls) { + for (auto typed_index : tbls) { + auto t = typed_index.index; if (t != DConstants::INVALID_INDEX) { out.insert(t); } @@ -130,7 +157,8 @@ static void CollectTableIndicesExcluding(LogicalOperator *node, LogicalOperator return; // skip this subtree entirely } auto tbls = node->GetTableIndex(); - for (auto t : tbls) { + for (auto typed_index : tbls) { + auto t = typed_index.index; if (t != DConstants::INVALID_INDEX) { out.insert(t); } @@ -146,73 +174,74 @@ void ApplyIndexMapToSubtree(LogicalOperator *node, const std::unordered_map(node)) { - if (map.find(get_ptr->table_index) != map.end()) { - get_ptr->table_index = map.at(get_ptr->table_index); + if (map.find(get_ptr->table_index.index) != map.end()) { + get_ptr->table_index.index = map.at(get_ptr->table_index.index); } } else if (auto proj_ptr = dynamic_cast(node)) { - if (map.find(proj_ptr->table_index) != map.end()) { - proj_ptr->table_index = map.at(proj_ptr->table_index); + if (map.find(proj_ptr->table_index.index) != map.end()) { + proj_ptr->table_index.index = map.at(proj_ptr->table_index.index); } } else if (auto setop_ptr = dynamic_cast(node)) { - if (map.find(setop_ptr->table_index) != map.end()) { - setop_ptr->table_index = map.at(setop_ptr->table_index); + if (map.find(setop_ptr->table_index.index) != map.end()) { + setop_ptr->table_index.index = map.at(setop_ptr->table_index.index); } } else if (auto insert_ptr = dynamic_cast(node)) { - if (map.find(insert_ptr->table_index) != map.end()) { - insert_ptr->table_index = map.at(insert_ptr->table_index); + if (map.find(insert_ptr->table_index.index) != map.end()) { + insert_ptr->table_index.index = map.at(insert_ptr->table_index.index); } } else if (auto dummy_ptr = dynamic_cast(node)) { - if (map.find(dummy_ptr->table_index) != map.end()) { - dummy_ptr->table_index = map.at(dummy_ptr->table_index); + if (map.find(dummy_ptr->table_index.index) != map.end()) { + dummy_ptr->table_index.index = map.at(dummy_ptr->table_index.index); } } else if (auto coldata_ptr = dynamic_cast(node)) { - if (map.find(coldata_ptr->table_index) != map.end()) { - coldata_ptr->table_index = map.at(coldata_ptr->table_index); + if (map.find(coldata_ptr->table_index.index) != map.end()) { + coldata_ptr->table_index.index = map.at(coldata_ptr->table_index.index); } } else if (auto upd_ptr = dynamic_cast(node)) { - if (map.find(upd_ptr->table_index) != map.end()) { - upd_ptr->table_index = map.at(upd_ptr->table_index); + if (map.find(upd_ptr->table_index.index) != map.end()) { + upd_ptr->table_index.index = map.at(upd_ptr->table_index.index); } } else if (auto del_ptr = dynamic_cast(node)) { - if (map.find(del_ptr->table_index) != map.end()) { - del_ptr->table_index = map.at(del_ptr->table_index); + if (map.find(del_ptr->table_index.index) != map.end()) { + del_ptr->table_index.index = map.at(del_ptr->table_index.index); } } else if (auto cte_ptr = dynamic_cast(node)) { - if (map.find(cte_ptr->table_index) != map.end()) { - cte_ptr->table_index = map.at(cte_ptr->table_index); + if (map.find(cte_ptr->table_index.index) != map.end()) { + cte_ptr->table_index.index = map.at(cte_ptr->table_index.index); } } else if (auto rcte_ptr = dynamic_cast(node)) { - if (map.find(rcte_ptr->table_index) != map.end()) { - rcte_ptr->table_index = map.at(rcte_ptr->table_index); + if (map.find(rcte_ptr->table_index.index) != map.end()) { + rcte_ptr->table_index.index = map.at(rcte_ptr->table_index.index); } } else if (auto eg_ptr = dynamic_cast(node)) { - if (map.find(eg_ptr->table_index) != map.end()) { - eg_ptr->table_index = map.at(eg_ptr->table_index); + if (map.find(eg_ptr->table_index.index) != map.end()) { + eg_ptr->table_index.index = map.at(eg_ptr->table_index.index); } } else if (auto cteref_ptr = dynamic_cast(node)) { - if (map.find(cteref_ptr->table_index) != map.end()) { - cteref_ptr->table_index = map.at(cteref_ptr->table_index); + if (map.find(cteref_ptr->table_index.index) != map.end()) { + cteref_ptr->table_index.index = map.at(cteref_ptr->table_index.index); } } else if (auto dg_ptr = dynamic_cast(node)) { - if (map.find(dg_ptr->table_index) != map.end()) { - dg_ptr->table_index = map.at(dg_ptr->table_index); + if (map.find(dg_ptr->table_index.index) != map.end()) { + dg_ptr->table_index.index = map.at(dg_ptr->table_index.index); } } else if (auto unnest_ptr = dynamic_cast(node)) { - if (map.find(unnest_ptr->unnest_index) != map.end()) { - unnest_ptr->unnest_index = map.at(unnest_ptr->unnest_index); + if (map.find(unnest_ptr->unnest_index.index) != map.end()) { + unnest_ptr->unnest_index.index = map.at(unnest_ptr->unnest_index.index); } } // Special handling for LogicalAggregate: multiple indices if (auto agg_ptr = dynamic_cast(node)) { - if (map.find(agg_ptr->aggregate_index) != map.end()) { - agg_ptr->aggregate_index = map.at(agg_ptr->aggregate_index); + if (map.find(agg_ptr->aggregate_index.index) != map.end()) { + agg_ptr->aggregate_index.index = map.at(agg_ptr->aggregate_index.index); } - if (map.find(agg_ptr->group_index) != map.end()) { - agg_ptr->group_index = map.at(agg_ptr->group_index); + if (map.find(agg_ptr->group_index.index) != map.end()) { + agg_ptr->group_index.index = map.at(agg_ptr->group_index.index); } - if (agg_ptr->groupings_index != DConstants::INVALID_INDEX && map.find(agg_ptr->groupings_index) != map.end()) { - agg_ptr->groupings_index = map.at(agg_ptr->groupings_index); + if (agg_ptr->groupings_index.index != DConstants::INVALID_INDEX && + map.find(agg_ptr->groupings_index.index) != map.end()) { + agg_ptr->groupings_index.index = map.at(agg_ptr->groupings_index.index); } } @@ -263,7 +292,7 @@ void ReplaceNode(unique_ptr &root, unique_ptr // Build index remapping for any new-subtree indices that collide with external indices std::unordered_map index_map; if (!new_subtree_indices.empty()) { - idx_t next_idx = binder ? binder->GenerateTableIndex() : GetNextTableIndex(root); + idx_t next_idx = binder ? binder->GenerateTableIndex().index : GetNextTableIndex(root); for (auto idx : new_subtree_indices) { if (idx == DConstants::INVALID_INDEX) { continue; @@ -272,11 +301,11 @@ void ReplaceNode(unique_ptr &root, unique_ptr // find a fresh index not in external_indices and not in new_subtree_indices while (external_indices.find(next_idx) != external_indices.end() || new_subtree_indices.find(next_idx) != new_subtree_indices.end()) { - next_idx = binder ? binder->GenerateTableIndex() : next_idx + 1; + next_idx = binder ? binder->GenerateTableIndex().index : next_idx + 1; } index_map[idx] = next_idx; external_indices.insert(next_idx); - next_idx = binder ? binder->GenerateTableIndex() : next_idx + 1; + next_idx = binder ? binder->GenerateTableIndex().index : next_idx + 1; } } } @@ -293,9 +322,9 @@ void ReplaceNode(unique_ptr &root, unique_ptr ColumnBindingReplacer replacer; if (!index_map.empty()) { for (auto &old_b : old_top_bindings) { - auto it = index_map.find(old_b.table_index); + auto it = index_map.find(old_b.table_index.index); if (it != index_map.end()) { - ColumnBinding new_b(it->second, old_b.column_index); + ColumnBinding new_b {TableIndex(it->second), ProjectionIndex(old_b.column_index)}; replacer.replacement_bindings.emplace_back(old_b, new_b); } } @@ -320,14 +349,14 @@ vector FindPacKey(ClientContext &context, const string &table_name) { vector out; out.reserve(cols.size()); for (auto &col_name : cols) { - string tmp = col_name; + Identifier tmp(col_name); try { auto col_idx = table_entry.GetColumnIndex(tmp); auto &col = table_entry.GetColumn(col_idx); if (!col.Type().IsNumeric()) { return {}; } - out.push_back(col.GetName()); + out.push_back(col.GetName().GetIdentifierName()); } catch (...) { return {}; } @@ -351,16 +380,16 @@ vector FindPacKey(ClientContext &context, const string &table_name) { // Validate columns exist and are numeric by looking up the table in the catalog if (dot_pos != string::npos) { string schema = table_name.substr(0, dot_pos); - auto entry = - catalog.GetEntry(context, CatalogType::TABLE_ENTRY, schema, unqualified, OnEntryNotFound::RETURN_NULL); + auto entry = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, Identifier(schema), Identifier(unqualified), + OnEntryNotFound::RETURN_NULL); if (entry) { return check_and_return_numeric_columns(entry->Cast(), pac_meta->primary_key_columns); } } else { CatalogSearchPath path(context); for (auto &entry_path : path.Get()) { - auto entry = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, entry_path.schema, table_name, - OnEntryNotFound::RETURN_NULL); + auto entry = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, entry_path.GetSchema(), + Identifier(table_name), OnEntryNotFound::RETURN_NULL); if (entry) { return check_and_return_numeric_columns(entry->Cast(), pac_meta->primary_key_columns); @@ -437,7 +466,8 @@ std::unordered_map> FindPacLinkPath(ClientContext &contex if (dot_pos != string::npos) { string schema = tbl_name.substr(0, dot_pos); string tbl = tbl_name.substr(dot_pos + 1); - auto entry = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, schema, tbl, OnEntryNotFound::RETURN_NULL); + auto entry = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, Identifier(schema), Identifier(tbl), + OnEntryNotFound::RETURN_NULL); if (entry) { return StringUtil::Lower(tbl); // return unqualified table name, normalized to lowercase } @@ -446,8 +476,8 @@ std::unordered_map> FindPacLinkPath(ClientContext &contex // Non-qualified: walk the search path; if found return unqualified table name CatalogSearchPath path(context); for (auto &entry_path : path.Get()) { - auto entry = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, entry_path.schema, tbl_name, - OnEntryNotFound::RETURN_NULL); + auto entry = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, entry_path.GetSchema(), + Identifier(tbl_name), OnEntryNotFound::RETURN_NULL); if (entry) { return StringUtil::Lower(tbl_name); // already unqualified, normalized to lowercase } @@ -538,7 +568,7 @@ ReplanGuard::~ReplanGuard() { // Configuration helpers string GetPacCompiledPath(ClientContext &context, const string &default_path) { Value v; - context.TryGetCurrentSetting("priv_compiled_path", v); + context.TryGetCurrentSetting(Identifier("priv_compiled_path"), v); string path = v.IsNull() ? default_path : v.ToString(); if (!path.empty() && path.back() != '/') { path.push_back('/'); @@ -548,7 +578,7 @@ string GetPacCompiledPath(ClientContext &context, const string &default_path) { int64_t GetPacM(ClientContext &context, int64_t default_m) { Value v; - context.TryGetCurrentSetting("pac_m", v); + context.TryGetCurrentSetting(Identifier("pac_m"), v); if (!v.IsNull()) { try { int64_t m = v.GetValue(); @@ -563,7 +593,7 @@ int64_t GetPacM(ClientContext &context, int64_t default_m) { bool IsPacNoiseEnabled(ClientContext &context, bool default_value) { Value v; - context.TryGetCurrentSetting("privacy_noise", v); + context.TryGetCurrentSetting(Identifier("privacy_noise"), v); if (v.IsNull()) { return default_value; } @@ -577,7 +607,7 @@ bool IsPacNoiseEnabled(ClientContext &context, bool default_value) { static string GetNormalizedStringSetting(ClientContext &context, const string &setting_name, const string &default_value) { Value v; - if (!context.TryGetCurrentSetting(setting_name, v) || v.IsNull()) { + if (!context.TryGetCurrentSetting(Identifier(setting_name), v) || v.IsNull()) { return default_value; } string s = v.ToString(); @@ -596,7 +626,7 @@ string GetPrivacyMode(ClientContext &context) { static bool TryGetDoubleSetting(ClientContext &context, const char *setting_name, double &out) { Value v; - if (!context.TryGetCurrentSetting(setting_name, v) || v.IsNull()) { + if (!context.TryGetCurrentSetting(Identifier(setting_name), v) || v.IsNull()) { return false; } try { @@ -652,7 +682,7 @@ bool TryGetDpDelta(ClientContext &context, double &out) { // Add implementation for GetPacCompileMethod string GetPacCompileMethod(ClientContext &context, const string &default_method) { Value v; - context.TryGetCurrentSetting("pac_compile_method", v); + context.TryGetCurrentSetting(Identifier("pac_compile_method"), v); if (v.IsNull()) { return default_method; } @@ -672,7 +702,7 @@ string GetPacCompileMethod(ClientContext &context, const string &default_method) // Helper to safely retrieve boolean settings with defaults bool GetBooleanSetting(ClientContext &context, const string &setting_name, bool default_value) { Value val; - if (context.TryGetCurrentSetting(setting_name, val) && !val.IsNull()) { + if (context.TryGetCurrentSetting(Identifier(setting_name), val) && !val.IsNull()) { return val.GetValue(); } return default_value; @@ -683,7 +713,7 @@ static void CollectOperatorsByTableIndex(LogicalOperator &op, std::unordered_map &table_index_to_op) { auto bindings = op.GetColumnBindings(); if (!bindings.empty()) { - idx_t op_table_index = bindings[0].table_index; + idx_t op_table_index = bindings[0].table_index.index; table_index_to_op[op_table_index] = &op; } for (auto &child : op.children) { @@ -693,6 +723,26 @@ static void CollectOperatorsByTableIndex(LogicalOperator &op, } } +// Casts and the optimizer's reversible storage encodings retain column lineage. +static optional_ptr FindLineageColumn(const Expression &expression) { + auto *expr = &expression; + while (expr->GetExpressionClass() == ExpressionClass::BOUND_FUNCTION) { + auto &function = expr->Cast(); + auto name = function.Function().GetName().GetIdentifierName(); + if (expr->GetExpressionType() != ExpressionType::OPERATOR_CAST && + !StringUtil::StartsWith(name, "__internal_compress_") && + !StringUtil::StartsWith(name, "__internal_decompress_")) { + break; + } + if (function.GetChildren().empty()) { + return nullptr; + } + expr = function.GetChildren()[0].get(); + } + return expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF ? &expr->Cast() + : nullptr; +} + // Helper to trace a binding back through the plan to find which LogicalGet it originates from LogicalGet *TraceBindingToSource(LogicalOperator &plan, const ColumnBinding &binding, idx_t *out_column_index) { // Build a map from table_index to the operator that produces it @@ -700,7 +750,7 @@ LogicalGet *TraceBindingToSource(LogicalOperator &plan, const ColumnBinding &bin CollectOperatorsByTableIndex(plan, table_index_to_op); // Start from the binding's table_index and trace backwards - idx_t current_table_index = binding.table_index; + idx_t current_table_index = binding.table_index.index; idx_t current_column_index = binding.column_index; // Limit iterations to prevent infinite loops @@ -737,10 +787,10 @@ LogicalGet *TraceBindingToSource(LogicalOperator &plan, const ColumnBinding &bin auto &expr = proj.expressions[current_column_index]; // If it's a BoundColumnRefExpression, follow it - if (expr->type == ExpressionType::BOUND_COLUMN_REF) { - auto &col_ref = expr->Cast(); - current_table_index = col_ref.binding.table_index; - current_column_index = col_ref.binding.column_index; + auto col_ref = FindLineageColumn(*expr); + if (col_ref) { + current_table_index = col_ref->Binding().table_index.index; + current_column_index = col_ref->Binding().column_index; continue; } @@ -757,10 +807,10 @@ LogicalGet *TraceBindingToSource(LogicalOperator &plan, const ColumnBinding &bin // This is a group column, trace through the group expression auto &group_expr = agg.groups[current_column_index]; - if (group_expr->type == ExpressionType::BOUND_COLUMN_REF) { - auto &col_ref = group_expr->Cast(); - current_table_index = col_ref.binding.table_index; - current_column_index = col_ref.binding.column_index; + auto col_ref = FindLineageColumn(*group_expr); + if (col_ref) { + current_table_index = col_ref->Binding().table_index.index; + current_column_index = col_ref->Binding().column_index; continue; } return nullptr; @@ -785,7 +835,7 @@ LogicalGet *TraceBindingToSource(LogicalOperator &plan, const ColumnBinding &bin if (current_column_index < left_count && op->children[0]) { auto left_bindings = op->children[0]->GetColumnBindings(); if (current_column_index < left_bindings.size()) { - current_table_index = left_bindings[current_column_index].table_index; + current_table_index = left_bindings[current_column_index].table_index.index; current_column_index = left_bindings[current_column_index].column_index; continue; } @@ -796,7 +846,7 @@ LogicalGet *TraceBindingToSource(LogicalOperator &plan, const ColumnBinding &bin idx_t right_col_idx = current_column_index - left_count; auto right_bindings = op->children[1]->GetColumnBindings(); if (right_col_idx < right_bindings.size()) { - current_table_index = right_bindings[right_col_idx].table_index; + current_table_index = right_bindings[right_col_idx].table_index.index; current_column_index = right_bindings[right_col_idx].column_index; continue; } @@ -809,7 +859,7 @@ LogicalGet *TraceBindingToSource(LogicalOperator &plan, const ColumnBinding &bin if (!op->children.empty() && op->children[0]) { auto child_bindings = op->children[0]->GetColumnBindings(); if (current_column_index < child_bindings.size()) { - current_table_index = child_bindings[current_column_index].table_index; + current_table_index = child_bindings[current_column_index].table_index.index; current_column_index = child_bindings[current_column_index].column_index; } else { return nullptr; @@ -832,16 +882,29 @@ static void CollectJoinKeyEquivalences(LogicalOperator &op, for (auto &cond : join.conditions) { // Only consider equality conditions - if (cond.comparison != ExpressionType::COMPARE_EQUAL) { + if (cond.GetComparisonType() != ExpressionType::COMPARE_EQUAL) { continue; } // Check if both sides are column references - if (cond.left && cond.right && cond.left->type == ExpressionType::BOUND_COLUMN_REF && - cond.right->type == ExpressionType::BOUND_COLUMN_REF) { - auto &left_ref = cond.left->Cast(); - auto &right_ref = cond.right->Cast(); - equivalences.emplace_back(left_ref.binding, right_ref.binding); + if (cond.LeftReference() && cond.RightReference() && + cond.LeftReference()->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF && + cond.RightReference()->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { + auto &left_ref = cond.LeftReference()->Cast(); + auto &right_ref = cond.RightReference()->Cast(); + equivalences.emplace_back(left_ref.BindingMutable(), right_ref.BindingMutable()); + } + } + } + + // Optimizer-inserted projections can rename a join key or encode it for + // grouping. Include these aliases in the same equivalence graph. + if (op.type == LogicalOperatorType::LOGICAL_PROJECTION) { + auto &projection = op.Cast(); + for (idx_t i = 0; i < projection.expressions.size(); i++) { + auto ref = FindLineageColumn(*projection.expressions[i]); + if (ref) { + equivalences.emplace_back(ColumnBinding(projection.table_index, ProjectionIndex(i)), ref->Binding()); } } } @@ -859,7 +922,7 @@ static void FindEquivalentBindings(const ColumnBinding &binding, const vector> &equivalences, std::unordered_set &visited, vector &result) { // Create a unique key for the binding - uint64_t key = (static_cast(binding.table_index) << 32) | binding.column_index; + uint64_t key = (static_cast(binding.table_index.index) << 32) | binding.column_index; if (visited.find(key) != visited.end()) { return; } @@ -868,10 +931,11 @@ static void FindEquivalentBindings(const ColumnBinding &binding, // Find all equivalences involving this binding for (auto &eq : equivalences) { - if (eq.first.table_index == binding.table_index && eq.first.column_index == binding.column_index) { + if (eq.first.table_index.index == binding.table_index.index && eq.first.column_index == binding.column_index) { FindEquivalentBindings(eq.second, equivalences, visited, result); } - if (eq.second.table_index == binding.table_index && eq.second.column_index == binding.column_index) { + if (eq.second.table_index.index == binding.table_index.index && + eq.second.column_index == binding.column_index) { FindEquivalentBindings(eq.first, equivalences, visited, result); } } @@ -899,7 +963,7 @@ bool ColumnBelongsToTable(LogicalOperator &plan, const string &table_name, const continue; } - if (table_entry->name == table_name) { + if (table_entry->name.GetIdentifierName() == table_name) { return true; } } @@ -909,11 +973,11 @@ bool ColumnBelongsToTable(LogicalOperator &plan, const string &table_name, const // Recursively remap BoundColumnRefExpression bindings in an expression tree. static void RemapBindingsInExpr(Expression &e, const std::unordered_map &map) { - if (e.type == ExpressionType::BOUND_COLUMN_REF) { + if (e.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { auto &col_ref = e.Cast(); - auto it = map.find(col_ref.binding.table_index); + auto it = map.find(col_ref.BindingMutable().table_index.index); if (it != map.end()) { - col_ref.binding.table_index = it->second; + col_ref.BindingMutable().table_index.index = it->second; } } ExpressionIterator::EnumerateChildren(e, [&map](Expression &child) { RemapBindingsInExpr(child, map); }); @@ -947,8 +1011,8 @@ void RemapBindingsInSubtree(LogicalOperator &op, const std::unordered_map(&op)) { for (auto &cond : cj->conditions) { - remap_expr(cond.left); - remap_expr(cond.right); + remap_expr(cond.LeftReference()); + remap_expr(cond.RightReference()); } for (auto &expr : cj->duplicate_eliminated_columns) { remap_expr(expr); @@ -996,9 +1060,9 @@ std::unordered_map RemapSubtreeIndices(LogicalOperator *subtree, B if (old_idx == DConstants::INVALID_INDEX) { continue; } - idx_t fresh = binder.GenerateTableIndex(); + idx_t fresh = binder.GenerateTableIndex().index; while (used.find(fresh) != used.end()) { - fresh = binder.GenerateTableIndex(); + fresh = binder.GenerateTableIndex().index; } map[old_idx] = fresh; used.insert(fresh); diff --git a/test/sql/dp_filterless.test b/test/sql/dp_filterless.test index a8e9ce3c..25d53b7b 100644 --- a/test/sql/dp_filterless.test +++ b/test/sql/dp_filterless.test @@ -367,7 +367,7 @@ EXPLAIN SELECT grp, SUM(amount) FROM filterless_group_cap_three GROUP BY grp; ---- -physical_plan :[\s\S]*UNNEST[\s\S]*arg_min_nulls_last[\s\S]* +physical_plan :[\s\S]*Unnest[\s\S]*arg_min_nulls_last[\s\S]* statement ok SET dp_max_groups_contributed = 1; @@ -541,7 +541,7 @@ EXPLAIN SELECT AVG(amount) FROM filterless_sampled_events WHERE qualifies; ---- -physical_plan :[\s\S]*UNGROUPED_AGGREGATE[\s\S]*priv_filterless_avg[\s\S]*priv_filterless_sum_pair[\s\S]*priv_filterless_count_pair[\s\S]* +physical_plan :[\s\S]*Ungrouped Aggregate[\s\S]*priv_filterless_avg[\s\S]*priv_filterless_sum_pair[\s\S]*priv_filterless_count_pair[\s\S]* # COUNT(expr) is zero when every qualifying value is NULL, so fused AVG keeps # SQL's NULL result instead of dividing by a synthetic denominator. diff --git a/test/sql/dp_sass_m.test b/test/sql/dp_sass_m.test index ba403015..e7b20a7a 100644 --- a/test/sql/dp_sass_m.test +++ b/test/sql/dp_sass_m.test @@ -866,7 +866,7 @@ EXPLAIN SELECT category AS g1, length(category) AS g2, COUNT(DISTINCT v) FROM sass_m_events GROUP BY g1, g2; ---- -logical_opt :[\s\S]*RANK_DENSE[\s\S]*ORDER BY[\s\S]*#0 ASC[\s\S]*#1 ASC[\s\S]* +logical_opt :[\s\S]*dense_rank[\s\S]*ORDER BY[\s\S]*#0 ASC[\s\S]*#1 ASC[\s\S]* # Noisy aggregate cells must use independent random streams even when their lane # lists and public domains are identical. Cover both the optimized m=64 path and diff --git a/test/sql/dp_standard.test b/test/sql/dp_standard.test index 22f9c780..16a1eaeb 100644 --- a/test/sql/dp_standard.test +++ b/test/sql/dp_standard.test @@ -502,7 +502,7 @@ WHERE EXISTS ( AND o2.store <> o1.store ); ---- -dp_standard: lateral or correlated subqueries are not supported by DP mechanisms +dp_standard: scalar or mark subqueries are not supported by DP mechanisms # --- MIN/MAX (dp_standard) --- # Values are clipped to the public domain [dp_minmax_lower_bound, dp_minmax_upper_bound], so global diff --git a/test/sql/pac_bitslice_compiler_uncorrelated_subqueries.test b/test/sql/pac_bitslice_compiler_uncorrelated_subqueries.test index edfb34d7..b61f4b6f 100644 --- a/test/sql/pac_bitslice_compiler_uncorrelated_subqueries.test +++ b/test/sql/pac_bitslice_compiler_uncorrelated_subqueries.test @@ -358,7 +358,9 @@ ORDER BY # Test 4: TPC-H Q9-style query with complex arithmetic in subquery # Tests: Subquery with multiple joins, complex arithmetic expression (subtraction of products), GROUP BY multiple columns -# This query computes profit by nation and year +# This query computes profit by nation and year. DuckDB 2.0 widens the arithmetic +# to DECIMAL(21,4), selecting the INT128/double SUM state rather than the +# approximate INT64 state used for DuckDB 1.5's DECIMAL(18,4). query III SELECT nation, @@ -389,10 +391,10 @@ ORDER BY nation, o_year DESC; ---- -FRANCE 1996 -619.3152 +FRANCE 1996 -620.0000 FRANCE 1995 0.0000 GERMANY 1996 0.0000 -GERMANY 1995 NULL +GERMANY 1995 -420.0000 # Clean up statement ok diff --git a/test/sql/pac_categorical_subqueries.test b/test/sql/pac_categorical_subqueries.test index 45e15655..da665632 100644 --- a/test/sql/pac_categorical_subqueries.test +++ b/test/sql/pac_categorical_subqueries.test @@ -524,12 +524,49 @@ SELECT count(*) FROM ( ) ); ---- -2 +1 + +# Independently evaluate all 64 sampled worlds with ordinary SQL. Only supplier +# 1 passes in at least half of the worlds. DuckDB 2.0's nullable-COUNT wrapper +# contains two COUNT states; both must remain world-specific through its CASE. +statement ok +SET priv_check = false; + +statement ok +SET priv_rewrite = false; + +query I +WITH sampled AS ( + SELECT l_suppkey, j AS world + FROM lineitem JOIN orders ON l_orderkey = o_orderkey + CROSS JOIN range(64) t(j) + WHERE ((priv_hash(hash(o_custkey)) >> j) & 1::UBIGINT) <> 0 +), counts AS ( + SELECT world, l_suppkey, COUNT(*) AS n FROM sampled GROUP BY ALL +), means AS ( + SELECT world, AVG(n) AS mean_count FROM counts GROUP BY world +), decisions AS ( + SELECT s_suppkey, SUM((COALESCE(n, 0) > mean_count)::INTEGER) AS true_worlds + FROM supplier CROSS JOIN range(64) t(j) + LEFT JOIN counts ON j = counts.world AND s_suppkey = l_suppkey + LEFT JOIN means ON j = means.world + GROUP BY s_suppkey +) +SELECT COUNT(*) FROM decisions WHERE true_worlds >= 32; +---- +1 + +statement ok +SET priv_rewrite = true; + +statement ok +SET priv_check = true; # ============================================================================ # Test 11: Categorical with AVG aggregate # Find parts where avg quantity exceeds threshold -# NOTE: Result is 2 with proper avg decomposition (as_noised_avg → sum/count) +# All four parts pass in a majority of worlds; both correlated and global +# averages must remain world-specific through the comparison. # ============================================================================ query I @@ -551,7 +588,42 @@ SELECT count(*) FROM ( ) ); ---- -2 +4 + +# Ordinary SQL oracle: report the number of passing worlds for each part. +statement ok +SET priv_check = false; + +statement ok +SET priv_rewrite = false; + +query II +WITH sampled AS ( + SELECT l_partkey, l_quantity, j AS world + FROM lineitem JOIN orders ON l_orderkey = o_orderkey + CROSS JOIN range(64) t(j) + WHERE ((priv_hash(hash(o_custkey)) >> j) & 1::UBIGINT) <> 0 +), avgs AS ( + SELECT world, l_partkey, AVG(l_quantity) AS a FROM sampled GROUP BY ALL +), means AS ( + SELECT world, AVG(l_quantity) AS m FROM sampled GROUP BY world +) +SELECT p_partkey, SUM((a > 0.8 * m)::INTEGER) AS true_worlds +FROM part CROSS JOIN range(64) t(j) +LEFT JOIN avgs ON j = avgs.world AND p_partkey = l_partkey +LEFT JOIN means ON j = means.world +GROUP BY p_partkey ORDER BY p_partkey; +---- +1 41 +2 37 +3 50 +4 50 + +statement ok +SET priv_rewrite = true; + +statement ok +SET priv_check = true; # ============================================================================ # Test 12: Correlated categorical subquery with multiple tables diff --git a/test/sql/pac_noise_independence.test b/test/sql/pac_noise_independence.test index a097cf6c..2e845584 100644 --- a/test/sql/pac_noise_independence.test +++ b/test/sql/pac_noise_independence.test @@ -106,7 +106,7 @@ query III SELECT count(DISTINCT l) > 3000, count(DISTINCT f) > 3000, count(DISTINCT d) > 3000 FROM (SELECT as_noised(range(64)::FLOAT[]) AS l, priv_finalize(range(64)::FLOAT[]) AS f, - as_noised_div(range(64)::FLOAT[], list_transform(range(64), x -> 1::FLOAT)) AS d + as_noised_div(range(64)::FLOAT[], list_transform(range(64), lambda x: 1::FLOAT)) AS d FROM range(4096)) t ---- true true true @@ -151,7 +151,7 @@ CREATE TABLE seeded_prepared(i BIGINT, l FLOAT, f FLOAT, d DOUBLE) statement ok PREPARE seeded_release AS INSERT INTO seeded_prepared SELECT i, as_noised(range(64)::FLOAT[]), priv_finalize(range(64)::FLOAT[]), - as_noised_div(range(64)::FLOAT[], list_transform(range(64), x -> 1::FLOAT)) + as_noised_div(range(64)::FLOAT[], list_transform(range(64), lambda x: 1::FLOAT)) FROM range(4096) t(i) statement ok diff --git a/test/sql/pac_plan.test b/test/sql/pac_plan.test index 3f5f4c41..ebbbfcc5 100644 --- a/test/sql/pac_plan.test +++ b/test/sql/pac_plan.test @@ -798,7 +798,7 @@ WHERE GROUP BY c_nationkey; ---- -logical_opt :.*COMPARISON_JOIN.*bit_or.*bit_or.* +logical_opt :.*Comparison Join.*bit_or.*bit_or.* # ============================================================ From b3a55c3721251cf051046190ba0f0e81809a1ebb Mon Sep 17 00:00:00 2001 From: ila Date: Sun, 1 Nov 2026 10:41:21 +0100 Subject: [PATCH 05/32] Fix Windows linkage and migrate deprecated DuckDB 2.0 APIs --- CMakeLists.txt | 4 +- src/aggregates/as_aggregate.cpp | 16 +-- src/aggregates/as_clip_min_max.cpp | 24 ++-- src/aggregates/as_clip_sum.cpp | 44 +++--- src/aggregates/as_count.cpp | 120 ++++++++-------- src/aggregates/as_finalize.cpp | 14 +- src/aggregates/as_min_max.cpp | 20 +-- src/aggregates/as_sum.cpp | 128 +++++++++--------- src/aggregates/dp_approx_bounds.cpp | 16 +-- src/aggregates/dp_laplace_noise.cpp | 110 +++++++-------- src/aggregates/filterless_aggregate.cpp | 34 ++--- src/categorical/pac_categorical.cpp | 90 ++++++------ src/categorical/pac_categorical_lambdas.cpp | 5 +- src/compiler/privacy_mechanisms.cpp | 4 +- .../categorical/pac_categorical_detection.hpp | 4 +- src/parser/privacy_parser.cpp | 2 +- src/query_processing/pac_topk_rewriter.cpp | 6 +- .../privacy_expression_builder.cpp | 15 +- .../dp_sass_stability_query.cpp | 9 +- src/utils/privacy_helpers.cpp | 4 +- 20 files changed, 339 insertions(+), 330 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1cc4b438..fb29f785 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,8 +71,8 @@ build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) if(WIN32) - target_link_libraries(${EXTENSION_NAME} PRIVATE bcrypt) - target_link_libraries(${LOADABLE_EXTENSION_NAME} PRIVATE bcrypt) + target_link_libraries(${EXTENSION_NAME} bcrypt) + target_link_libraries(${LOADABLE_EXTENSION_NAME} bcrypt) endif() install( diff --git a/src/aggregates/as_aggregate.cpp b/src/aggregates/as_aggregate.cpp index 23aa01cb..123bc969 100644 --- a/src/aggregates/as_aggregate.cpp +++ b/src/aggregates/as_aggregate.cpp @@ -377,7 +377,7 @@ uint64_t PacRepairHash32(uint64_t num) { // k : minimum per-subsample support; cells below it are released as NULL // ============================================================================ struct PacAggregateLocalState : public FunctionLocalState { - explicit PacAggregateLocalState(PacRandomEngine gen_p) : gen(std::move(gen_p)) { + explicit PacAggregateLocalState(PacRandomEngine gen_p) : gen(gen_p) { } PacRandomEngine gen; }; @@ -420,11 +420,11 @@ static void PacAggregateScalar(DataChunk &args, ExpressionState &state, Vector & } UnifiedVectorFormat vvals, vcnts; - vals.ToUnifiedFormat(count, vvals); - cnts.ToUnifiedFormat(count, vcnts); + vals.ToUnifiedFormat(vvals); + cnts.ToUnifiedFormat(vcnts); UnifiedVectorFormat kvals; - k_vec.ToUnifiedFormat(count, kvals); + k_vec.ToUnifiedFormat(kvals); auto *kdata = UnifiedVectorFormat::GetData(kvals); for (idx_t row = 0; row < count; row++) { @@ -453,10 +453,10 @@ static void PacAggregateScalar(DataChunk &args, ExpressionState &state, Vector & idx_t vals_len = ve.length; idx_t cnts_len = ce.length; - auto &vals_child = ListVector::GetEntry(vals); - auto &cnts_child = ListVector::GetEntry(cnts); - vals_child.Flatten(ve.offset + ve.length); - cnts_child.Flatten(ce.offset + ce.length); + auto &vals_child = ListVector::GetChild(vals); + auto &cnts_child = ListVector::GetChild(cnts); + vals_child.Flatten(); + cnts_child.Flatten(); auto *vdata = FlatVector::GetData(vals_child); auto *cdata = FlatVector::GetData(cnts_child); diff --git a/src/aggregates/as_clip_min_max.cpp b/src/aggregates/as_clip_min_max.cpp index 5154effb..5f529993 100644 --- a/src/aggregates/as_clip_min_max.cpp +++ b/src/aggregates/as_clip_min_max.cpp @@ -115,12 +115,12 @@ template static void PacClipMinMaxUpdate(Vector inputs[], PacClipMinMaxStateWrapper &state, idx_t count, ArenaAllocator &allocator, int sample_lanes = 0) { UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); - if (hash_data.validity.AllValid() && value_data.validity.AllValid()) { + if (hash_data.validity.CannotHaveNull() && value_data.validity.CannotHaveNull()) { for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); auto v_idx = value_data.sel->get_index(i); @@ -144,9 +144,9 @@ template static void PacClipMinMaxScatterUpdate(Vector inputs[], Vector &states, idx_t count, ArenaAllocator &allocator, int sample_lanes = 0) { UnifiedVectorFormat hash_data, value_data, sdata; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - states.ToUnifiedFormat(count, sdata); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + states.ToUnifiedFormat(sdata); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); @@ -224,12 +224,12 @@ template static void PacClipMinMaxUpdateFloat(Vector inputs[], PacClipMinMaxStateWrapper &state, idx_t count, ArenaAllocator &allocator, int sample_lanes = 0) { UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); - if (hash_data.validity.AllValid() && value_data.validity.AllValid()) { + if (hash_data.validity.CannotHaveNull() && value_data.validity.CannotHaveNull()) { for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); auto v_idx = value_data.sel->get_index(i); @@ -253,9 +253,9 @@ template static void PacClipMinMaxScatterUpdateFloat(Vector inputs[], Vector &states, idx_t count, ArenaAllocator &allocator, int sample_lanes = 0) { UnifiedVectorFormat hash_data, value_data, sdata; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - states.ToUnifiedFormat(count, sdata); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + states.ToUnifiedFormat(sdata); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); auto state_ptrs = UnifiedVectorFormat::GetData *>(sdata); diff --git a/src/aggregates/as_clip_sum.cpp b/src/aggregates/as_clip_sum.cpp index 3ea6a073..6a4cc29c 100644 --- a/src/aggregates/as_clip_sum.cpp +++ b/src/aggregates/as_clip_sum.cpp @@ -201,12 +201,12 @@ template &state, idx_t count, ArenaAllocator &allocator, int sample_lanes = 0) { UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); - if (hash_data.validity.AllValid() && value_data.validity.AllValid()) { + if (hash_data.validity.CannotHaveNull() && value_data.validity.CannotHaveNull()) { for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); auto v_idx = value_data.sel->get_index(i); @@ -231,9 +231,9 @@ template (hash_data); auto values = UnifiedVectorFormat::GetData(value_data); @@ -291,8 +291,8 @@ static void PacClipSumUpdateHugeInt(Vector inputs[], AggregateInputData &aggr, i auto &state = *reinterpret_cast *>(state_p); int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); for (idx_t i = 0; i < count; i++) { @@ -310,9 +310,9 @@ static void PacClipSumScatterUpdateHugeInt(Vector inputs[], AggregateInputData & idx_t count) { int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, value_data, sdata; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - states.ToUnifiedFormat(count, sdata); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + states.ToUnifiedFormat(sdata); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); auto state_ptrs = UnifiedVectorFormat::GetData *>(sdata); @@ -334,8 +334,8 @@ static void PacClipSumUpdateUHugeInt(Vector inputs[], AggregateInputData &aggr, auto &state = *reinterpret_cast *>(state_p); int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); for (idx_t i = 0; i < count; i++) { @@ -377,9 +377,9 @@ static void PacClipSumScatterUpdateUHugeInt(Vector inputs[], AggregateInputData idx_t count) { int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, value_data, sdata; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - states.ToUnifiedFormat(count, sdata); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + states.ToUnifiedFormat(sdata); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); auto state_ptrs = UnifiedVectorFormat::GetData *>(sdata); @@ -428,12 +428,12 @@ static void PacClipSumUpdateFloat(Vector inputs[], AggregateInputData &aggr, idx auto &state = *reinterpret_cast *>(state_p); int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); - if (hash_data.validity.AllValid() && value_data.validity.AllValid()) { + if (hash_data.validity.CannotHaveNull() && value_data.validity.CannotHaveNull()) { for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); auto v_idx = value_data.sel->get_index(i); @@ -461,9 +461,9 @@ static void PacClipSumScatterUpdateFloat(Vector inputs[], AggregateInputData &ag idx_t count) { int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, value_data, sdata; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - states.ToUnifiedFormat(count, sdata); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + states.ToUnifiedFormat(sdata); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); auto state_ptrs = UnifiedVectorFormat::GetData *>(sdata); diff --git a/src/aggregates/as_count.cpp b/src/aggregates/as_count.cpp index c264518f..b0b15711 100644 --- a/src/aggregates/as_count.cpp +++ b/src/aggregates/as_count.cpp @@ -49,7 +49,7 @@ void PacCountUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count ScatterState &agg = *reinterpret_cast(state_ptr); int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat idata; - inputs[0].ToUnifiedFormat(count, idata); + inputs[0].ToUnifiedFormat(idata); auto input_data = UnifiedVectorFormat::GetData(idata); for (idx_t i = 0; i < count; i++) { auto idx = idata.sel->get_index(i); @@ -71,8 +71,8 @@ void PacCountColumnUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input ScatterState &agg = *reinterpret_cast(state_ptr); int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, col_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, col_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(col_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -92,8 +92,8 @@ void PacCountColumnUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input void PacCountScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat idata, sdata; - inputs[0].ToUnifiedFormat(count, idata); - states.ToUnifiedFormat(count, sdata); + inputs[0].ToUnifiedFormat(idata); + states.ToUnifiedFormat(sdata); auto input_data = UnifiedVectorFormat::GetData(idata); auto state_p = UnifiedVectorFormat::GetData(sdata); for (idx_t i = 0; i < count; i++) { @@ -112,9 +112,9 @@ void PacCountScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vec void PacCountColumnScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, col_data, sdata; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, col_data); - states.ToUnifiedFormat(count, sdata); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(col_data); + states.ToUnifiedFormat(sdata); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto state_p = UnifiedVectorFormat::GetData(sdata); for (idx_t i = 0; i < count; i++) { @@ -137,7 +137,7 @@ static void DpSampleCountMaskUpdate(Vector inputs[], AggregateInputData &aggr, i ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { ScatterState &agg = *reinterpret_cast(state_ptr); UnifiedVectorFormat idata; - inputs[0].ToUnifiedFormat(count, idata); + inputs[0].ToUnifiedFormat(idata); auto input_data = UnifiedVectorFormat::GetData(idata); for (idx_t i = 0; i < count; i++) { auto idx = idata.sel->get_index(i); @@ -155,8 +155,8 @@ static void DpSampleCountMaskUpdate(Vector inputs[], AggregateInputData &aggr, i static void DpSampleCountMaskScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { UnifiedVectorFormat idata, sdata; - inputs[0].ToUnifiedFormat(count, idata); - states.ToUnifiedFormat(count, sdata); + inputs[0].ToUnifiedFormat(idata); + states.ToUnifiedFormat(sdata); auto input_data = UnifiedVectorFormat::GetData(idata); auto state_p = UnifiedVectorFormat::GetData(sdata); for (idx_t i = 0; i < count; i++) { @@ -521,7 +521,7 @@ static void DpSampleMCountUpdate(Vector inputs[], AggregateInputData &aggr, idx_ auto &state = *reinterpret_cast(state_ptr); int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data; - inputs[0].ToUnifiedFormat(count, hash_data); + inputs[0].ToUnifiedFormat(hash_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -538,8 +538,8 @@ static void DpSampleMCountColumnUpdate(Vector inputs[], AggregateInputData &aggr auto &state = *reinterpret_cast(state_ptr); int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, col_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, col_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(col_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -554,8 +554,8 @@ static void DpSampleMCountColumnUpdate(Vector inputs[], AggregateInputData &aggr static void DpSampleMCountScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); for (idx_t i = 0; i < count; i++) { @@ -571,9 +571,9 @@ static void DpSampleMCountColumnScatterUpdate(Vector inputs[], AggregateInputDat idx_t count) { int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, col_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, col_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(col_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); for (idx_t i = 0; i < count; i++) { @@ -592,12 +592,12 @@ static void DpSampleMCountIfUpdate(Vector inputs[], AggregateInputData &aggr, id auto &state = *reinterpret_cast(state_ptr); int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, predicate_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, predicate_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(predicate_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto &predicate_child = ListVector::GetEntry(inputs[1]); + auto &predicate_child = ListVector::GetChild(inputs[1]); UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), predicate_child_data); + predicate_child.ToUnifiedFormat(predicate_child_data); auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -618,14 +618,14 @@ static void DpSampleMCountIfScatterUpdate(Vector inputs[], AggregateInputData &a idx_t count) { int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, predicate_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, predicate_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(predicate_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); - auto &predicate_child = ListVector::GetEntry(inputs[1]); + auto &predicate_child = ListVector::GetChild(inputs[1]); UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), predicate_child_data); + predicate_child.ToUnifiedFormat(predicate_child_data); auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -726,8 +726,8 @@ static void DpSampleMCountDistinctIfUpdate(Vector inputs[], AggregateInputData & auto &state = *reinterpret_cast(state_ptr); int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, include_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, include_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(include_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto include_values = UnifiedVectorFormat::GetData(include_data); for (idx_t i = 0; i < count; i++) { @@ -745,9 +745,9 @@ static void DpSampleMCountDistinctIfScatterUpdate(Vector inputs[], AggregateInpu idx_t count) { int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, include_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, include_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(include_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto include_values = UnifiedVectorFormat::GetData(include_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); @@ -862,7 +862,7 @@ static void DpSampleMCountDistinctValuesUpdateOne(DpSampleMCountDistinctValuesSt } auto sets = EnsureDpSampleMCountDistinctValueSets(state, allocator, sample_count); auto &lane_set = sets[DpSassLaneIndex(hash, sample_count)]; - auto &child_vec = ListVector::GetEntry(list_input); + auto &child_vec = ListVector::GetChild(list_input); for (idx_t offset = 0; offset < entry.length; offset++) { auto child_idx = child_data.sel->get_index(entry.offset + offset); if (child_data.validity.RowIsValid(child_idx)) { @@ -877,12 +877,12 @@ static void DpSampleMCountDistinctValuesUpdate(Vector inputs[], AggregateInputDa auto &state = *reinterpret_cast(state_ptr); int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, list_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, list_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(list_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto &child_vec = ListVector::GetEntry(inputs[1]); + auto &child_vec = ListVector::GetChild(inputs[1]); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), child_data); + child_vec.ToUnifiedFormat(child_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); auto list_idx = list_data.sel->get_index(i); @@ -898,14 +898,14 @@ static void DpSampleMCountDistinctValuesScatterUpdate(Vector inputs[], Aggregate idx_t count) { int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, list_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, list_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(list_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); - auto &child_vec = ListVector::GetEntry(inputs[1]); + auto &child_vec = ListVector::GetChild(inputs[1]); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), child_data); + child_vec.ToUnifiedFormat(child_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); auto list_idx = list_data.sel->get_index(i); @@ -1033,7 +1033,7 @@ static void PacSampleMCountUpdate(Vector inputs[], AggregateInputData &aggr, idx auto &state = *reinterpret_cast(state_ptr); auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data; - inputs[0].ToUnifiedFormat(count, hash_data); + inputs[0].ToUnifiedFormat(hash_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -1050,8 +1050,8 @@ static void PacSampleMCountColumnUpdate(Vector inputs[], AggregateInputData &agg auto &state = *reinterpret_cast(state_ptr); auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data, col_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, col_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(col_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -1067,8 +1067,8 @@ static void PacSampleMCountScatterUpdate(Vector inputs[], AggregateInputData &ag idx_t count) { auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); for (idx_t i = 0; i < count; i++) { @@ -1084,9 +1084,9 @@ static void PacSampleMCountColumnScatterUpdate(Vector inputs[], AggregateInputDa idx_t count) { auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data, col_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, col_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(col_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); for (idx_t i = 0; i < count; i++) { @@ -1105,12 +1105,12 @@ static void PacSampleMCountIfUpdate(Vector inputs[], AggregateInputData &aggr, i auto &state = *reinterpret_cast(state_ptr); auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data, predicate_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, predicate_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(predicate_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); - auto &predicate_child = ListVector::GetEntry(inputs[1]); + auto &predicate_child = ListVector::GetChild(inputs[1]); UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), predicate_child_data); + predicate_child.ToUnifiedFormat(predicate_child_data); auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -1132,14 +1132,14 @@ static void PacSampleMCountIfScatterUpdate(Vector inputs[], AggregateInputData & idx_t count) { auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data, predicate_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, predicate_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(predicate_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); - auto &predicate_child = ListVector::GetEntry(inputs[1]); + auto &predicate_child = ListVector::GetChild(inputs[1]); UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[1]), predicate_child_data); + predicate_child.ToUnifiedFormat(predicate_child_data); auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); diff --git a/src/aggregates/as_finalize.cpp b/src/aggregates/as_finalize.cpp index aa8566f8..73357426 100644 --- a/src/aggregates/as_finalize.cpp +++ b/src/aggregates/as_finalize.cpp @@ -27,14 +27,14 @@ static void PacFinalizeFunction(DataChunk &args, ExpressionState &state, Vector idx_t count = args.size(); UnifiedVectorFormat list_data; - list_vec.ToUnifiedFormat(count, list_data); + list_vec.ToUnifiedFormat(list_data); auto result_data = FlatVector::GetDataMutable(result); auto &result_validity = FlatVector::ValidityMutable(result); - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); auto list_entries = UnifiedVectorFormat::GetData(list_data); @@ -80,11 +80,11 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader) { PacRandomEngine gen(seed, 0); UnifiedVectorFormat list_data; - source.ToUnifiedFormat(count, list_data); + source.ToUnifiedFormat(list_data); - auto &child_vec = ListVector::GetEntry(source); + auto &child_vec = ListVector::GetChild(source); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(source), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); auto list_entries = UnifiedVectorFormat::GetData(list_data); @@ -138,7 +138,7 @@ void RegisterPacFinalizeFunction(ExtensionLoader &loader) { auto child_data = FlatVector::GetDataMutable(child_vec); auto list_entries = FlatVector::GetDataMutable(result); UnifiedVectorFormat source_data; - source.ToUnifiedFormat(count, source_data); + source.ToUnifiedFormat(source_data); idx_t offset = 0; for (idx_t i = 0; i < count; i++) { auto src_idx = source_data.sel->get_index(i); diff --git a/src/aggregates/as_min_max.cpp b/src/aggregates/as_min_max.cpp index 75218dd3..818a7855 100644 --- a/src/aggregates/as_min_max.cpp +++ b/src/aggregates/as_min_max.cpp @@ -31,8 +31,8 @@ static void PacMinMaxUpdate(Vector inputs[], AggregateInputData &aggr, idx_t inp int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); @@ -53,9 +53,9 @@ template static void PacMinMaxScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, value_data, sdata; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - states.ToUnifiedFormat(count, sdata); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + states.ToUnifiedFormat(sdata); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); @@ -466,8 +466,8 @@ static void DpSampleMMinMaxUpdate(Vector inputs[], AggregateInputData &aggr, idx auto &state = *reinterpret_cast *>(state_ptr); int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); for (idx_t i = 0; i < count; i++) { @@ -485,9 +485,9 @@ static void DpSampleMMinMaxScatterUpdate(Vector inputs[], AggregateInputData &ag idx_t count) { int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, value_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); auto state_ptrs = UnifiedVectorFormat::GetData *>(state_data); diff --git a/src/aggregates/as_sum.cpp b/src/aggregates/as_sum.cpp index 07f08014..37cee971 100644 --- a/src/aggregates/as_sum.cpp +++ b/src/aggregates/as_sum.cpp @@ -250,12 +250,12 @@ inline void PacSumUpdateOne(PacSumIntStateWrapper &agg, uint64_t key_has template static void PacSumUpdate(Vector inputs[], State &state, idx_t count, ArenaAllocator &allocator, int sample_lanes = 0) { UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); - if (hash_data.validity.AllValid() && value_data.validity.AllValid()) { + if (hash_data.validity.CannotHaveNull() && value_data.validity.CannotHaveNull()) { for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); auto v_idx = value_data.sel->get_index(i); @@ -281,9 +281,9 @@ template (hash_data); auto values = UnifiedVectorFormat::GetData(value_data); @@ -1060,9 +1060,9 @@ static void DpSampleAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t i int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, sum_data, count_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, sum_data); - inputs[2].ToUnifiedFormat(count, count_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(sum_data); + inputs[2].ToUnifiedFormat(count_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto sums = UnifiedVectorFormat::GetData(sum_data); @@ -1086,10 +1086,10 @@ static void DpSampleAvgScatterUpdate(Vector inputs[], AggregateInputData &aggr, int sample_lanes = GetPrivSampleLanes(aggr); UnifiedVectorFormat hash_data, sum_data, count_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, sum_data); - inputs[2].ToUnifiedFormat(count, count_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(sum_data); + inputs[2].ToUnifiedFormat(count_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto sums = UnifiedVectorFormat::GetData(sum_data); @@ -1235,8 +1235,8 @@ static void DpSampleMSumUpdate(Vector inputs[], AggregateInputData &aggr, idx_t auto &state = *reinterpret_cast(state_ptr); int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); for (idx_t i = 0; i < count; i++) { @@ -1252,9 +1252,9 @@ static void DpSampleMSumUpdate(Vector inputs[], AggregateInputData &aggr, idx_t static void DpSampleMSumScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, value_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); @@ -1283,8 +1283,8 @@ static void DpSampleMNullableSumUpdate(Vector inputs[], AggregateInputData &aggr auto &state = *reinterpret_cast(state_ptr); int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); for (idx_t i = 0; i < count; i++) { @@ -1301,9 +1301,9 @@ static void DpSampleMNullableSumScatterUpdate(Vector inputs[], AggregateInputDat idx_t count) { int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, value_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); @@ -1323,14 +1323,14 @@ static void DpSampleMSumIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_ auto &state = *reinterpret_cast(state_ptr); int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, value_data, predicate_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - inputs[2].ToUnifiedFormat(count, predicate_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + inputs[2].ToUnifiedFormat(predicate_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); - auto &predicate_child = ListVector::GetEntry(inputs[2]); + auto &predicate_child = ListVector::GetChild(inputs[2]); UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[2]), predicate_child_data); + predicate_child.ToUnifiedFormat(predicate_child_data); auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -1351,16 +1351,16 @@ static void DpSampleMSumIfUpdate(Vector inputs[], AggregateInputData &aggr, idx_ static void DpSampleMSumIfScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, value_data, predicate_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - inputs[2].ToUnifiedFormat(count, predicate_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + inputs[2].ToUnifiedFormat(predicate_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); - auto &predicate_child = ListVector::GetEntry(inputs[2]); + auto &predicate_child = ListVector::GetChild(inputs[2]); UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[2]), predicate_child_data); + predicate_child.ToUnifiedFormat(predicate_child_data); auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -1559,9 +1559,9 @@ static void DpSampleMAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t auto &state = *reinterpret_cast(state_ptr); int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, sum_data, count_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, sum_data); - inputs[2].ToUnifiedFormat(count, count_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(sum_data); + inputs[2].ToUnifiedFormat(count_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto sums = UnifiedVectorFormat::GetData(sum_data); auto counts = UnifiedVectorFormat::GetData(count_data); @@ -1580,10 +1580,10 @@ static void DpSampleMAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t static void DpSampleMAvgScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { int sample_count = GetPrivSampleCount(aggr); UnifiedVectorFormat hash_data, sum_data, count_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, sum_data); - inputs[2].ToUnifiedFormat(count, count_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(sum_data); + inputs[2].ToUnifiedFormat(count_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto sums = UnifiedVectorFormat::GetData(sum_data); auto counts = UnifiedVectorFormat::GetData(count_data); @@ -1678,8 +1678,8 @@ static void PacSampleMSumUpdate(Vector inputs[], AggregateInputData &aggr, idx_t auto &state = *reinterpret_cast(state_ptr); auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); for (idx_t i = 0; i < count; i++) { @@ -1695,9 +1695,9 @@ static void PacSampleMSumUpdate(Vector inputs[], AggregateInputData &aggr, idx_t static void PacSampleMSumScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data, value_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); @@ -1717,14 +1717,14 @@ static void PacSampleMSumIfUpdate(Vector inputs[], AggregateInputData &aggr, idx auto &state = *reinterpret_cast(state_ptr); auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data, value_data, predicate_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - inputs[2].ToUnifiedFormat(count, predicate_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + inputs[2].ToUnifiedFormat(predicate_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); - auto &predicate_child = ListVector::GetEntry(inputs[2]); + auto &predicate_child = ListVector::GetChild(inputs[2]); UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[2]), predicate_child_data); + predicate_child.ToUnifiedFormat(predicate_child_data); auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -1747,16 +1747,16 @@ static void PacSampleMSumIfScatterUpdate(Vector inputs[], AggregateInputData &ag idx_t count) { auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data, value_data, predicate_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, value_data); - inputs[2].ToUnifiedFormat(count, predicate_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + inputs[2].ToUnifiedFormat(predicate_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); - auto &predicate_child = ListVector::GetEntry(inputs[2]); + auto &predicate_child = ListVector::GetChild(inputs[2]); UnifiedVectorFormat predicate_child_data; - predicate_child.ToUnifiedFormat(ListVector::GetListSize(inputs[2]), predicate_child_data); + predicate_child.ToUnifiedFormat(predicate_child_data); auto predicate_values = UnifiedVectorFormat::GetData(predicate_child_data); for (idx_t i = 0; i < count; i++) { auto h_idx = hash_data.sel->get_index(i); @@ -1818,9 +1818,9 @@ static void PacSampleMAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t auto &state = *reinterpret_cast(state_ptr); auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data, sum_data, count_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, sum_data); - inputs[2].ToUnifiedFormat(count, count_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(sum_data); + inputs[2].ToUnifiedFormat(count_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto sums = UnifiedVectorFormat::GetData(sum_data); auto counts = UnifiedVectorFormat::GetData(count_data); @@ -1839,10 +1839,10 @@ static void PacSampleMAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t static void PacSampleMAvgScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { auto &bind = aggr.bind_data->Cast(); UnifiedVectorFormat hash_data, sum_data, count_data, state_data; - inputs[0].ToUnifiedFormat(count, hash_data); - inputs[1].ToUnifiedFormat(count, sum_data); - inputs[2].ToUnifiedFormat(count, count_data); - states.ToUnifiedFormat(count, state_data); + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(sum_data); + inputs[2].ToUnifiedFormat(count_data); + states.ToUnifiedFormat(state_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto sums = UnifiedVectorFormat::GetData(sum_data); auto counts = UnifiedVectorFormat::GetData(count_data); diff --git a/src/aggregates/dp_approx_bounds.cpp b/src/aggregates/dp_approx_bounds.cpp index 52a9f2f7..2706be5c 100644 --- a/src/aggregates/dp_approx_bounds.cpp +++ b/src/aggregates/dp_approx_bounds.cpp @@ -241,7 +241,7 @@ struct DpApproxBoundsInputs { const double *values; DpApproxBoundsInputs(Vector inputs[], idx_t count) { - inputs[0].ToUnifiedFormat(count, value); + inputs[0].ToUnifiedFormat(value); values = UnifiedVectorFormat::GetData(value); } @@ -274,7 +274,7 @@ static void DpApproxBoundsUpdate(Vector inputs[], AggregateInputData &input, idx static void DpApproxBoundsScatterUpdate(Vector inputs[], AggregateInputData &input, idx_t, Vector &states, idx_t count) { UnifiedVectorFormat state_data; - states.ToUnifiedFormat(count, state_data); + states.ToUnifiedFormat(state_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); UpdateRows(inputs, input, count, [&](idx_t row) { return state_ptrs[state_data.sel->get_index(row)]; }); } @@ -576,7 +576,7 @@ static void InsertBoundedValue(DpBoundedValueListState &state, double value, idx template static void UpdateBoundedValueRows(Vector inputs[], AggregateInputData &input, idx_t count, STATE_GETTER get_state) { UnifiedVectorFormat values; - inputs[0].ToUnifiedFormat(count, values); + inputs[0].ToUnifiedFormat(values); auto value_data = UnifiedVectorFormat::GetData(values); auto max_values = input.bind_data->Cast().max_values; for (idx_t row = 0; row < count; row++) { @@ -599,7 +599,7 @@ static void DpBoundedValueListUpdate(Vector inputs[], AggregateInputData &input, static void DpBoundedValueListScatterUpdate(Vector inputs[], AggregateInputData &input, idx_t, Vector &states, idx_t count) { UnifiedVectorFormat state_data; - states.ToUnifiedFormat(count, state_data); + states.ToUnifiedFormat(state_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); UpdateBoundedValueRows(inputs, input, count, [&](idx_t row) { return state_ptrs[state_data.sel->get_index(row)]; }); } @@ -766,11 +766,11 @@ static void WriteDebug(Vector &result, idx_t row, const DpApproxBoundsResult &va template static void UpdateMeanRows(Vector inputs[], AggregateInputData &input, idx_t count, STATE_GETTER get_state) { UnifiedVectorFormat lists; - inputs[0].ToUnifiedFormat(count, lists); + inputs[0].ToUnifiedFormat(lists); auto entries = UnifiedVectorFormat::GetData(lists); - auto &child = ListVector::GetEntry(inputs[0]); + auto &child = ListVector::GetChild(inputs[0]); UnifiedVectorFormat values; - child.ToUnifiedFormat(ListVector::GetListSize(inputs[0]), values); + child.ToUnifiedFormat(values); auto value_data = UnifiedVectorFormat::GetData(values); auto max_contributions = input.bind_data->Cast().max_contributions; for (idx_t row = 0; row < count; row++) { @@ -805,7 +805,7 @@ static void DpApproxBoundsMeanUpdate(Vector inputs[], AggregateInputData &input, static void DpApproxBoundsMeanScatterUpdate(Vector inputs[], AggregateInputData &input, idx_t, Vector &states, idx_t count) { UnifiedVectorFormat state_data; - states.ToUnifiedFormat(count, state_data); + states.ToUnifiedFormat(state_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); UpdateMeanRows(inputs, input, count, [&](idx_t row) { return state_ptrs[state_data.sel->get_index(row)]; }); } diff --git a/src/aggregates/dp_laplace_noise.cpp b/src/aggregates/dp_laplace_noise.cpp index d33173bf..5c25a7e3 100644 --- a/src/aggregates/dp_laplace_noise.cpp +++ b/src/aggregates/dp_laplace_noise.cpp @@ -130,12 +130,12 @@ static void DpSassStabilityFunction(DataChunk &args, ExpressionState &, Vector & idx_t count = args.size(); auto &list_vec = args.data[0]; UnifiedVectorFormat list_data; - list_vec.ToUnifiedFormat(count, list_data); + list_vec.ToUnifiedFormat(list_data); auto list_entries = UnifiedVectorFormat::GetData(list_data); - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); auto &entries = StructVector::GetEntries(result); @@ -274,14 +274,14 @@ static void DpSassRecordStabilityFunction(DataChunk &args, ExpressionState &, Ve idx_t count = args.size(); auto &list_vec = args.data[0]; UnifiedVectorFormat list_data, aggregate_index_data; - list_vec.ToUnifiedFormat(count, list_data); - args.data[1].ToUnifiedFormat(count, aggregate_index_data); + list_vec.ToUnifiedFormat(list_data); + args.data[1].ToUnifiedFormat(aggregate_index_data); auto list_entries = UnifiedVectorFormat::GetData(list_data); auto aggregate_indexes = UnifiedVectorFormat::GetData(aggregate_index_data); - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); auto result_data = FlatVector::GetDataMutable(result); @@ -496,8 +496,8 @@ static void DpLaplaceNoiseFunction(DataChunk &args, ExpressionState &, Vector &r auto count = args.size(); UnifiedVectorFormat values; UnifiedVectorFormat scales; - args.data[0].ToUnifiedFormat(count, values); - args.data[1].ToUnifiedFormat(count, scales); + args.data[0].ToUnifiedFormat(values); + args.data[1].ToUnifiedFormat(scales); auto value_data = UnifiedVectorFormat::GetData(values); auto scale_data = UnifiedVectorFormat::GetData(scales); // BinaryExecutor preserves a CONSTANT vector by invoking its callback once. Noise is volatile, so @@ -775,28 +775,28 @@ static void DpSmoothMedianNoiseFunctionInternal(DataChunk &args, ExpressionState auto &list_vec = args.data[0]; UnifiedVectorFormat list_data, epsilon_data, delta_data, sample_lanes_data, lower_data, upper_data, empty_data, aggregate_index_data, aggregate_count_data, group_identity_data; - list_vec.ToUnifiedFormat(count, list_data); - args.data[1].ToUnifiedFormat(count, epsilon_data); - args.data[2].ToUnifiedFormat(count, delta_data); - args.data[3].ToUnifiedFormat(count, sample_lanes_data); + list_vec.ToUnifiedFormat(list_data); + args.data[1].ToUnifiedFormat(epsilon_data); + args.data[2].ToUnifiedFormat(delta_data); + args.data[3].ToUnifiedFormat(sample_lanes_data); if (bounded) { - args.data[4].ToUnifiedFormat(count, lower_data); - args.data[5].ToUnifiedFormat(count, upper_data); + args.data[4].ToUnifiedFormat(lower_data); + args.data[5].ToUnifiedFormat(upper_data); } if (has_empty_default) { - args.data[6].ToUnifiedFormat(count, empty_data); + args.data[6].ToUnifiedFormat(empty_data); } if (record_scale) { - args.data[args.ColumnCount() - 1].ToUnifiedFormat(count, aggregate_index_data); + args.data[args.ColumnCount() - 1].ToUnifiedFormat(aggregate_index_data); } else if (has_cell_identity) { - args.data[7].ToUnifiedFormat(count, aggregate_index_data); - args.data[8].ToUnifiedFormat(count, aggregate_count_data); - args.data[9].ToUnifiedFormat(count, group_identity_data); + args.data[7].ToUnifiedFormat(aggregate_index_data); + args.data[8].ToUnifiedFormat(aggregate_count_data); + args.data[9].ToUnifiedFormat(group_identity_data); } - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); auto list_entries = UnifiedVectorFormat::GetData(list_data); auto epsilons = UnifiedVectorFormat::GetData(epsilon_data); @@ -897,16 +897,16 @@ static void DpAggregateFunction(DataChunk &args, ExpressionState &state, Vector idx_t count = args.size(); auto &list_vec = args.data[0]; // samples (counts at args.data[1] is intentionally ignored) UnifiedVectorFormat list_data, epsilon_data, delta_data, sample_lanes_data, lower_data, upper_data; - list_vec.ToUnifiedFormat(count, list_data); - args.data[2].ToUnifiedFormat(count, epsilon_data); - args.data[3].ToUnifiedFormat(count, delta_data); - args.data[4].ToUnifiedFormat(count, sample_lanes_data); - args.data[5].ToUnifiedFormat(count, lower_data); - args.data[6].ToUnifiedFormat(count, upper_data); - - auto &child_vec = ListVector::GetEntry(list_vec); + list_vec.ToUnifiedFormat(list_data); + args.data[2].ToUnifiedFormat(epsilon_data); + args.data[3].ToUnifiedFormat(delta_data); + args.data[4].ToUnifiedFormat(sample_lanes_data); + args.data[5].ToUnifiedFormat(lower_data); + args.data[6].ToUnifiedFormat(upper_data); + + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); auto list_entries = UnifiedVectorFormat::GetData(list_data); auto epsilons = UnifiedVectorFormat::GetData(epsilon_data); @@ -1003,25 +1003,25 @@ static void DpGuptMeanNoiseFunctionInternal(DataChunk &args, ExpressionState &st auto &list_vec = args.data[0]; UnifiedVectorFormat list_data, epsilon_data, sample_lanes_data, lower_data, upper_data, empty_data, aggregate_index_data, aggregate_count_data, group_identity_data; - list_vec.ToUnifiedFormat(count, list_data); - args.data[1].ToUnifiedFormat(count, epsilon_data); - args.data[2].ToUnifiedFormat(count, sample_lanes_data); - args.data[3].ToUnifiedFormat(count, lower_data); - args.data[4].ToUnifiedFormat(count, upper_data); + list_vec.ToUnifiedFormat(list_data); + args.data[1].ToUnifiedFormat(epsilon_data); + args.data[2].ToUnifiedFormat(sample_lanes_data); + args.data[3].ToUnifiedFormat(lower_data); + args.data[4].ToUnifiedFormat(upper_data); if (has_empty_default) { - args.data[5].ToUnifiedFormat(count, empty_data); + args.data[5].ToUnifiedFormat(empty_data); } if (record_scale) { - args.data[args.ColumnCount() - 1].ToUnifiedFormat(count, aggregate_index_data); + args.data[args.ColumnCount() - 1].ToUnifiedFormat(aggregate_index_data); } else if (has_cell_identity) { - args.data[6].ToUnifiedFormat(count, aggregate_index_data); - args.data[7].ToUnifiedFormat(count, aggregate_count_data); - args.data[8].ToUnifiedFormat(count, group_identity_data); + args.data[6].ToUnifiedFormat(aggregate_index_data); + args.data[7].ToUnifiedFormat(aggregate_count_data); + args.data[8].ToUnifiedFormat(group_identity_data); } - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); auto list_entries = UnifiedVectorFormat::GetData(list_data); auto epsilons = UnifiedVectorFormat::GetData(epsilon_data); @@ -1224,22 +1224,22 @@ static void DpNonEmptyMeanNoiseFunctionInternal(DataChunk &args, ExpressionState UnifiedVectorFormat list_data, epsilon_data, sample_lanes_data, lower_data, upper_data, baseline_data, sensitivity_data, aggregate_index_data; UnifiedVectorFormat aggregate_count_data, group_identity_data; - list_vec.ToUnifiedFormat(count, list_data); - args.data[1].ToUnifiedFormat(count, epsilon_data); - args.data[2].ToUnifiedFormat(count, sample_lanes_data); - args.data[3].ToUnifiedFormat(count, lower_data); - args.data[4].ToUnifiedFormat(count, upper_data); - args.data[5].ToUnifiedFormat(count, baseline_data); - args.data[6].ToUnifiedFormat(count, sensitivity_data); - args.data[7].ToUnifiedFormat(count, aggregate_index_data); + list_vec.ToUnifiedFormat(list_data); + args.data[1].ToUnifiedFormat(epsilon_data); + args.data[2].ToUnifiedFormat(sample_lanes_data); + args.data[3].ToUnifiedFormat(lower_data); + args.data[4].ToUnifiedFormat(upper_data); + args.data[5].ToUnifiedFormat(baseline_data); + args.data[6].ToUnifiedFormat(sensitivity_data); + args.data[7].ToUnifiedFormat(aggregate_index_data); if (!record_scales) { - args.data[8].ToUnifiedFormat(count, aggregate_count_data); - args.data[9].ToUnifiedFormat(count, group_identity_data); + args.data[8].ToUnifiedFormat(aggregate_count_data); + args.data[9].ToUnifiedFormat(group_identity_data); } - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); auto list_entries = UnifiedVectorFormat::GetData(list_data); auto epsilons = UnifiedVectorFormat::GetData(epsilon_data); diff --git a/src/aggregates/filterless_aggregate.cpp b/src/aggregates/filterless_aggregate.cpp index a0aef8b6..cade04e6 100644 --- a/src/aggregates/filterless_aggregate.cpp +++ b/src/aggregates/filterless_aggregate.cpp @@ -756,12 +756,12 @@ struct FilterlessInputVectors { const INPUT_TYPE *numeric_values[VALUE_COUNT]; FilterlessInputVectors(Vector inputs[], idx_t count) { - inputs[0].ToUnifiedFormat(count, pu); - inputs[1].ToUnifiedFormat(count, active); + inputs[0].ToUnifiedFormat(pu); + inputs[1].ToUnifiedFormat(active); pu_values = UnifiedVectorFormat::GetData(pu); active_values = UnifiedVectorFormat::GetData(active); for (idx_t i = 0; i < VALUE_COUNT; i++) { - inputs[2 + i].ToUnifiedFormat(count, values[i]); + inputs[2 + i].ToUnifiedFormat(values[i]); numeric_values[i] = UnifiedVectorFormat::GetData(values[i]); } } @@ -842,7 +842,7 @@ static void FilterlessUpdate(Vector inputs[], AggregateInputData &aggr, idx_t in static void FilterlessScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { UnifiedVectorFormat state_data; - states.ToUnifiedFormat(count, state_data); + states.ToUnifiedFormat(state_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); FilterlessUpdateRows<2, double>(inputs, aggr, count, [&](idx_t row) { return state_ptrs[state_data.sel->get_index(row)]; }); @@ -858,7 +858,7 @@ static void FilterlessAvgUpdate(Vector inputs[], AggregateInputData &aggr, idx_t static void FilterlessAvgScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { UnifiedVectorFormat state_data; - states.ToUnifiedFormat(count, state_data); + states.ToUnifiedFormat(state_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); FilterlessUpdateRows<4, double>(inputs, aggr, count, [&](idx_t row) { return state_ptrs[state_data.sel->get_index(row)]; }); @@ -877,7 +877,7 @@ template static void FilterlessExactScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { UnifiedVectorFormat state_data; - states.ToUnifiedFormat(count, state_data); + states.ToUnifiedFormat(state_data); auto state_ptrs = UnifiedVectorFormat::GetData(state_data); FilterlessUpdateRows<2, INPUT_TYPE>(inputs, aggr, count, [&](idx_t row) { return state_ptrs[state_data.sel->get_index(row)]; }); @@ -1111,12 +1111,12 @@ static void FilterlessFusedAvgUpdateRows(Vector inputs[], AggregateInputData &ag STATE_GETTER get_state) { UnifiedVectorFormat pu_data, active_data, answer_sum_data, answer_count_data, histogram_sum_data, histogram_count_data; - inputs[0].ToUnifiedFormat(count, pu_data); - inputs[1].ToUnifiedFormat(count, active_data); - inputs[2].ToUnifiedFormat(count, answer_sum_data); - inputs[3].ToUnifiedFormat(count, answer_count_data); - inputs[4].ToUnifiedFormat(count, histogram_sum_data); - inputs[5].ToUnifiedFormat(count, histogram_count_data); + inputs[0].ToUnifiedFormat(pu_data); + inputs[1].ToUnifiedFormat(active_data); + inputs[2].ToUnifiedFormat(answer_sum_data); + inputs[3].ToUnifiedFormat(answer_count_data); + inputs[4].ToUnifiedFormat(histogram_sum_data); + inputs[5].ToUnifiedFormat(histogram_count_data); auto pus = UnifiedVectorFormat::GetData(pu_data); auto active = UnifiedVectorFormat::GetData(active_data); auto answer_sums = UnifiedVectorFormat::GetData(answer_sum_data); @@ -1165,7 +1165,7 @@ template static void FilterlessFusedAvgScatterUpdate(Vector inputs[], AggregateInputData &aggr, idx_t, Vector &states, idx_t count) { UnifiedVectorFormat state_data; - states.ToUnifiedFormat(count, state_data); + states.ToUnifiedFormat(state_data); auto state_ptrs = UnifiedVectorFormat::GetData *>(state_data); FilterlessFusedAvgUpdateRows(inputs, aggr, count, [&](idx_t row) { return state_ptrs[state_data.sel->get_index(row)]; }); @@ -1393,9 +1393,9 @@ template static void FilterlessLowerPairUpdateRows(Vector inputs[], idx_t count, ArenaAllocator &allocator, STATE_GETTER get_state) { UnifiedVectorFormat value_data, active_data, sampled_data; - inputs[0].ToUnifiedFormat(count, value_data); - inputs[1].ToUnifiedFormat(count, active_data); - inputs[2].ToUnifiedFormat(count, sampled_data); + inputs[0].ToUnifiedFormat(value_data); + inputs[1].ToUnifiedFormat(active_data); + inputs[2].ToUnifiedFormat(sampled_data); auto values = UnifiedVectorFormat::GetData(value_data); auto active = UnifiedVectorFormat::GetData(active_data); auto sampled = UnifiedVectorFormat::GetData(sampled_data); @@ -1433,7 +1433,7 @@ template static void FilterlessLowerPairScatterUpdate(Vector inputs[], AggregateInputData &input, idx_t, Vector &states, idx_t count) { UnifiedVectorFormat state_data; - states.ToUnifiedFormat(count, state_data); + states.ToUnifiedFormat(state_data); auto state_ptrs = UnifiedVectorFormat::GetData *>(state_data); FilterlessLowerPairUpdateRows(inputs, count, input.allocator, [&](idx_t row) { return state_ptrs[state_data.sel->get_index(row)]; }); diff --git a/src/categorical/pac_categorical.cpp b/src/categorical/pac_categorical.cpp index e3b720f7..9fe38b78 100644 --- a/src/categorical/pac_categorical.cpp +++ b/src/categorical/pac_categorical.cpp @@ -98,7 +98,7 @@ struct PacCategoricalBindData : public FunctionData { // ============================================================================ struct PacCategoricalLocalState : public FunctionLocalState { PacRandomEngine gen; - explicit PacCategoricalLocalState(PacRandomEngine gen_p) : gen(std::move(gen_p)) { + explicit PacCategoricalLocalState(PacRandomEngine gen_p) : gen(gen_p) { } }; @@ -179,16 +179,16 @@ static void PacSelectFunction(DataChunk &args, ExpressionState &state, Vector &r idx_t count = args.size(); UnifiedVectorFormat hash_data; - args.data[0].ToUnifiedFormat(count, hash_data); + args.data[0].ToUnifiedFormat(hash_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto &list_vec = args.data[1]; UnifiedVectorFormat list_data; - list_vec.ToUnifiedFormat(count, list_data); + list_vec.ToUnifiedFormat(list_data); - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); auto result_data = FlatVector::GetDataMutable(result); @@ -242,13 +242,13 @@ static void PacFilterFromBoolListFunction(DataChunk &args, ExpressionState &stat auto params = GetPacFilterParams(state); UnifiedVectorFormat list_data; - list_vec.ToUnifiedFormat(count, list_data); + list_vec.ToUnifiedFormat(list_data); auto result_data = FlatVector::GetDataMutable(result); - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); for (idx_t i = 0; i < count; i++) { @@ -319,22 +319,22 @@ static void PacFilterCmpFunction(DataChunk &args, ExpressionState &state, Vector // arg0: PAC_FLOAT scalar value, arg1: LIST counters, arg2 (optional): PAC_FLOAT correction UnifiedVectorFormat val_data; - args.data[0].ToUnifiedFormat(count, val_data); + args.data[0].ToUnifiedFormat(val_data); auto vals = UnifiedVectorFormat::GetData(val_data); auto &list_vec = args.data[1]; UnifiedVectorFormat list_data; - list_vec.ToUnifiedFormat(count, list_data); + list_vec.ToUnifiedFormat(list_data); - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); UnifiedVectorFormat correction_data; const PAC_FLOAT *corrections = nullptr; if (HAS_CORRECTION_ARG) { - args.data[2].ToUnifiedFormat(count, correction_data); + args.data[2].ToUnifiedFormat(correction_data); corrections = UnifiedVectorFormat::GetData(correction_data); } @@ -407,18 +407,18 @@ static void PacSelectCmpFunction(DataChunk &args, ExpressionState &state, Vector // arg0: UBIGINT hash, arg1: PAC_FLOAT scalar value, arg2: LIST counters UnifiedVectorFormat hash_data, val_data; - args.data[0].ToUnifiedFormat(count, hash_data); - args.data[1].ToUnifiedFormat(count, val_data); + args.data[0].ToUnifiedFormat(hash_data); + args.data[1].ToUnifiedFormat(val_data); auto hashes = UnifiedVectorFormat::GetData(hash_data); auto vals = UnifiedVectorFormat::GetData(val_data); auto &list_vec = args.data[2]; UnifiedVectorFormat list_data; - list_vec.ToUnifiedFormat(count, list_data); + list_vec.ToUnifiedFormat(list_data); - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); auto result_data = FlatVector::GetDataMutable(result); @@ -475,10 +475,10 @@ static void PacCoalesceFunction(DataChunk &args, ExpressionState &state, Vector idx_t count = args.size(); UnifiedVectorFormat input_data; - input.ToUnifiedFormat(count, input_data); + input.ToUnifiedFormat(input_data); // Fast path: if no NULL lists, zero-copy pass-through (common case) - if (input_data.validity.AllValid()) { + if (input_data.validity.CannotHaveNull()) { result.Reference(input); return; } @@ -508,7 +508,7 @@ static void PacCoalesceFunction(DataChunk &args, ExpressionState &state, Vector child_validity.EnsureWritable(); // Get the source child vector for non-NULL lists - auto &input_child = ListVector::GetEntry(input); + auto &input_child = ListVector::GetChild(input); bool child_is_flat = (input_child.GetVectorType() == VectorType::FLAT_VECTOR); auto flat_src_data = child_is_flat ? FlatVector::GetData(input_child) : nullptr; auto *flat_src_validity = child_is_flat ? &FlatVector::Validity(input_child) : nullptr; @@ -517,7 +517,7 @@ static void PacCoalesceFunction(DataChunk &args, ExpressionState &state, Vector UnifiedVectorFormat input_child_data; const PAC_FLOAT *src_data = nullptr; if (!child_is_flat) { - input_child.ToUnifiedFormat(ListVector::GetListSize(input), input_child_data); + input_child.ToUnifiedFormat(input_child_data); src_data = UnifiedVectorFormat::GetData(input_child_data); } @@ -584,14 +584,14 @@ static void PacNoisedFunction(DataChunk &args, ExpressionState &state, Vector &r auto gen = PacRandomStream(bind_data.seed, *bind_data.random_state); UnifiedVectorFormat list_data; - list_vec.ToUnifiedFormat(count, list_data); + list_vec.ToUnifiedFormat(list_data); auto result_data = FlatVector::GetDataMutable(result); auto &result_validity = FlatVector::ValidityMutable(result); - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); for (idx_t i = 0; i < count; i++) { @@ -645,14 +645,14 @@ static void PacDivFunction(DataChunk &args, ExpressionState &state, Vector &resu idx_t count = args.size(); UnifiedVectorFormat num_data, den_data; - num_vec.ToUnifiedFormat(count, num_data); - den_vec.ToUnifiedFormat(count, den_data); + num_vec.ToUnifiedFormat(num_data); + den_vec.ToUnifiedFormat(den_data); - auto &num_child = ListVector::GetEntry(num_vec); - auto &den_child = ListVector::GetEntry(den_vec); + auto &num_child = ListVector::GetChild(num_vec); + auto &den_child = ListVector::GetChild(den_vec); UnifiedVectorFormat num_child_data, den_child_data; - num_child.ToUnifiedFormat(ListVector::GetListSize(num_vec), num_child_data); - den_child.ToUnifiedFormat(ListVector::GetListSize(den_vec), den_child_data); + num_child.ToUnifiedFormat(num_child_data); + den_child.ToUnifiedFormat(den_child_data); auto num_values = UnifiedVectorFormat::GetData(num_child_data); auto den_values = UnifiedVectorFormat::GetData(den_child_data); auto num_entries = UnifiedVectorFormat::GetData(num_data); @@ -730,17 +730,17 @@ static void PacNoisedDivFunction(DataChunk &args, ExpressionState &state, Vector auto gen = PacRandomStream(bind_data.seed, *bind_data.random_state); UnifiedVectorFormat sum_data, cnt_data; - sum_vec.ToUnifiedFormat(count, sum_data); - cnt_vec.ToUnifiedFormat(count, cnt_data); + sum_vec.ToUnifiedFormat(sum_data); + cnt_vec.ToUnifiedFormat(cnt_data); auto result_data = FlatVector::GetDataMutable(result); auto &result_validity = FlatVector::ValidityMutable(result); - auto &sum_child = ListVector::GetEntry(sum_vec); - auto &cnt_child = ListVector::GetEntry(cnt_vec); + auto &sum_child = ListVector::GetChild(sum_vec); + auto &cnt_child = ListVector::GetChild(cnt_vec); UnifiedVectorFormat sum_child_data, cnt_child_data; - sum_child.ToUnifiedFormat(ListVector::GetListSize(sum_vec), sum_child_data); - cnt_child.ToUnifiedFormat(ListVector::GetListSize(cnt_vec), cnt_child_data); + sum_child.ToUnifiedFormat(sum_child_data); + cnt_child.ToUnifiedFormat(cnt_child_data); auto sum_values = UnifiedVectorFormat::GetData(sum_child_data); auto cnt_values = UnifiedVectorFormat::GetData(cnt_child_data); @@ -1059,19 +1059,19 @@ static void PacListAggregateUpdate(Vector inputs[], AggregateInputData &, idx_t auto &list_vec = inputs[0]; UnifiedVectorFormat list_data; - list_vec.ToUnifiedFormat(count, list_data); + list_vec.ToUnifiedFormat(list_data); - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); UnifiedVectorFormat sdata; - state_vector.ToUnifiedFormat(count, sdata); + state_vector.ToUnifiedFormat(sdata); auto states = UnifiedVectorFormat::GetData(sdata); // Check if child vector is flat with all valid entries (enables dense path) - bool child_is_flat = child_vec.GetVectorType() == VectorType::FLAT_VECTOR && child_data.validity.AllValid(); + bool child_is_flat = child_vec.GetVectorType() == VectorType::FLAT_VECTOR && child_data.validity.CannotHaveNull(); for (idx_t i = 0; i < count; i++) { auto list_idx = list_data.sel->get_index(i); @@ -1097,8 +1097,8 @@ static void PacListAggregateUpdate(Vector inputs[], AggregateInputData &, idx_t template static void PacListAggregateCombine(Vector &source_vec, Vector &target_vec, AggregateInputData &, idx_t count) { UnifiedVectorFormat sdata, tdata; - source_vec.ToUnifiedFormat(count, sdata); - target_vec.ToUnifiedFormat(count, tdata); + source_vec.ToUnifiedFormat(sdata); + target_vec.ToUnifiedFormat(tdata); auto sources = UnifiedVectorFormat::GetData(sdata); auto targets = UnifiedVectorFormat::GetData(tdata); @@ -1118,7 +1118,7 @@ template static void PacListAggregateFinalize(Vector &state_vector, AggregateFinalizeInputData &, Vector &result, idx_t count, idx_t offset) { UnifiedVectorFormat sdata; - state_vector.ToUnifiedFormat(count, sdata); + state_vector.ToUnifiedFormat(sdata); auto states = UnifiedVectorFormat::GetData(sdata); result.SetVectorType(VectorType::FLAT_VECTOR); diff --git a/src/categorical/pac_categorical_lambdas.cpp b/src/categorical/pac_categorical_lambdas.cpp index f55abc1b..8bc311e5 100644 --- a/src/categorical/pac_categorical_lambdas.cpp +++ b/src/categorical/pac_categorical_lambdas.cpp @@ -439,8 +439,9 @@ unique_ptr TryRewriteFilterComparison(OptimizerExtensionInput &input arg_types.push_back(a->GetReturnType()); } auto &catalog = Catalog::GetSystemCatalog(input.context); - auto &entry = - catalog.GetEntry(input.context, DEFAULT_SCHEMA, Identifier(func_name)); + auto &entry = catalog.GetEntry( + input.context, QualifiedName(Catalog::GetSystemCatalog(input.context).GetName(), + Identifier::DefaultSchema(), Identifier(func_name))); ErrorData error; FunctionBinder func_binder(input.context); auto best = func_binder.BindFunction(Identifier(func_name), entry.functions, arg_types, error); diff --git a/src/compiler/privacy_mechanisms.cpp b/src/compiler/privacy_mechanisms.cpp index f9b6ab65..5df90e32 100644 --- a/src/compiler/privacy_mechanisms.cpp +++ b/src/compiler/privacy_mechanisms.cpp @@ -107,7 +107,9 @@ static unique_ptr BindAggregateLocal(OptimizerExtensionInput &input, } auto &entry = Catalog::GetSystemCatalog(input.context) - .GetEntry(input.context, DEFAULT_SCHEMA, Identifier(func_name)); + .GetEntry( + input.context, QualifiedName(Catalog::GetSystemCatalog(input.context).GetName(), + Identifier::DefaultSchema(), Identifier(func_name))); auto best = function_binder.BindFunction(Identifier(entry.name), entry.functions, arg_types, error); if (!best.IsValid()) { throw InternalException("dp: failed to bind aggregate function '" + func_name + "': " + error.Message()); diff --git a/src/include/categorical/pac_categorical_detection.hpp b/src/include/categorical/pac_categorical_detection.hpp index 0c505ecf..401a57cc 100644 --- a/src/include/categorical/pac_categorical_detection.hpp +++ b/src/include/categorical/pac_categorical_detection.hpp @@ -187,7 +187,9 @@ static inline uint64_t HashBinding(const ColumnBinding &binding) { static inline unique_ptr RebindAggregate(ClientContext &context, const string &func_name, vector> children, bool is_distinct) { auto &catalog = Catalog::GetSystemCatalog(context); - auto &func_entry = catalog.GetEntry(context, DEFAULT_SCHEMA, Identifier(func_name)); + auto &func_entry = catalog.GetEntry( + context, QualifiedName(Catalog::GetSystemCatalog(context).GetName(), Identifier::DefaultSchema(), + Identifier(func_name))); vector arg_types; for (auto &child : children) { arg_types.push_back(child->GetReturnType()); diff --git a/src/parser/privacy_parser.cpp b/src/parser/privacy_parser.cpp index 8eff92d8..7c2d5936 100644 --- a/src/parser/privacy_parser.cpp +++ b/src/parser/privacy_parser.cpp @@ -255,7 +255,7 @@ static unique_ptr PACDDLBindFunction(ClientContext &context, Table */ static void PACDDLExecuteFunction(ClientContext &context, TableFunctionInput &data, DataChunk &output) { // Nothing to output - DDL was already executed in bind - output.SetCardinality(0); + output.SetChildCardinality(0); } // ============================================================================ diff --git a/src/query_processing/pac_topk_rewriter.cpp b/src/query_processing/pac_topk_rewriter.cpp index 34c33cb4..bceeb63a 100644 --- a/src/query_processing/pac_topk_rewriter.cpp +++ b/src/query_processing/pac_topk_rewriter.cpp @@ -40,14 +40,14 @@ static void PacMeanFunction(DataChunk &args, ExpressionState &state, Vector &res idx_t count = args.size(); UnifiedVectorFormat list_data; - list_vec.ToUnifiedFormat(count, list_data); + list_vec.ToUnifiedFormat(list_data); auto result_data = FlatVector::GetDataMutable(result); auto &result_validity = FlatVector::ValidityMutable(result); - auto &child_vec = ListVector::GetEntry(list_vec); + auto &child_vec = ListVector::GetChild(list_vec); UnifiedVectorFormat child_data; - child_vec.ToUnifiedFormat(ListVector::GetListSize(list_vec), child_data); + child_vec.ToUnifiedFormat(child_data); auto child_values = UnifiedVectorFormat::GetData(child_data); auto list_entries = UnifiedVectorFormat::GetData(list_data); diff --git a/src/query_processing/privacy_expression_builder.cpp b/src/query_processing/privacy_expression_builder.cpp index 0cc3ec9c..e523d982 100644 --- a/src/query_processing/privacy_expression_builder.cpp +++ b/src/query_processing/privacy_expression_builder.cpp @@ -417,9 +417,10 @@ unique_ptr BindPacAggregate(OptimizerExtensionInput &input, const st arg_types.push_back(correction_expr->GetReturnType()); } - auto &entry = - Catalog::GetSystemCatalog(input.context) - .GetEntry(input.context, DEFAULT_SCHEMA, Identifier(pac_func_name)); + auto &entry = Catalog::GetSystemCatalog(input.context) + .GetEntry( + input.context, QualifiedName(Catalog::GetSystemCatalog(input.context).GetName(), + Identifier::DefaultSchema(), Identifier(pac_func_name))); auto best = function_binder.BindFunction(Identifier(entry.name), entry.functions, arg_types, error); if (!best.IsValid()) { throw InternalException("Privacy compiler: failed to bind " + pac_func_name); @@ -473,7 +474,9 @@ unique_ptr BindBitOrAggregate(OptimizerExtensionInput &input, unique ErrorData error; vector bit_or_types = {hash_expr->GetReturnType()}; auto &bit_or_entry = Catalog::GetSystemCatalog(input.context) - .GetEntry(input.context, DEFAULT_SCHEMA, "bit_or"); + .GetEntry( + input.context, QualifiedName(Catalog::GetSystemCatalog(input.context).GetName(), + Identifier::DefaultSchema(), "bit_or")); auto bit_or_best = function_binder.BindFunction(Identifier(bit_or_entry.name), bit_or_entry.functions, bit_or_types, error); if (!bit_or_best.IsValid()) { @@ -1201,7 +1204,9 @@ unique_ptr BindPlainAggregate(OptimizerExtensionInput &input, const arg_types.push_back(child->GetReturnType()); } auto &entry = Catalog::GetSystemCatalog(input.context) - .GetEntry(input.context, DEFAULT_SCHEMA, Identifier(func_name)); + .GetEntry( + input.context, QualifiedName(Catalog::GetSystemCatalog(input.context).GetName(), + Identifier::DefaultSchema(), Identifier(func_name))); auto best = function_binder.BindFunction(Identifier(entry.name), entry.functions, arg_types, error); if (!best.IsValid()) { throw InternalException("Privacy compiler: failed to bind aggregate '" + func_name + "'"); diff --git a/src/table_functions/dp_sass_stability_query.cpp b/src/table_functions/dp_sass_stability_query.cpp index b85d6486..a36911ce 100644 --- a/src/table_functions/dp_sass_stability_query.cpp +++ b/src/table_functions/dp_sass_stability_query.cpp @@ -330,15 +330,14 @@ static unique_ptr DpSassStabilityQueryInit(ClientConte static void DpSassStabilityQueryFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) { auto &bind_data = data_p.bind_data->Cast(); auto &state = data_p.global_state->Cast(); - idx_t count = 0; - while (state.offset < bind_data.rows.size() && count < STANDARD_VECTOR_SIZE) { + idx_t count = MinValue(bind_data.rows.size() - state.offset, STANDARD_VECTOR_SIZE); + output.SetChildCardinality(count); + for (idx_t row_idx = 0; row_idx < count; row_idx++) { auto &row = bind_data.rows[state.offset++]; for (idx_t col = 0; col < row.values.size(); col++) { - output.SetValue(col, count, row.values[col]); + output.data[col].SetValue(row_idx, row.values[col]); } - count++; } - output.SetCardinality(count); } } // namespace diff --git a/src/utils/privacy_helpers.cpp b/src/utils/privacy_helpers.cpp index 215170e5..803fbbfe 100644 --- a/src/utils/privacy_helpers.cpp +++ b/src/utils/privacy_helpers.cpp @@ -56,8 +56,8 @@ unique_ptr BindRankingWindow(ClientContext &context, Expr default: throw InternalException("Unsupported privacy ranking window"); } - auto &entry = - Catalog::GetSystemCatalog(context).GetEntry(context, DEFAULT_SCHEMA, name); + auto &entry = Catalog::GetSystemCatalog(context).GetEntry( + context, QualifiedName(Catalog::GetSystemCatalog(context).GetName(), Identifier::DefaultSchema(), name)); FunctionBinder binder(context); vector order_types; vector arg_order_types; From 86050becd416cefd3b0c2e2d2a6f4077d14a2e1d Mon Sep 17 00:00:00 2001 From: ila Date: Wed, 4 Nov 2026 18:26:45 +0100 Subject: [PATCH 06/32] Keep small SUM groups buffered and enable native state spilling --- CMakeLists.txt | 1 + docs/pac/internal_operators.md | 39 +++ src/aggregates/as_sum.cpp | 122 ++++++-- src/aggregates/as_sum_state_export.cpp | 268 ++++++++++++++++++ .../aggregates/as_sum_state_export.hpp | 9 + test/sql/pac_sum.test | 187 ++++++++++++ 6 files changed, 602 insertions(+), 24 deletions(-) create mode 100644 src/aggregates/as_sum_state_export.cpp create mode 100644 src/include/aggregates/as_sum_state_export.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fb29f785..f1509fd8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ set(EXTENSION_SOURCES src/aggregates/as_count.cpp src/aggregates/as_min_max.cpp src/aggregates/as_sum.cpp + src/aggregates/as_sum_state_export.cpp src/aggregates/as_finalize.cpp src/aggregates/as_clip_sum.cpp src/aggregates/as_clip_min_max.cpp diff --git a/docs/pac/internal_operators.md b/docs/pac/internal_operators.md index 196d7570..b029f990 100644 --- a/docs/pac/internal_operators.md +++ b/docs/pac/internal_operators.md @@ -175,6 +175,45 @@ No new aggregate functions, no new noise mechanisms, no plan decomposition. --- +## SUM state spilling (DuckDB 2.0) + +`as_sum` and `as_noised_sum` expose DuckDB's native state-export/import callbacks, +including the DECIMAL implementations selected during binding. DuckDB can export +arena-backed hash-aggregate states to spillable columns and import them when +combining partitions. Every aggregate in the hash table must expose a compatible +state layout for DuckDB to select this path. + +The exported BLOB contains buffered values and masks, update counts, positive and +negative integer cascade levels and overflow budgets, or floating-point counters. +It contains no addresses. Import reconstructs owned arena allocations and retains +the inline-level optimization. Export does not finalize, resample, or add noise; +the native update kernels and aggregate-state sizes remain unchanged. The binary +representation is internal to this implementation, rather than a portable state +interchange format. + +Small groups also retain their one/two-value buffers during combine. Finalization +materializes these buffers on copies using temporary storage backed by +`ArenaAllocator::GetUnderlyingAllocator()`, so the allocations are released after +the callback instead of accumulating in a query-long arena. The original buffered +state remains reusable. This addresses the Q78 path where buffers expand only in +final combine, after sink-phase spill decisions. + +Regression coverage extends `test/sql/pac_sum.test`: round trips include signed, +unsigned, floating-point and DECIMAL inputs, buffered/empty states, high cascade +levels, combining imported states, and truncated-input rejection. The pressure +test uses 800,000 rows in 100,000 contiguous groups, three SUMs and a 32 MB memory +limit; its complete result checksum must equal the in-memory result. + +```sh +build/release/test/unittest 'test/sql/pac_sum.test' +``` + +For the Q78 investigation, retain the complete query, sampling settings and thread +count when comparing builds. Record peak buffer memory, temporary storage and +result equivalence alongside latency. Passing the state-spill regression does not +establish that Q78 SF30 fits the target machine: joins and active aggregate +partitions still require working memory. + ## TODO - Categorical expressions (`pac_categorical` rewriter) diff --git a/src/aggregates/as_sum.cpp b/src/aggregates/as_sum.cpp index 37cee971..3c3c5747 100644 --- a/src/aggregates/as_sum.cpp +++ b/src/aggregates/as_sum.cpp @@ -1,5 +1,6 @@ #include "duckdb/common/vector/list_vector.hpp" #include "aggregates/as_sum.hpp" +#include "aggregates/as_sum_state_export.hpp" #include "categorical/pac_categorical.hpp" #include "duckdb/common/types/decimal.hpp" #include "duckdb/parser/parsed_data/create_aggregate_function_info.hpp" @@ -197,6 +198,48 @@ inline void PacSumFlushBuffer(PacSumDoubleStateWrapper &src, PacSumDoubleStateWr } } +template +static bool SumHasAllocatedState(const WrapperT &state) { + return state.GetState() +#ifndef PAC_SIGNEDSUM + || state.GetNegState() +#endif + ; +} + +template +static bool CombineSumBuffers(const WrapperT &src, WrapperT &dst) { + if (SumHasAllocatedState(src) || SumHasAllocatedState(dst)) { + return false; + } + auto src_count = src.n_buffered & WrapperT::BUF_MASK; + auto dst_count = dst.n_buffered & WrapperT::BUF_MASK; + if (src_count + dst_count > WrapperT::BUF_SIZE) { + return false; + } + for (idx_t i = 0; i < src_count; i++) { + dst.val_buf[dst_count + i] = src.val_buf[i]; + dst.hash_buf[dst_count + i] = src.hash_buf[i]; + } + dst.n_buffered = src_count + dst_count; + return true; +} + +template +static WrapperT *PrepareSumFinalization(WrapperT &source, WrapperT &temporary, ArenaAllocator &scratch, + ArenaAllocator *&allocator) { + auto *state = &source; + if (!SumHasAllocatedState(source)) { + // Do not leave a full state in the source arena for every one/two-row + // group. The source stays buffered and can safely be finalized again. + temporary = source; + state = &temporary; + allocator = &scratch; + } + PacSumFlushBuffer(*state, *state, *allocator); + return state; +} + // Buffering update for int wrappers template AUTOVECTORIZE inline void PacSumUpdateOne(WrapperT &agg, uint64_t key_hash, ValueT value, ArenaAllocator &a) { @@ -325,6 +368,10 @@ AUTOVECTORIZE static void PacSumCombineInt(Vector &src, Vector &dst, idx_t count for (idx_t i = 0; i < count; i++) { #ifndef PAC_NOBUFFERING + if (CombineSumBuffers(*src_wrapper[i], *dst_wrapper[i])) { + continue; + } + PacSumFlushBuffer>(*dst_wrapper[i], *dst_wrapper[i], allocator); // Flush src's buffer into dst's inner state (avoids allocating src inner) PacSumFlushBuffer>(*src_wrapper[i], *dst_wrapper[i], allocator); #endif @@ -408,6 +455,10 @@ AUTOVECTORIZE static void PacSumCombineDouble(Vector &src, Vector &dst, idx_t co auto dst_wrapper = FlatVector::GetData(dst); for (idx_t i = 0; i < count; i++) { #ifndef PAC_NOBUFFERING + if (CombineSumBuffers(*src_wrapper[i], *dst_wrapper[i])) { + continue; + } + PacSumFlushBuffer(*dst_wrapper[i], *dst_wrapper[i], allocator); // Flush src's buffer into dst's inner state (avoids allocating src inner) PacSumFlushBuffer(*src_wrapper[i], *dst_wrapper[i], allocator); #endif @@ -439,12 +490,18 @@ void PacSumFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &r *input.bind_data->Cast().random_state) : PacRandomEngine(0, 0); +#ifndef PAC_NOBUFFERING + ArenaAllocator scratch(input.allocator.GetUnderlyingAllocator()); +#endif for (idx_t i = 0; i < count; i++) { + auto *state = state_ptrs[i]; + auto *allocator = &input.allocator; #ifndef PAC_NOBUFFERING - PacSumFlushBuffer(*state_ptrs[i], *state_ptrs[i], input.allocator); + State temporary; + state = PrepareSumFinalization(*state, temporary, scratch, allocator); #endif PAC_FLOAT buf[64] = {0}; - auto *pos = state_ptrs[i]->GetState(); + auto *pos = state->GetState(); if (!pos) { result_mask.SetInvalid(offset + i); // return NULL (no values seen) continue; @@ -456,12 +513,12 @@ void PacSumFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &r } #ifndef PAC_SIGNEDSUM // Double-sided mode: use SFINAE helpers (cast to unsigned for int states only) - FlushPosState(pos, input.allocator); + FlushPosState(pos, *allocator); GetPosStateTotals(pos, buf); uint64_t update_count = pos->update_count; - update_count += SubtractNegStateTotals(state_ptrs[i], buf, key_hash, input.allocator); + update_count += SubtractNegStateTotals(state, buf, key_hash, *allocator); #else - pos->Flush(input.allocator); + pos->Flush(*allocator); pos->GetTotals(buf); uint64_t update_count = pos->update_count; #endif @@ -675,17 +732,23 @@ void PacSumDoubleInitialize(AggregateStateInput &, data_ptr_t *states, idx_t cou } } -// Helper to register both 2-param and 3-param (with optional mi) versions +// State export is cold-path metadata; the update kernels and native state sizes are unchanged. +static AggregateFunction SpillableSum(AggregateFunction function) { + RegisterSumStateExport(function, function.GetStateInitCallback() == PacSumDoubleInitialize); + return function; +} + +// Helper to register both 2-param and 3-param (with optional correction) versions static void AddFcn(AggregateFunctionSet &set, const LogicalType &value_type, const LogicalType &result_type, aggregate_size_t state_size, aggregate_initialize_t init, aggregate_update_t scatter, aggregate_combine_t combine, aggregate_finalize_t finalize, aggregate_cluster_update_t update, aggregate_destructor_t destructor = nullptr) { - set.AddFunction(AggregateFunction("as_noised_sum", {LogicalType::UBIGINT, value_type}, result_type, state_size, - init, scatter, combine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, - update, PacSumBind, destructor)); - set.AddFunction(AggregateFunction("as_noised_sum", {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, - result_type, state_size, init, scatter, combine, finalize, - FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacSumBind, destructor)); + set.AddFunction(SpillableSum(AggregateFunction( + "as_noised_sum", {LogicalType::UBIGINT, value_type}, result_type, state_size, init, scatter, combine, finalize, + FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacSumBind, destructor))); + set.AddFunction(SpillableSum(AggregateFunction( + "as_noised_sum", {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, result_type, state_size, init, + scatter, combine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacSumBind, destructor))); } // Helper to get the right AggregateFunction for as_sum given a physical type (for DECIMAL support) @@ -730,9 +793,14 @@ unique_ptr BindDecimalPacSum(BindAggregateFunctionInput &bind_inpu auto &function = bind_input.GetBoundFunction(); auto &args = bind_input.GetArguments(); auto decimal_type = args[1]->GetReturnType(); // value is arg 1 (arg 0 is hash) - function.ReplaceImplementation(GetPacSumAggregate(decimal_type.InternalType())); + function.ReplaceImplementation(SpillableSum(GetPacSumAggregate(decimal_type.InternalType()))); function.SetName("as_noised_sum"); function.GetArguments()[1] = decimal_type; + if (args.size() == 3) { + // Specialization uses a two-argument factory; retain correction in the + // exported call signature so FINALIZE/COMBINE can rebind it correctly. + function.GetArguments().push_back(LogicalType::DOUBLE); + } function.SetReturnType(LogicalType::DECIMAL(Decimal::MAX_WIDTH_DECIMAL, DecimalType::GetScale(decimal_type))); return PacBindWithScaleDivisor(ctx, args, 1.0); } @@ -840,11 +908,17 @@ static void PacSumAvgFinalizeCountersInternal(Vector &states, AggregateFinalizeI child_validity.SetAllValid(total_elements); } +#ifndef PAC_NOBUFFERING + ArenaAllocator scratch(input.allocator.GetUnderlyingAllocator()); +#endif for (idx_t i = 0; i < count; i++) { + auto *state = state_ptrs[i]; + auto *allocator = &input.allocator; #ifndef PAC_NOBUFFERING - PacSumFlushBuffer(*state_ptrs[i], *state_ptrs[i], input.allocator); + State temporary; + state = PrepareSumFinalization(*state, temporary, scratch, allocator); #endif - auto *s = state_ptrs[i]->GetState(); + auto *s = state->GetState(); // Set up the list entry - always needed even for NULL results idx_t base = (offset + i) * 64; @@ -862,13 +936,13 @@ static void PacSumAvgFinalizeCountersInternal(Vector &states, AggregateFinalizeI update_count = s->update_count; #ifndef PAC_SIGNEDSUM // Double-sided mode: use SFINAE helpers - FlushPosState(s, input.allocator); + FlushPosState(s, *allocator); GetPosStateTotals(s, buf); // For signed int states, subtract neg_state counters element-wise. The neg_state's worlds are OR-ed // into key_hash: a world that only received negative values must not be zeroed below. - update_count += SubtractNegStateTotals(state_ptrs[i], buf, key_hash, input.allocator); + update_count += SubtractNegStateTotals(state, buf, key_hash, *allocator); #else - s->Flush(input.allocator); + s->Flush(*allocator); s->GetTotals(buf); #endif } @@ -931,12 +1005,12 @@ static void AddCountersFcn(AggregateFunctionSet &set, const LogicalType &value_t aggregate_initialize_t init, aggregate_update_t scatter, aggregate_combine_t combine, aggregate_finalize_t finalize, aggregate_cluster_update_t update) { auto list_double_type = LogicalType::LIST(PacFloatLogicalType()); - set.AddFunction(AggregateFunction("as_sum", {LogicalType::UBIGINT, value_type}, list_double_type, state_size, init, - scatter, combine, finalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, update, - PacSumBind)); - set.AddFunction(AggregateFunction("as_sum", {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, - list_double_type, state_size, init, scatter, combine, finalize, - FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacSumBind)); + set.AddFunction(SpillableSum(AggregateFunction("as_sum", {LogicalType::UBIGINT, value_type}, list_double_type, + state_size, init, scatter, combine, finalize, + FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacSumBind))); + set.AddFunction(SpillableSum(AggregateFunction("as_sum", {LogicalType::UBIGINT, value_type, LogicalType::DOUBLE}, + list_double_type, state_size, init, scatter, combine, finalize, + FunctionNullHandling::DEFAULT_NULL_HANDLING, update, PacSumBind))); } void RegisterPacSumCountersFunctions(ExtensionLoader &loader) { diff --git a/src/aggregates/as_sum_state_export.cpp b/src/aggregates/as_sum_state_export.cpp new file mode 100644 index 00000000..3e060cfc --- /dev/null +++ b/src/aggregates/as_sum_state_export.cpp @@ -0,0 +1,268 @@ +#include "aggregates/as_sum_state_export.hpp" +#include "aggregates/as_sum.hpp" +#include "duckdb/common/vector/flat_vector.hpp" +#include "duckdb/common/vector/string_vector.hpp" + +namespace duckdb { +namespace { + +// Spill the native counters, not finalized totals: finalization can round and +// applies sampling correction. No process-local pointers enter the exported BLOB. +class SumSpillWriter { +public: + string data; + + template + void Write(const T &value) { + WriteData(&value, sizeof(value)); + } + + void WriteData(const void *ptr, idx_t size) { + data.append(reinterpret_cast(ptr), size); + } +}; + +class SumSpillReader { +public: + explicit SumSpillReader(const string_t &blob) : ptr(blob.GetData()), remaining(blob.GetSize()) { + } + + template + void Read(T &value) { + ReadData(&value, sizeof(value)); + } + + void ReadData(void *value, idx_t size) { + if (remaining < size) { + throw InvalidInputException("Truncated stochastic SUM state"); + } + memcpy(value, ptr, size); + ptr += size; + remaining -= size; + } + + bool ReadFlag() { + uint8_t flag; + Read(flag); + if (flag > 1) { + throw InvalidInputException("Invalid stochastic SUM state flag"); + } + return flag != 0; + } + + void Finish() { + if (remaining) { + throw InvalidInputException("Unexpected trailing stochastic SUM state data"); + } + } + +private: + const char *ptr; + idx_t remaining; +}; + +template +struct SumStateExport; + +template +struct SumStateExport> { + using State = PacSumIntState; + + template + static void WriteLevel(SumSpillWriter &out, const T *ptr) { + out.Write(uint8_t(ptr != nullptr)); + if (ptr) { + out.WriteData(ptr, sizeof(T) * N); + } + } + + template + static void ReadLevel(SumSpillReader &in, T *&ptr, ArenaAllocator &allocator) { + if (!in.ReadFlag()) { + return; + } + ptr = reinterpret_cast(allocator.Allocate(sizeof(T) * N)); + in.ReadData(ptr, sizeof(T) * N); + } + + static void Write(SumSpillWriter &out, const State &state) { + out.Write(state.key_hash); + out.Write(state.update_count); +#ifndef PAC_EXACTSUM + out.Write(state.max_level_used); + for (int k = 0; k <= state.max_level_used; k++) { + out.Write(state.exact_total[k]); + WriteLevel(out, state.u.levels[k]); + } +#elif defined(PAC_NOCASCADING) + out.Write(state.probabilistic_total128); +#else + out.Write(state.exact_total8); + out.Write(state.exact_total16); + out.Write(state.exact_total32); + out.Write(state.exact_total64); + WriteLevel(out, state.probabilistic_total8); + WriteLevel(out, state.probabilistic_total16); + WriteLevel(out, state.probabilistic_total32); + WriteLevel(out, state.probabilistic_total64); + WriteLevel(out, state.probabilistic_total128); +#endif + } + + static void Read(SumSpillReader &in, State &state, ArenaAllocator &allocator) { + memset(&state, 0, sizeof(state)); + in.Read(state.key_hash); + in.Read(state.update_count); +#ifndef PAC_EXACTSUM + in.Read(state.max_level_used); + state.inline_level_idx = -1; + if (state.max_level_used < -1 || state.max_level_used >= PAC_APPROX_NUM_LEVELS) { + throw InvalidInputException("Invalid stochastic SUM cascade level"); + } + for (int k = 0; k <= state.max_level_used; k++) { + in.Read(state.exact_total[k]); + if (in.ReadFlag()) { + // Retain the inline optimization when all pointer slots fit before it. + if (state.max_level_used < 9) { + state.AllocateLevel(allocator, k); + } else { + state.u.levels[k] = + reinterpret_cast(allocator.Allocate(sizeof(uint64_t) * PAC_APPROX_SWAR_ELEMENTS)); + } + in.ReadData(state.u.levels[k], sizeof(uint64_t) * PAC_APPROX_SWAR_ELEMENTS); + } + } +#elif defined(PAC_NOCASCADING) + in.Read(state.probabilistic_total128); +#else + in.Read(state.exact_total8); + in.Read(state.exact_total16); + in.Read(state.exact_total32); + in.Read(state.exact_total64); + ReadLevel(in, state.probabilistic_total8, allocator); + ReadLevel(in, state.probabilistic_total16, allocator); + ReadLevel(in, state.probabilistic_total32, allocator); + ReadLevel(in, state.probabilistic_total64, allocator); + ReadLevel(in, state.probabilistic_total128, allocator); +#endif + } +}; + +template <> +struct SumStateExport { + static void Write(SumSpillWriter &out, const PacSumDoubleState &state) { + out.Write(state.key_hash); + out.Write(state.update_count); + out.Write(state.probabilistic_total); + } + + static void Read(SumSpillReader &in, PacSumDoubleState &state, ArenaAllocator &) { + in.Read(state.key_hash); + in.Read(state.update_count); + in.Read(state.probabilistic_total); + } +}; + +#ifndef PAC_NOBUFFERING +template +struct SumStateExport> { + using State = PacSumStateWrapper; + + static void Write(SumSpillWriter &out, const State &state) { + auto buffered = state.n_buffered & State::BUF_MASK; + out.Write(uint8_t(buffered)); + for (idx_t i = 0; i < buffered; i++) { + out.Write(state.val_buf[i]); + out.Write(state.hash_buf[i]); + } + auto *inner = state.GetState(); + out.Write(uint8_t(inner != nullptr)); + if (inner) { + SumStateExport::Write(out, *inner); + } +#ifndef PAC_SIGNEDSUM + out.Write(uint8_t(state.neg_state != nullptr)); + if (state.neg_state) { + SumStateExport>::Write(out, *state.neg_state); + } +#endif + } + + static void Read(SumSpillReader &in, State &state, ArenaAllocator &allocator) { + memset(&state, 0, sizeof(state)); + uint8_t buffered; + in.Read(buffered); + if (buffered > State::BUF_SIZE) { + throw InvalidInputException("Invalid stochastic SUM buffer size"); + } + for (idx_t i = 0; i < buffered; i++) { + in.Read(state.val_buf[i]); + in.Read(state.hash_buf[i]); + } + if (in.ReadFlag()) { + auto *inner = state.EnsureState(allocator); + SumStateExport::Read(in, *inner, allocator); + } + state.n_buffered |= buffered; +#ifndef PAC_SIGNEDSUM + if (in.ReadFlag()) { + auto *negative = state.EnsureNegState(allocator); + SumStateExport>::Read(in, *negative, allocator); + } +#endif + } +}; +#endif + +template +struct SumSpillCallbacks { + static AggregateStateLayout Layout(AggregateLayoutInput &input) { + auto layout = AggregateStateLayout(LogicalType::BLOB, sizeof(STATE)); + if (input.function.GetLogicalArguments().size() == 3 && input.bind_data) { + layout.constant_parameters.emplace(2, Value::DOUBLE(input.bind_data->Cast().correction)); + } + return layout; + } + + static void Export(Vector &states, AggregateFinalizeInputData &, Vector &result, idx_t count, idx_t offset) { + UnifiedVectorFormat format; + states.ToUnifiedFormat(format); + auto addresses = UnifiedVectorFormat::GetData(format); + auto writer = FlatVector::Writer(result, count, offset); + for (idx_t i = 0; i < count; i++) { + SumSpillWriter out; + SumStateExport::Write(out, *addresses[format.sel->get_index(i)]); + writer.WriteValue(StringVector::AddStringOrBlob(result, out.data)); + } + } + + static void Import(AggregateImportInputData &input) { + auto values = input.input_vec.Values(); + for (idx_t i = 0; i < input.input_vec.size(); i++) { + auto value = values[i]; + if (!value.IsValid()) { + throw InvalidInputException("NULL stochastic SUM state"); + } + SumSpillReader in(value.GetValue()); + auto &state = *reinterpret_cast(input.dest_buffer + i * input.layout.total_state_size); + SumStateExport::Read(in, state, input.allocator); + in.Finish(); + } + } + + static void Register(AggregateFunction &function) { + function.SetStateExportCallbacks(Layout, Export, Import); + } +}; + +} // namespace + +void RegisterSumStateExport(AggregateFunction &function, bool double_state) { + if (double_state) { + SumSpillCallbacks::Register(function); + } else { + SumSpillCallbacks>::Register(function); + } +} + +} // namespace duckdb diff --git a/src/include/aggregates/as_sum_state_export.hpp b/src/include/aggregates/as_sum_state_export.hpp new file mode 100644 index 00000000..67a0968f --- /dev/null +++ b/src/include/aggregates/as_sum_state_export.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include "duckdb/function/aggregate_function.hpp" + +namespace duckdb { + +void RegisterSumStateExport(AggregateFunction &function, bool double_state); + +} // namespace duckdb diff --git a/test/sql/pac_sum.test b/test/sql/pac_sum.test index 8b857ae0..80734dba 100644 --- a/test/sql/pac_sum.test +++ b/test/sql/pac_sum.test @@ -374,3 +374,190 @@ a AS (SELECT as_sum(mask, v) AS lanes FROM t) SELECT count(*) FILTER (WHERE lanes[j+1] / 2 = 196605) FROM a CROSS JOIN range(64) worlds(j) ---- 64 + + +# DuckDB 2.0 state export preserves buffers, positive/negative cascades, and +# pointer-backed counters. Values include a level above the inline-pointer area. +statement ok +SET priv_rewrite=false; + +statement ok +SET threads=1; + +statement ok +SET pac_mi=0; + +statement ok +SET pac_sample_diversity_check=false; + +statement ok +CREATE TABLE spill_sum_input AS +SELECT i%5 g,priv_hash(hash(i)) h, + CASE WHEN i%3=0 THEN -(1::BIGINT<<48) ELSE 1024::BIGINT END v +FROM range(10000)t(i); + +statement ok +CREATE TABLE spill_sum_states AS +SELECT g,as_sum(h,v) EXPORT_STATE s, + as_sum(h,v::DOUBLE) EXPORT_STATE d, + as_noised_sum(h,v::DECIMAL(18,2)) EXPORT_STATE dec, + as_sum(h,abs(v)::UBIGINT,0.75) EXPORT_STATE u +FROM spill_sum_input GROUP BY g; + +query I +SELECT count(*) FROM spill_sum_states a JOIN ( + SELECT g,as_sum(h,v) s,as_sum(h,v::DOUBLE) d, + as_noised_sum(h,v::DECIMAL(18,2)) dec, + as_sum(h,abs(v)::UBIGINT,0.75) u + FROM spill_sum_input GROUP BY g +)b USING(g) +WHERE finalize(a.s) IS DISTINCT FROM b.s OR finalize(a.d) IS DISTINCT FROM b.d + OR finalize(a.dec) IS DISTINCT FROM b.dec OR finalize(a.u) IS DISTINCT FROM b.u; +---- +0 + +# One or two values stay buffered; NULL-only/empty groups have no inner state. +query I +SELECT finalize(s) IS NOT DISTINCT FROM expected FROM ( + SELECT as_sum(h,NULL::BIGINT) EXPORT_STATE s,as_sum(h,NULL::BIGINT) expected + FROM spill_sum_input +); +---- +true + +query I +SELECT finalize(s) IS NOT DISTINCT FROM expected FROM ( + SELECT as_sum(h,v) EXPORT_STATE s,as_sum(h,v) expected + FROM spill_sum_input WHERE false +); +---- +true + +query I +SELECT finalize(s) IS NOT DISTINCT FROM expected FROM ( + SELECT as_sum(h,v) EXPORT_STATE s,as_sum(h,v) expected + FROM (SELECT h,v FROM spill_sum_input LIMIT 1) +); +---- +true + +query I +SELECT finalize(s) IS NOT DISTINCT FROM expected FROM ( + SELECT as_sum(h,v) EXPORT_STATE s,as_sum(h,v) expected + FROM (SELECT h,v FROM spill_sum_input LIMIT 2) +); +---- +true + +# Compare the complete result under arena pressure with an in-memory execution. +# Mixed integer, floating-point, and DECIMAL states share the same hash table. +statement ok +SET memory_limit='256MB'; + +statement ok +CREATE TABLE spill_sum_expected AS +SELECT count(*) n,sum(hash(a,b,c)) checksum FROM ( + SELECT i//8 g, + as_sum(18446744073709551615::UBIGINT,1::BIGINT) a, + as_sum(18446744073709551615::UBIGINT,2::DOUBLE) b, + as_noised_sum(18446744073709551615::UBIGINT,3::DECIMAL(18,2)) c + FROM range(800000)t(i) GROUP BY g +); + +statement ok +SET debug_force_external=true; + +statement ok +SET memory_limit='32MB'; + +query II +SELECT actual.n,actual.checksum=expected.checksum FROM ( + SELECT count(*) n,sum(hash(a,b,c)) checksum FROM ( + SELECT i//8 g, + as_sum(18446744073709551615::UBIGINT,1::BIGINT) a, + as_sum(18446744073709551615::UBIGINT,2::DOUBLE) b, + as_noised_sum(18446744073709551615::UBIGINT,3::DECIMAL(18,2)) c + FROM range(800000)t(i) GROUP BY g + ) +)actual,spill_sum_expected expected; +---- +100000 true + +statement ok +SET debug_force_external=false; + +statement ok +RESET memory_limit; + + +# Exported states remain combinable, including the high and negative levels. +query I +SELECT finalize(combine(a.s,b.s)) IS NOT DISTINCT FROM c.expected FROM +(SELECT as_sum(h,v) EXPORT_STATE s FROM spill_sum_input WHERE g<3)a, +(SELECT as_sum(h,v) EXPORT_STATE s FROM spill_sum_input WHERE g>=3)b, +(SELECT as_sum(h,v) expected FROM spill_sum_input)c; +---- +true + +statement error +SELECT finalize(to_aggregate_state(''::BLOB,'as_sum',['UBIGINT','BIGINT'])); +---- +Truncated stochastic SUM state + + +# DECIMAL binding replaces its implementation; retain the logical third +# argument (sampling correction) when rebinding exported states. +query I +SELECT finalize(s) IS NOT DISTINCT FROM expected FROM ( + SELECT as_noised_sum(h,v::DECIMAL(18,2),0.75) EXPORT_STATE s, + as_noised_sum(h,v::DECIMAL(18,2),0.75) expected + FROM spill_sum_input +); +---- +true + + +# Q78-shaped failure: sparse local groups stay buffered in the sink. Combining +# them must not manufacture a permanent full state for each final group. +statement ok +SET debug_force_external=true; + +statement ok +SET memory_limit='32MB'; + +query II +SELECT count(*),count(*) FILTER (WHERE a=4 AND b=8 AND c=12) FROM ( + SELECT i%100000 g, + as_noised_sum(18446744073709551615::UBIGINT,1::BIGINT) a, + as_noised_sum(18446744073709551615::UBIGINT,2::DOUBLE) b, + as_noised_sum(18446744073709551615::UBIGINT,3::DECIMAL(18,2)) c + FROM range(200000)t(i) GROUP BY g +); +---- +100000 100000 + +statement ok +SET debug_force_external=false; + +statement ok +RESET memory_limit; + +# The two-value buffer boundary: combine two buffered states, or a buffered +# destination and a materialized source. Every world must retain every value. +query I +SELECT count(*) FROM ( + SELECT finalize(combine(a.s,b.s)) lanes FROM + (SELECT as_sum(18446744073709551615::UBIGINT,v) EXPORT_STATE s FROM (VALUES (13::BIGINT),(17))t(v))a, + (SELECT as_sum(18446744073709551615::UBIGINT,29::BIGINT) EXPORT_STATE s)b +),unnest(lanes)t(v) WHERE v=118; +---- +64 + +query I +SELECT count(*) FROM ( + SELECT finalize(combine(a.s,b.s)) lanes FROM + (SELECT as_sum(18446744073709551615::UBIGINT,v) EXPORT_STATE s FROM (VALUES (1::BIGINT),(2))t(v))a, + (SELECT as_sum(18446744073709551615::UBIGINT,v) EXPORT_STATE s FROM (VALUES (3::BIGINT),(4),(5))t(v))b +),unnest(lanes)t(v) WHERE v=30; +---- +64 From fa150705f08be298aa694b8873d24eccfbd142ff Mon Sep 17 00:00:00 2001 From: ila Date: Wed, 4 Nov 2026 19:16:22 +0100 Subject: [PATCH 07/32] Document stochastic SUM spill limitations --- docs/pac/internal_operators.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/pac/internal_operators.md b/docs/pac/internal_operators.md index b029f990..28607229 100644 --- a/docs/pac/internal_operators.md +++ b/docs/pac/internal_operators.md @@ -177,12 +177,18 @@ No new aggregate functions, no new noise mechanisms, no plan decomposition. ## SUM state spilling (DuckDB 2.0) -`as_sum` and `as_noised_sum` expose DuckDB's native state-export/import callbacks, -including the DECIMAL implementations selected during binding. DuckDB can export +The membership-mask-plus-scalar overloads of `as_sum` and `as_noised_sum` expose +DuckDB's native state-export/import callbacks, including the DECIMAL +implementations selected during binding. DuckDB can export arena-backed hash-aggregate states to spillable columns and import them when combining partitions. Every aggregate in the hash table must expose a compatible state layout for DuckDB to select this path. +COUNT, MIN/MAX, AVG's COUNT component, and the list-input SUM overload do not yet +provide these callbacks. Combining one of them with a spillable SUM in the same +hash aggregate disables arena-state spilling for that table. These queries can +still run in memory, but may exhaust a low memory limit even when SUM alone spills. + The exported BLOB contains buffered values and masks, update counts, positive and negative integer cascade levels and overflow budgets, or floating-point counters. It contains no addresses. Import reconstructs owned arena allocations and retains From 3e6723d28c778d85ff2232d6fb2b30571b025ab4 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 5 Nov 2026 09:39:35 +0100 Subject: [PATCH 08/32] Fix nested AVG and copied CTEs and spill mixed stochastic aggregates --- .../workflows/MainDistributionPipeline.yml | 1 + docs/pac/internal_operators.md | 24 ++-- src/aggregates/as_count.cpp | 118 +++++++++++++++--- src/aggregates/as_min_max.cpp | 60 +++++++++ src/aggregates/as_sum_state_export.cpp | 115 ++--------------- src/categorical/pac_categorical.cpp | 28 ++++- src/compiler/privacy_compiler.cpp | 2 +- src/core/privacy_extension.cpp | 7 +- .../aggregates/aggregate_state_export.hpp | 106 ++++++++++++++++ src/query_processing/pac_avg_rewriter.cpp | 3 + src/utils/privacy_helpers.cpp | 7 ++ test/sql/pac_avg.test | 41 ++++++ test/sql/pac_categorical_subqueries.test | 59 +++++++++ test/sql/pac_sum.test | 70 ++++++++++- 14 files changed, 498 insertions(+), 143 deletions(-) create mode 100644 src/include/aggregates/aggregate_state_export.hpp diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 7b01c057..f7485d1e 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,6 +19,7 @@ jobs: duckdb_version: v2.0.0 ci_tools_version: ebdcc1961780b0ce1272a0d7637dc58447c34885 extension_name: privacy + test_config: '{"test_env_variables": {"T": "\"test/*\" --test-flags=--output-sql=true"}}' code-quality-check: name: Code Quality Check diff --git a/docs/pac/internal_operators.md b/docs/pac/internal_operators.md index 28607229..78736e1a 100644 --- a/docs/pac/internal_operators.md +++ b/docs/pac/internal_operators.md @@ -175,22 +175,20 @@ No new aggregate functions, no new noise mechanisms, no plan decomposition. --- -## SUM state spilling (DuckDB 2.0) +## Stochastic aggregate state spilling (DuckDB 2.0) -The membership-mask-plus-scalar overloads of `as_sum` and `as_noised_sum` expose -DuckDB's native state-export/import callbacks, including the DECIMAL -implementations selected during binding. DuckDB can export -arena-backed hash-aggregate states to spillable columns and import them when -combining partitions. Every aggregate in the hash table must expose a compatible -state layout for DuckDB to select this path. - -COUNT, MIN/MAX, AVG's COUNT component, and the list-input SUM overload do not yet -provide these callbacks. Combining one of them with a spillable SUM in the same -hash aggregate disables arena-state spilling for that table. These queries can -still run in memory, but may exhaust a low memory limit even when SUM alone spills. +The membership-mask-plus-scalar SUM, COUNT, and MIN/MAX implementations expose +DuckDB's native state-export/import callbacks, including DECIMAL specializations. +List-input SUM, COUNT, AVG, and MIN/MAX also provide callbacks. AVG over scalar +inputs decomposes into the spillable SUM and COUNT implementations before column +pruning. DuckDB can export arena-backed hash-aggregate states to spillable columns +and import them when combining partitions. Every aggregate in a hash table must +expose a compatible state layout for DuckDB to select this path. The exported BLOB contains buffered values and masks, update counts, positive and -negative integer cascade levels and overflow budgets, or floating-point counters. +negative integer cascade levels and overflow budgets, floating-point counters, +COUNT SWAR bytes and overflow totals, MIN/MAX bounds, and list-aggregate values +and counts. Folded correction arguments are retained when rebinding exported states. It contains no addresses. Import reconstructs owned arena allocations and retains the inline-level optimization. Export does not finalize, resample, or add noise; the native update kernels and aggregate-state sizes remain unchanged. The binary diff --git a/src/aggregates/as_count.cpp b/src/aggregates/as_count.cpp index b0b15711..dc138b3a 100644 --- a/src/aggregates/as_count.cpp +++ b/src/aggregates/as_count.cpp @@ -1,6 +1,7 @@ #include "duckdb/common/vector/list_vector.hpp" #include #include "aggregates/as_count.hpp" +#include "aggregates/aggregate_state_export.hpp" #include "categorical/pac_categorical.hpp" #include "duckdb/common/types/value_map.hpp" #include "duckdb/parser/parsed_data/create_aggregate_function_info.hpp" @@ -31,6 +32,72 @@ using ScatterState = PacCountState; using ScatterState = PacCountStateWrapper; #endif +// Preserve SWAR bytes and overflow totals separately: exporting must not flush or +// mutate a live aggregate, and the totals pointer is local to the process. +struct CountStateCodec { + static void WriteInner(AggregateStateWriter &out, const PacCountState &state) { +#ifndef PAC_NOCASCADING + out.Write(state.probabilistic_total8); + out.Write(state.swar_fill); +#endif + out.Write(state.key_hash); + out.Write(state.update_count); + out.Write(uint8_t(state.probabilistic_total != nullptr)); + if (state.probabilistic_total) { + out.WriteData(state.probabilistic_total, 64 * sizeof(uint64_t)); + } + } + static void ReadInner(AggregateStateReader &in, PacCountState &state, ArenaAllocator &allocator) { +#ifndef PAC_NOCASCADING + in.Read(state.probabilistic_total8); + in.Read(state.swar_fill); + if (state.swar_fill >= 255) { + throw InvalidInputException("Invalid stochastic COUNT SWAR fill"); + } +#endif + in.Read(state.key_hash); + in.Read(state.update_count); + if (in.ReadFlag()) { + in.ReadData(state.EnsureTotals(allocator), 64 * sizeof(uint64_t)); + } + } + static void Write(AggregateStateWriter &out, const ScatterState &state) { +#if defined(PAC_NOBUFFERING) || defined(PAC_NOCASCADING) + WriteInner(out, state); +#else + auto buffered = state.n_buffered & 7; + out.Write(uint8_t(buffered)); + for (idx_t i = 0; i < buffered; i++) { + out.Write(state.hash_buf[i]); + } + auto *inner = state.GetState(); + out.Write(uint8_t(inner != nullptr)); + if (inner) { + WriteInner(out, *inner); + } +#endif + } + static void Read(AggregateStateReader &in, ScatterState &state, ArenaAllocator &allocator) { + memset(&state, 0, sizeof(state)); +#if defined(PAC_NOBUFFERING) || defined(PAC_NOCASCADING) + ReadInner(in, state, allocator); +#else + uint8_t buffered; + in.Read(buffered); + if (buffered > 3) { + throw InvalidInputException("Invalid stochastic COUNT buffer size"); + } + for (idx_t i = 0; i < buffered; i++) { + in.Read(state.hash_buf[i]); + } + if (in.ReadFlag()) { + ReadInner(in, *state.EnsureState(allocator), allocator); + } + state.n_buffered |= buffered; +#endif + } +}; + // State functions - uses ScatterState for both simple and scatter static idx_t PacCountStateSize(AggregateStateInput &) { return sizeof(ScatterState); @@ -43,6 +110,26 @@ static void PacCountInitialize(AggregateStateInput &, data_ptr_t *states, idx_t } } +static AggregateStateLayout CountStateLayout(AggregateLayoutInput &input) { + auto layout = AggregateStateExport::Layout(input); + // COUNT's ANY argument affects updates only. Rebinding a finalized state + // needs its folded correction, including the two-argument shorthand. + if (input.function.GetLogicalArguments().size() == 2 && input.bind_data) { + layout.constant_parameters.emplace(1, Value::DOUBLE(input.bind_data->Cast().correction)); + } + return layout; +} + +static void RegisterCountFunctions(ExtensionLoader &loader, CreateAggregateFunctionInfo &info) { + info.functions.ApplyToFunctions([](AggregateFunction &function) { + if (function.GetStateInitCallback() == PacCountInitialize) { + AggregateStateExport::Register(function); + function.GetCallbacks().get_state_type = CountStateLayout; + } + }); + loader.RegisterFunction(std::move(info)); +} + void PacCountUpdate(Vector inputs[], AggregateInputData &aggr, idx_t input_count, const ClusteredAggr &clusters, idx_t count) { ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_ptr, idx_t count) { @@ -366,7 +453,7 @@ void RegisterPacCountFunctions(ExtensionLoader &loader) { desc.examples = { "SELECT c_mktsegment, as_noised_count(priv_hash(hash(c_custkey))) FROM customer GROUP BY c_mktsegment"}; info.descriptions.push_back(std::move(desc)); - loader.RegisterFunction(std::move(info)); + RegisterCountFunctions(loader, info); } // ============================================================================ @@ -469,7 +556,7 @@ void RegisterDpSampleCountFunctions(ExtensionLoader &loader) { FunctionDescription desc; desc.description = "[INTERNAL] Returns 64 sample-median DP counters for COUNT-like values."; info.descriptions.push_back(std::move(desc)); - loader.RegisterFunction(std::move(info)); + RegisterCountFunctions(loader, info); AggregateFunctionSet mask_set("as_sample_count_mask"); mask_set.AddFunction(AggregateFunction( @@ -480,7 +567,7 @@ void RegisterDpSampleCountFunctions(ExtensionLoader &loader) { FunctionDescription mask_desc; mask_desc.description = "[INTERNAL] Returns 64 sample-median DP counters from precomputed sample masks."; mask_info.descriptions.push_back(std::move(mask_desc)); - loader.RegisterFunction(std::move(mask_info)); + RegisterCountFunctions(loader, mask_info); } // Shared storage for variable-m count aggregates. It lazily allocates one counter per @@ -786,7 +873,7 @@ void RegisterDpSampleMCountFunctions(ExtensionLoader &loader) { FunctionDescription desc; desc.description = "[INTERNAL] Experimental variable-m SAA counters for COUNT-like values."; info.descriptions.push_back(std::move(desc)); - loader.RegisterFunction(std::move(info)); + RegisterCountFunctions(loader, info); } void RegisterDpSampleMCountDistinctFunctions(ExtensionLoader &loader) { @@ -801,7 +888,7 @@ void RegisterDpSampleMCountDistinctFunctions(ExtensionLoader &loader) { desc.description = "[INTERNAL] Experimental variable-m SAA lane counters for pre-deduplicated COUNT(DISTINCT) values."; info.descriptions.push_back(std::move(desc)); - loader.RegisterFunction(std::move(info)); + RegisterCountFunctions(loader, info); AggregateFunctionSet if_set("as_sample_m_count_distinct_if"); if_set.AddFunction(AggregateFunction("as_sample_m_count_distinct_if", {LogicalType::UBIGINT, LogicalType::BOOLEAN}, @@ -815,7 +902,7 @@ void RegisterDpSampleMCountDistinctFunctions(ExtensionLoader &loader) { "[INTERNAL] Experimental variable-m SAA lane counters for conditionally pre-deduplicated COUNT(DISTINCT) " "values."; if_info.descriptions.push_back(std::move(if_desc)); - loader.RegisterFunction(std::move(if_info)); + RegisterCountFunctions(loader, if_info); } // Mixed aggregate queries cannot use the dedicated three-stage DISTINCT plan. Their @@ -1003,7 +1090,7 @@ void RegisterDpSampleMCountDistinctValuesFunctions(ExtensionLoader &loader) { FunctionDescription desc; desc.description = "[INTERNAL] Experimental variable-m SAA lane counters for per-PU COUNT(DISTINCT) value lists."; info.descriptions.push_back(std::move(desc)); - loader.RegisterFunction(std::move(info)); + RegisterCountFunctions(loader, info); } void RegisterDpSampleMCountIfFunctions(ExtensionLoader &loader) { @@ -1018,7 +1105,7 @@ void RegisterDpSampleMCountIfFunctions(ExtensionLoader &loader) { FunctionDescription desc; desc.description = "[INTERNAL] Experimental variable-m SAA counters for lane-selected COUNT-like values."; info.descriptions.push_back(std::move(desc)); - loader.RegisterFunction(std::move(info)); + RegisterCountFunctions(loader, info); } static void PacSampleMCountUpdateOne(DpSampleMCountState &state, ArenaAllocator &allocator, uint64_t hash, @@ -1204,7 +1291,7 @@ void RegisterPacSampleMCountFunctions(ExtensionLoader &loader) { FunctionDescription desc; desc.description = "[INTERNAL] Variable-m PAC counters with 50% world membership."; info.descriptions.push_back(std::move(desc)); - loader.RegisterFunction(std::move(info)); + RegisterCountFunctions(loader, info); } void RegisterPacSampleMCountIfFunctions(ExtensionLoader &loader) { @@ -1219,7 +1306,7 @@ void RegisterPacSampleMCountIfFunctions(ExtensionLoader &loader) { FunctionDescription desc; desc.description = "[INTERNAL] Variable-m PAC counters with per-lane predicates."; info.descriptions.push_back(std::move(desc)); - loader.RegisterFunction(std::move(info)); + RegisterCountFunctions(loader, info); } // ============================================================================ @@ -1246,7 +1333,7 @@ void RegisterPacCountCountersFunctions(ExtensionLoader &loader) { FunctionDescription counters_desc; counters_desc.description = "[INTERNAL] Returns 64 PAC subsample counters as LIST for categorical queries."; counters_info.descriptions.push_back(std::move(counters_desc)); - loader.RegisterFunction(std::move(counters_info)); + RegisterCountFunctions(loader, counters_info); } // ============================================================================ @@ -1271,7 +1358,7 @@ void RegisterPacAvgFunctions(ExtensionLoader &loader) { avg_desc.examples = {"SELECT c_mktsegment, as_noised_avg(priv_hash(hash(c_custkey)), c_acctbal) FROM customer " "GROUP BY c_mktsegment"}; avg_info.descriptions.push_back(std::move(avg_desc)); - loader.RegisterFunction(std::move(avg_info)); + RegisterCountFunctions(loader, avg_info); // as_avg(UBIGINT, value_type[, correction]) → LIST — same implementation as as_count(UBIGINT, ANY) auto list_float_type = LogicalType::LIST(PacFloatLogicalType()); @@ -1284,11 +1371,12 @@ void RegisterPacAvgFunctions(ExtensionLoader &loader) { "as_avg", {LogicalType::UBIGINT, LogicalType::ANY, LogicalType::DOUBLE}, list_float_type, PacCountStateSize, PacCountInitialize, PacCountColumnScatterUpdate, PacCountCombine, PacCountFinalizeCounters, FunctionNullHandling::DEFAULT_NULL_HANDLING, PacCountColumnUpdate, PacCountBind)); + AddPacListAggregateOverload(counters_set, "avg"); CreateAggregateFunctionInfo avg_counters_info(counters_set); FunctionDescription avg_counters_desc; avg_counters_desc.description = "[INTERNAL] Returns 64 PAC subsample counters as LIST for categorical queries."; avg_counters_info.descriptions.push_back(std::move(avg_counters_desc)); - loader.RegisterFunction(std::move(avg_counters_info)); + RegisterCountFunctions(loader, avg_counters_info); } // ============================================================================ @@ -1316,7 +1404,7 @@ void RegisterPacNoisedClipCountFunctions(ExtensionLoader &loader) { FunctionNullHandling::SPECIAL_HANDLING, PacCountColumnUpdate, PacCountBind)); CreateAggregateFunctionInfo info(fcn_set); - loader.RegisterFunction(std::move(info)); + RegisterCountFunctions(loader, info); } void RegisterPacClipCountFunctions(ExtensionLoader &loader) { @@ -1334,7 +1422,7 @@ void RegisterPacClipCountFunctions(ExtensionLoader &loader) { AddPacListAggregateOverload(fcn_set, "clip_count"); CreateAggregateFunctionInfo info(fcn_set); - loader.RegisterFunction(std::move(info)); + RegisterCountFunctions(loader, info); } } // namespace duckdb diff --git a/src/aggregates/as_min_max.cpp b/src/aggregates/as_min_max.cpp index 818a7855..6a4a29e4 100644 --- a/src/aggregates/as_min_max.cpp +++ b/src/aggregates/as_min_max.cpp @@ -1,5 +1,6 @@ #include "duckdb/common/vector/list_vector.hpp" #include "aggregates/as_min_max.hpp" +#include "aggregates/aggregate_state_export.hpp" #include "categorical/pac_categorical.hpp" #include "duckdb/parser/parsed_data/create_aggregate_function_info.hpp" @@ -18,6 +19,63 @@ template using MinMaxState = PacMinMaxStateWrapper; #endif +template +struct MinMaxStateCodec { + using State = MinMaxState; + using Inner = PacMinMaxState; + static void WriteInner(AggregateStateWriter &out, const Inner &state) { + out.Write(uint8_t(state.initialized)); + out.Write(state.update_count); + out.Write(state.key_hash); + out.Write(state.global_bound); + out.Write(state.extremes); + } + static void ReadInner(AggregateStateReader &in, Inner &state) { + state.initialized = in.ReadFlag(); + in.Read(state.update_count); + in.Read(state.key_hash); + in.Read(state.global_bound); + in.Read(state.extremes); + } + static void Write(AggregateStateWriter &out, const State &state) { +#ifdef PAC_NOBUFFERING + WriteInner(out, state); +#else + auto buffered = state.n_buffered & State::BUF_MASK; + out.Write(uint8_t(buffered)); + for (idx_t i = 0; i < buffered; i++) { + out.Write(state.val_buf[i]); + out.Write(state.hash_buf[i]); + } + auto *inner = state.GetState(); + out.Write(uint8_t(inner != nullptr)); + if (inner) { + WriteInner(out, *inner); + } +#endif + } + static void Read(AggregateStateReader &in, State &state, ArenaAllocator &allocator) { + memset(&state, 0, sizeof(state)); +#ifdef PAC_NOBUFFERING + ReadInner(in, state); +#else + uint8_t buffered; + in.Read(buffered); + if (buffered > State::BUF_SIZE) { + throw InvalidInputException("Invalid stochastic MIN/MAX buffer size"); + } + for (idx_t i = 0; i < buffered; i++) { + in.Read(state.val_buf[i]); + in.Read(state.hash_buf[i]); + } + if (in.ReadFlag()) { + ReadInner(in, *state.EnsureState(allocator)); + } + state.n_buffered |= buffered; +#endif + } +}; + // ============================================================================ // Update functions // ============================================================================ @@ -187,6 +245,7 @@ static unique_ptr PacMinMaxBind(BindAggregateFunctionInput &bind_i function.SetStateClusterUpdateCallback(PacMinMaxUpdate); \ function.SetStateCombineCallback(PacMinMaxCombine); \ function.SetStateFinalizeCallback(PacMinMaxFinalize); \ + AggregateStateExport, MinMaxStateCodec>::Register(function); \ break switch (physical_type) { @@ -293,6 +352,7 @@ static unique_ptr PacMinMaxCountersBind(BindAggregateFunctionInput function.SetStateClusterUpdateCallback(PacMinMaxUpdate); \ function.SetStateCombineCallback(PacMinMaxCombine); \ function.SetStateFinalizeCallback(PacMinMaxFinalizeCounters); \ + AggregateStateExport, MinMaxStateCodec>::Register(function); \ break switch (physical_type) { diff --git a/src/aggregates/as_sum_state_export.cpp b/src/aggregates/as_sum_state_export.cpp index 3e060cfc..97a479eb 100644 --- a/src/aggregates/as_sum_state_export.cpp +++ b/src/aggregates/as_sum_state_export.cpp @@ -1,5 +1,6 @@ #include "aggregates/as_sum_state_export.hpp" #include "aggregates/as_sum.hpp" +#include "aggregates/aggregate_state_export.hpp" #include "duckdb/common/vector/flat_vector.hpp" #include "duckdb/common/vector/string_vector.hpp" @@ -8,59 +9,6 @@ namespace { // Spill the native counters, not finalized totals: finalization can round and // applies sampling correction. No process-local pointers enter the exported BLOB. -class SumSpillWriter { -public: - string data; - - template - void Write(const T &value) { - WriteData(&value, sizeof(value)); - } - - void WriteData(const void *ptr, idx_t size) { - data.append(reinterpret_cast(ptr), size); - } -}; - -class SumSpillReader { -public: - explicit SumSpillReader(const string_t &blob) : ptr(blob.GetData()), remaining(blob.GetSize()) { - } - - template - void Read(T &value) { - ReadData(&value, sizeof(value)); - } - - void ReadData(void *value, idx_t size) { - if (remaining < size) { - throw InvalidInputException("Truncated stochastic SUM state"); - } - memcpy(value, ptr, size); - ptr += size; - remaining -= size; - } - - bool ReadFlag() { - uint8_t flag; - Read(flag); - if (flag > 1) { - throw InvalidInputException("Invalid stochastic SUM state flag"); - } - return flag != 0; - } - - void Finish() { - if (remaining) { - throw InvalidInputException("Unexpected trailing stochastic SUM state data"); - } - } - -private: - const char *ptr; - idx_t remaining; -}; - template struct SumStateExport; @@ -69,7 +17,7 @@ struct SumStateExport> { using State = PacSumIntState; template - static void WriteLevel(SumSpillWriter &out, const T *ptr) { + static void WriteLevel(AggregateStateWriter &out, const T *ptr) { out.Write(uint8_t(ptr != nullptr)); if (ptr) { out.WriteData(ptr, sizeof(T) * N); @@ -77,7 +25,7 @@ struct SumStateExport> { } template - static void ReadLevel(SumSpillReader &in, T *&ptr, ArenaAllocator &allocator) { + static void ReadLevel(AggregateStateReader &in, T *&ptr, ArenaAllocator &allocator) { if (!in.ReadFlag()) { return; } @@ -85,7 +33,7 @@ struct SumStateExport> { in.ReadData(ptr, sizeof(T) * N); } - static void Write(SumSpillWriter &out, const State &state) { + static void Write(AggregateStateWriter &out, const State &state) { out.Write(state.key_hash); out.Write(state.update_count); #ifndef PAC_EXACTSUM @@ -109,7 +57,7 @@ struct SumStateExport> { #endif } - static void Read(SumSpillReader &in, State &state, ArenaAllocator &allocator) { + static void Read(AggregateStateReader &in, State &state, ArenaAllocator &allocator) { memset(&state, 0, sizeof(state)); in.Read(state.key_hash); in.Read(state.update_count); @@ -150,13 +98,13 @@ struct SumStateExport> { template <> struct SumStateExport { - static void Write(SumSpillWriter &out, const PacSumDoubleState &state) { + static void Write(AggregateStateWriter &out, const PacSumDoubleState &state) { out.Write(state.key_hash); out.Write(state.update_count); out.Write(state.probabilistic_total); } - static void Read(SumSpillReader &in, PacSumDoubleState &state, ArenaAllocator &) { + static void Read(AggregateStateReader &in, PacSumDoubleState &state, ArenaAllocator &) { in.Read(state.key_hash); in.Read(state.update_count); in.Read(state.probabilistic_total); @@ -168,7 +116,7 @@ template struct SumStateExport> { using State = PacSumStateWrapper; - static void Write(SumSpillWriter &out, const State &state) { + static void Write(AggregateStateWriter &out, const State &state) { auto buffered = state.n_buffered & State::BUF_MASK; out.Write(uint8_t(buffered)); for (idx_t i = 0; i < buffered; i++) { @@ -188,7 +136,7 @@ struct SumStateExport> { #endif } - static void Read(SumSpillReader &in, State &state, ArenaAllocator &allocator) { + static void Read(AggregateStateReader &in, State &state, ArenaAllocator &allocator) { memset(&state, 0, sizeof(state)); uint8_t buffered; in.Read(buffered); @@ -214,54 +162,13 @@ struct SumStateExport> { }; #endif -template -struct SumSpillCallbacks { - static AggregateStateLayout Layout(AggregateLayoutInput &input) { - auto layout = AggregateStateLayout(LogicalType::BLOB, sizeof(STATE)); - if (input.function.GetLogicalArguments().size() == 3 && input.bind_data) { - layout.constant_parameters.emplace(2, Value::DOUBLE(input.bind_data->Cast().correction)); - } - return layout; - } - - static void Export(Vector &states, AggregateFinalizeInputData &, Vector &result, idx_t count, idx_t offset) { - UnifiedVectorFormat format; - states.ToUnifiedFormat(format); - auto addresses = UnifiedVectorFormat::GetData(format); - auto writer = FlatVector::Writer(result, count, offset); - for (idx_t i = 0; i < count; i++) { - SumSpillWriter out; - SumStateExport::Write(out, *addresses[format.sel->get_index(i)]); - writer.WriteValue(StringVector::AddStringOrBlob(result, out.data)); - } - } - - static void Import(AggregateImportInputData &input) { - auto values = input.input_vec.Values(); - for (idx_t i = 0; i < input.input_vec.size(); i++) { - auto value = values[i]; - if (!value.IsValid()) { - throw InvalidInputException("NULL stochastic SUM state"); - } - SumSpillReader in(value.GetValue()); - auto &state = *reinterpret_cast(input.dest_buffer + i * input.layout.total_state_size); - SumStateExport::Read(in, state, input.allocator); - in.Finish(); - } - } - - static void Register(AggregateFunction &function) { - function.SetStateExportCallbacks(Layout, Export, Import); - } -}; - } // namespace void RegisterSumStateExport(AggregateFunction &function, bool double_state) { if (double_state) { - SumSpillCallbacks::Register(function); + AggregateStateExport>::Register(function); } else { - SumSpillCallbacks>::Register(function); + AggregateStateExport, SumStateExport>>::Register(function); } } diff --git a/src/categorical/pac_categorical.cpp b/src/categorical/pac_categorical.cpp index 9fe38b78..9e574a12 100644 --- a/src/categorical/pac_categorical.cpp +++ b/src/categorical/pac_categorical.cpp @@ -1,3 +1,4 @@ +#include "aggregates/aggregate_state_export.hpp" // // PAC Categorical Query Support - Implementation // @@ -824,6 +825,19 @@ struct PacListAggregateState { uint64_t counts[64]; // Count of non-null values (for avg/count) }; +struct ListStateCodec { + static void Write(AggregateStateWriter &out, const PacListAggregateState &state) { + out.Write(state.key_hash); + out.Write(state.values); + out.Write(state.counts); + } + static void Read(AggregateStateReader &in, PacListAggregateState &state, ArenaAllocator &) { + in.Read(state.key_hash); + in.Read(state.values); + in.Read(state.counts); + } +}; + // ============================================================================ // Template-specialized operations for each aggregate type // These are designed to be inlined and auto-vectorized by the compiler @@ -1161,16 +1175,20 @@ static void PacListAggregateDestructor(Vector &, AggregateInputData &, idx_t) { template static AggregateFunction CreatePacListAggregate(const Identifier &name) { auto list_double_type = LogicalType::LIST(PacFloatLogicalType()); - return AggregateFunction(Identifier(name), {list_double_type}, list_double_type, - AggregateFunction::StateSize, PacListAggregateInit, - PacListAggregateUpdate, PacListAggregateCombine, - PacListAggregateFinalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, nullptr, - nullptr, PacListAggregateDestructor); + auto function = AggregateFunction( + Identifier(name), {list_double_type}, list_double_type, AggregateFunction::StateSize, + PacListAggregateInit, PacListAggregateUpdate, PacListAggregateCombine, + PacListAggregateFinalize, FunctionNullHandling::DEFAULT_NULL_HANDLING, nullptr, nullptr, + PacListAggregateDestructor); + AggregateStateExport::Register(function); + return function; } void AddPacListAggregateOverload(AggregateFunctionSet &set, const string &aggr_type) { if (aggr_type == "sum") { set.AddFunction(CreatePacListAggregate(set.name)); + } else if (aggr_type == "avg") { + set.AddFunction(CreatePacListAggregate(set.name)); } else if (aggr_type == "count") { set.AddFunction(CreatePacListAggregate(set.name)); } else if (aggr_type == "min") { diff --git a/src/compiler/privacy_compiler.cpp b/src/compiler/privacy_compiler.cpp index 6e1988c6..f1ca7f6b 100644 --- a/src/compiler/privacy_compiler.cpp +++ b/src/compiler/privacy_compiler.cpp @@ -1304,7 +1304,7 @@ static void CompilePacMechanism(const PrivacyCompatibilityResult &check, Optimiz // currently referenced. If we defer decomposition to the post-optimizer // (RewritePacAvgToDiv), the newly-appended count column won't be visible // through those thin projections, causing "Failed to bind column reference" - // errors. The post-optimizer still handles user-written as_avg() in SQL. + // errors. User-written as_avg() is decomposed by its separate pre-optimizer hook. RewritePacAvgToDiv(input, plan); // Clip rewrite: when priv_clip_support is set, refine PAC aggregates to use diff --git a/src/core/privacy_extension.cpp b/src/core/privacy_extension.cpp index 0afc2849..e1e167d8 100644 --- a/src/core/privacy_extension.cpp +++ b/src/core/privacy_extension.cpp @@ -303,12 +303,11 @@ static void LoadInternal(ExtensionLoader &loader) { pac_topk_rule.optimizer_info = pac_info; OptimizerExtension::Register(db.config, std::move(pac_topk_rule)); - // Register as_avg rewrite rule (post-optimizer: decomposes as_noised_avg/as_avg into sum/count + division). - // This post-optimizer handles user-written as_avg() in SQL. Compiler-generated as_noised_avg is already - // decomposed during the pre-optimizer phase (in CompilePrivQuery) so this is a no-op for those. + // Decompose user-written averages before column pruning can hide the new COUNT column. + // Compiler-generated averages are already decomposed by CompilePrivQuery. { OptimizerExtension pac_avg_rule; - pac_avg_rule.optimize_function = RewritePacAvgToDiv; + pac_avg_rule.pre_optimize_function = RewritePacAvgToDiv; OptimizerExtension::Register(db.config, std::move(pac_avg_rule)); } diff --git a/src/include/aggregates/aggregate_state_export.hpp b/src/include/aggregates/aggregate_state_export.hpp new file mode 100644 index 00000000..4993754d --- /dev/null +++ b/src/include/aggregates/aggregate_state_export.hpp @@ -0,0 +1,106 @@ +#pragma once + +#include "aggregates/as_aggregate.hpp" +#include "duckdb/common/vector/string_vector.hpp" + +namespace duckdb { + +// Cold-path state transport shared by the stochastic aggregate implementations. +class AggregateStateWriter { +public: + string data; + + template + void Write(const T &value) { + WriteData(&value, sizeof(value)); + } + + void WriteData(const void *ptr, idx_t size) { + data.append(reinterpret_cast(ptr), size); + } +}; + +class AggregateStateReader { +public: + explicit AggregateStateReader(const string_t &blob) : ptr(blob.GetData()), remaining(blob.GetSize()) { + } + + template + void Read(T &value) { + ReadData(&value, sizeof(value)); + } + + void ReadData(void *value, idx_t size) { + if (remaining < size) { + throw InvalidInputException("Truncated stochastic aggregate state"); + } + memcpy(value, ptr, size); + ptr += size; + remaining -= size; + } + + bool ReadFlag() { + uint8_t flag; + Read(flag); + if (flag > 1) { + throw InvalidInputException("Invalid stochastic aggregate state flag"); + } + return flag != 0; + } + + void Finish() { + if (remaining) { + throw InvalidInputException("Unexpected trailing stochastic aggregate state data"); + } + } + +private: + const char *ptr; + idx_t remaining; +}; + +template +struct AggregateStateExport { + static AggregateStateLayout Layout(AggregateLayoutInput &input) { + auto layout = AggregateStateLayout(LogicalType::BLOB, sizeof(STATE)); + auto &arguments = input.function.GetLogicalArguments(); + if (arguments.size() == 3 && input.bind_data) { + layout.constant_parameters.emplace(2, Value::DOUBLE(input.bind_data->Cast().correction)); + } + return layout; + } + + static void Export(Vector &states, AggregateFinalizeInputData &, Vector &result, idx_t count, idx_t offset) { + UnifiedVectorFormat format; + states.ToUnifiedFormat(format); + auto addresses = UnifiedVectorFormat::GetData(format); + auto writer = FlatVector::Writer(result, count, offset); + for (idx_t i = 0; i < count; i++) { + AggregateStateWriter out; + CODEC::Write(out, *addresses[format.sel->get_index(i)]); + writer.WriteValue(StringVector::AddStringOrBlob(result, out.data)); + } + } + + static void Import(AggregateImportInputData &input) { + auto values = input.input_vec.Values(); + for (idx_t i = 0; i < input.input_vec.size(); i++) { + auto value = values[i]; + if (!value.IsValid()) { + throw InvalidInputException("NULL stochastic aggregate state"); + } + AggregateStateReader in(value.GetValue()); + auto &state = *reinterpret_cast(input.dest_buffer + i * input.layout.total_state_size); + CODEC::Read(in, state, input.allocator); + in.Finish(); + } + } + + static void Register(BaseAggregateFunction &function) { + function.GetCallbacks().get_state_type = Layout; + function.SetExportAggregateStateCallback(Export); + function.SetImportAggregateStateCallback(Import); + } +}; + +} // namespace duckdb diff --git a/src/query_processing/pac_avg_rewriter.cpp b/src/query_processing/pac_avg_rewriter.cpp index a090df3a..e25ea488 100644 --- a/src/query_processing/pac_avg_rewriter.cpp +++ b/src/query_processing/pac_avg_rewriter.cpp @@ -62,6 +62,9 @@ static void RewritePacAvgAggregate(OptimizerExtensionInput &input, unique_ptrCast(); + if (aggr.GetChildren().size() == 1) { + continue; // List AVG already has its own aggregate implementation. + } if (aggr.FunctionMutable().GetName().GetIdentifierName() == "as_noised_avg" || aggr.FunctionMutable().GetName().GetIdentifierName() == "as_avg") { avg_positions.push_back(i); diff --git a/src/utils/privacy_helpers.cpp b/src/utils/privacy_helpers.cpp index 803fbbfe..a126551f 100644 --- a/src/utils/privacy_helpers.cpp +++ b/src/utils/privacy_helpers.cpp @@ -221,6 +221,13 @@ void ApplyIndexMapToSubtree(LogicalOperator *node, const std::unordered_maptable_index.index) != map.end()) { cteref_ptr->table_index.index = map.at(cteref_ptr->table_index.index); } + // A copied reader must point at the copied definition, while references + // to definitions outside this subtree retain their original identity. + auto definition = map.find(cteref_ptr->cte_index.index); + if (definition != map.end()) { + cteref_ptr->cte_index.index = definition->second; + } + } else if (auto dg_ptr = dynamic_cast(node)) { if (map.find(dg_ptr->table_index.index) != map.end()) { dg_ptr->table_index.index = map.at(dg_ptr->table_index.index); diff --git a/test/sql/pac_avg.test b/test/sql/pac_avg.test index 16d40abd..6feb0497 100644 --- a/test/sql/pac_avg.test +++ b/test/sql/pac_avg.test @@ -212,3 +212,44 @@ DROP TABLE test_data # Cleanup PAC metadata at end of test statement ok PRAGMA clear_privacy_metadata; + +# Decompose handwritten AVG before HAVING's projection map prunes its new COUNT. +statement ok +SET priv_rewrite=false + +statement ok +SET pac_sample_diversity_check=false + +query III +SELECT g,n,a FROM ( + SELECT i%2 g,as_noised_count(18446744073709551615::UBIGINT,i) n, + as_noised_avg(18446744073709551615::UBIGINT,i) a + FROM range(8)t(i) GROUP BY g + HAVING as_noised_count(18446744073709551615::UBIGINT,i)>0 +) ORDER BY g +---- +0 8 3 +1 8 4 + +# AVG over stochastic intermediate results must preserve each lane (TPC-DS Q65). +query I +WITH a AS ( + SELECT as_avg(list_transform(range(64),lambda j:(i+j)::FLOAT)) lanes FROM range(4)t(i) +) +SELECT count(*) FROM a,range(64)t(j) WHERE lanes[j+1]=j+1.5 +---- +64 + +query I +WITH a AS ( + SELECT as_avg(list_transform(range(64),lambda j:(i+j)::FLOAT)) EXPORT_STATE s FROM range(4)t(i) +) +SELECT count(*) FROM a,range(64)t(j) WHERE finalize(s)[j+1]=j+1.5 +---- +64 + +statement ok +SET pac_sample_diversity_check=true + +statement ok +SET priv_rewrite=true diff --git a/test/sql/pac_categorical_subqueries.test b/test/sql/pac_categorical_subqueries.test index da665632..e5503ce2 100644 --- a/test/sql/pac_categorical_subqueries.test +++ b/test/sql/pac_categorical_subqueries.test @@ -870,3 +870,62 @@ DROP TABLE scalar_case_reason; statement ok DROP TABLE scalar_case_pu; + +# Mixed DISTINCT/non-DISTINCT branches copy DuckDB 2.0 correlated CTEs. +# A copied CTE_SCAN must reference the copied definition (TPC-DS Q94). +statement ok +SET privacy_seed=42 + +statement ok +SET pac_hash_repair=true + +statement ok +SET pac_mi=0 + +statement ok +SET pac_sample_diversity_check=false + +statement ok +SET threads=1 + +statement ok +CREATE PU TABLE cte_person(id BIGINT,PRIVACY_KEY(id)) + +statement ok +CREATE TABLE cte_fact(owner BIGINT,k BIGINT,warehouse BIGINT,v BIGINT) + +statement ok +CREATE TABLE cte_returns(k BIGINT) + +statement ok +ALTER TABLE cte_fact ADD PRIVACY_LINK(owner) REFERENCES cte_person(id) + +statement ok +INSERT INTO cte_person SELECT i FROM range(32)t(i) + +statement ok +INSERT INTO cte_fact SELECT i//2,i//2,i%2,1 FROM range(64)t(i) + +statement ok +INSERT INTO cte_returns VALUES(0),(10),(16) + +# The ordinary world-zero SQL reference gives 15 surviving orders, two facts each; +# both stochastic aggregates apply the existing 2x sampling correction. +query II +SELECT count(DISTINCT f.k),sum(f.v) FROM cte_fact f +WHERE EXISTS(SELECT 1 FROM cte_fact x WHERE x.k=f.k AND x.warehouse<>f.warehouse) +AND NOT EXISTS(SELECT 1 FROM cte_returns r WHERE r.k=f.k) +---- +30 60 + +statement ok +DROP TABLE cte_fact + +statement ok +DROP TABLE cte_returns + +statement ok +DROP TABLE cte_person + +statement ok +SET pac_sample_diversity_check=true diff --git a/test/sql/pac_sum.test b/test/sql/pac_sum.test index 80734dba..97f15376 100644 --- a/test/sql/pac_sum.test +++ b/test/sql/pac_sum.test @@ -502,7 +502,7 @@ true statement error SELECT finalize(to_aggregate_state(''::BLOB,'as_sum',['UBIGINT','BIGINT'])); ---- -Truncated stochastic SUM state +Truncated stochastic aggregate state # DECIMAL binding replaces its implementation; retain the logical third @@ -561,3 +561,71 @@ SELECT count(*) FROM ( ),unnest(lanes)t(v) WHERE v=30; ---- 64 + +# All stochastic aggregates in a hash table must provide a state layout. +# Exercise COUNT's SWAR overflow, MIN/MAX, AVG decomposition and list-input states. +statement ok +SET priv_rewrite=false + +statement ok +SET pac_mi=0 + +statement ok +SET pac_sample_diversity_check=false + +statement ok +SET threads=1 + +statement ok +SET memory_limit='512MB' + +statement ok +CREATE TABLE mixed_spill_expected AS +SELECT count(*) n,sum(hash(s,c,a,mn,mx)) checksum FROM ( + SELECT i//8 g,as_sum(18446744073709551615::UBIGINT,1::BIGINT) s, + as_count(18446744073709551615::UBIGINT) c, + as_noised_avg(18446744073709551615::UBIGINT,2::DOUBLE) a, + as_min(18446744073709551615::UBIGINT,i%8) mn, + as_max(18446744073709551615::UBIGINT,i%8) mx + FROM range(800000)t(i) GROUP BY g +) + +statement ok +SET memory_limit='32MB' + +query I +SELECT r.n=e.n AND r.checksum=e.checksum FROM mixed_spill_expected e,( + SELECT count(*) n,sum(hash(s,c,a,mn,mx)) checksum FROM ( + SELECT i//8 g,as_sum(18446744073709551615::UBIGINT,1::BIGINT) s, + as_count(18446744073709551615::UBIGINT) c, + as_noised_avg(18446744073709551615::UBIGINT,2::DOUBLE) a, + as_min(18446744073709551615::UBIGINT,i%8) mn, + as_max(18446744073709551615::UBIGINT,i%8) mx + FROM range(800000)t(i) GROUP BY g + ))r +---- +true + +statement ok +SET memory_limit='512MB' + +statement ok +DROP TABLE mixed_spill_expected + +query II +SELECT finalize(s)=expected,finalize(c)=expected FROM ( + SELECT as_noised_count(18446744073709551615::UBIGINT,0.75) EXPORT_STATE s, + as_noised_count(18446744073709551615::UBIGINT,1::BIGINT,0.75) EXPORT_STATE c, + as_noised_count(18446744073709551615::UBIGINT,0.75) expected FROM range(1000) +) +---- +true true + +statement ok +SET pac_sample_diversity_check=true + +statement ok +SET memory_limit='80%' + +statement ok +SET priv_rewrite=true From 68b575c933e61fdc38a2786474b0de57636cd178 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 5 Nov 2026 11:03:06 +0100 Subject: [PATCH 09/32] Scope spill follow-up to state transport and retain SUM memory optimizations --- .../workflows/MainDistributionPipeline.yml | 1 - docs/pac/internal_operators.md | 22 +++---- src/aggregates/as_count.cpp | 22 +++---- src/aggregates/as_min_max.cpp | 3 + src/aggregates/as_sum_state_export.cpp | 30 +++++++--- src/categorical/pac_categorical.cpp | 2 - src/compiler/privacy_compiler.cpp | 2 +- src/core/privacy_extension.cpp | 7 ++- src/query_processing/pac_avg_rewriter.cpp | 3 - src/utils/privacy_helpers.cpp | 7 --- test/sql/pac_avg.test | 41 ------------- test/sql/pac_categorical_subqueries.test | 59 ------------------- 12 files changed, 53 insertions(+), 146 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index f7485d1e..7b01c057 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,7 +19,6 @@ jobs: duckdb_version: v2.0.0 ci_tools_version: ebdcc1961780b0ce1272a0d7637dc58447c34885 extension_name: privacy - test_config: '{"test_env_variables": {"T": "\"test/*\" --test-flags=--output-sql=true"}}' code-quality-check: name: Code Quality Check diff --git a/docs/pac/internal_operators.md b/docs/pac/internal_operators.md index 78736e1a..fa5974fd 100644 --- a/docs/pac/internal_operators.md +++ b/docs/pac/internal_operators.md @@ -179,9 +179,8 @@ No new aggregate functions, no new noise mechanisms, no plan decomposition. The membership-mask-plus-scalar SUM, COUNT, and MIN/MAX implementations expose DuckDB's native state-export/import callbacks, including DECIMAL specializations. -List-input SUM, COUNT, AVG, and MIN/MAX also provide callbacks. AVG over scalar -inputs decomposes into the spillable SUM and COUNT implementations before column -pruning. DuckDB can export arena-backed hash-aggregate states to spillable columns +List-input SUM, COUNT, and MIN/MAX also provide callbacks. Scalar AVG uses the +SUM and COUNT implementations through the existing rewrite. DuckDB can export arena-backed hash-aggregate states to spillable columns and import them when combining partitions. Every aggregate in a hash table must expose a compatible state layout for DuckDB to select this path. @@ -190,17 +189,20 @@ negative integer cascade levels and overflow budgets, floating-point counters, COUNT SWAR bytes and overflow totals, MIN/MAX bounds, and list-aggregate values and counts. Folded correction arguments are retained when rebinding exported states. It contains no addresses. Import reconstructs owned arena allocations and retains -the inline-level optimization. Export does not finalize, resample, or add noise; +the inline-level optimization. COUNT and MIN/MAX expand restored buffers using their existing +kernels only on the import path, so a subsequent combine cannot overwrite a +restored destination buffer. SUM retains restored buffers for its buffered combiner. Export does not finalize, resample, or add noise; the native update kernels and aggregate-state sizes remain unchanged. The binary representation is internal to this implementation, rather than a portable state interchange format. -Small groups also retain their one/two-value buffers during combine. Finalization -materializes these buffers on copies using temporary storage backed by -`ArenaAllocator::GetUnderlyingAllocator()`, so the allocations are released after -the callback instead of accumulating in a query-long arena. The original buffered -state remains reusable. This addresses the Q78 path where buffers expand only in -final combine, after sink-phase spill decisions. +SUM retains one/two-value buffers during combine. Finalization materializes copies +in temporary storage backed by `ArenaAllocator::GetUnderlyingAllocator()`, releasing +these allocations after the callback. These two optimizations address the Q78 +allocation path: expanding many small groups during final combine/finalization can +exhaust memory after sink-phase spilling decisions. Disk spilling handles stored +states; it does not itself bound these late allocations. COUNT and MIN/MAX retain +their original in-memory update, combine, and finalization paths. Regression coverage extends `test/sql/pac_sum.test`: round trips include signed, unsigned, floating-point and DECIMAL inputs, buffered/empty states, high cascade diff --git a/src/aggregates/as_count.cpp b/src/aggregates/as_count.cpp index dc138b3a..d796d15e 100644 --- a/src/aggregates/as_count.cpp +++ b/src/aggregates/as_count.cpp @@ -94,6 +94,9 @@ struct CountStateCodec { ReadInner(in, *state.EnsureState(allocator), allocator); } state.n_buffered |= buffered; + if (buffered) { + state.FlushBuffer(state, allocator); + } #endif } }; @@ -873,7 +876,7 @@ void RegisterDpSampleMCountFunctions(ExtensionLoader &loader) { FunctionDescription desc; desc.description = "[INTERNAL] Experimental variable-m SAA counters for COUNT-like values."; info.descriptions.push_back(std::move(desc)); - RegisterCountFunctions(loader, info); + loader.RegisterFunction(std::move(info)); } void RegisterDpSampleMCountDistinctFunctions(ExtensionLoader &loader) { @@ -888,7 +891,7 @@ void RegisterDpSampleMCountDistinctFunctions(ExtensionLoader &loader) { desc.description = "[INTERNAL] Experimental variable-m SAA lane counters for pre-deduplicated COUNT(DISTINCT) values."; info.descriptions.push_back(std::move(desc)); - RegisterCountFunctions(loader, info); + loader.RegisterFunction(std::move(info)); AggregateFunctionSet if_set("as_sample_m_count_distinct_if"); if_set.AddFunction(AggregateFunction("as_sample_m_count_distinct_if", {LogicalType::UBIGINT, LogicalType::BOOLEAN}, @@ -902,7 +905,7 @@ void RegisterDpSampleMCountDistinctFunctions(ExtensionLoader &loader) { "[INTERNAL] Experimental variable-m SAA lane counters for conditionally pre-deduplicated COUNT(DISTINCT) " "values."; if_info.descriptions.push_back(std::move(if_desc)); - RegisterCountFunctions(loader, if_info); + loader.RegisterFunction(std::move(if_info)); } // Mixed aggregate queries cannot use the dedicated three-stage DISTINCT plan. Their @@ -1090,7 +1093,7 @@ void RegisterDpSampleMCountDistinctValuesFunctions(ExtensionLoader &loader) { FunctionDescription desc; desc.description = "[INTERNAL] Experimental variable-m SAA lane counters for per-PU COUNT(DISTINCT) value lists."; info.descriptions.push_back(std::move(desc)); - RegisterCountFunctions(loader, info); + loader.RegisterFunction(std::move(info)); } void RegisterDpSampleMCountIfFunctions(ExtensionLoader &loader) { @@ -1105,7 +1108,7 @@ void RegisterDpSampleMCountIfFunctions(ExtensionLoader &loader) { FunctionDescription desc; desc.description = "[INTERNAL] Experimental variable-m SAA counters for lane-selected COUNT-like values."; info.descriptions.push_back(std::move(desc)); - RegisterCountFunctions(loader, info); + loader.RegisterFunction(std::move(info)); } static void PacSampleMCountUpdateOne(DpSampleMCountState &state, ArenaAllocator &allocator, uint64_t hash, @@ -1291,7 +1294,7 @@ void RegisterPacSampleMCountFunctions(ExtensionLoader &loader) { FunctionDescription desc; desc.description = "[INTERNAL] Variable-m PAC counters with 50% world membership."; info.descriptions.push_back(std::move(desc)); - RegisterCountFunctions(loader, info); + loader.RegisterFunction(std::move(info)); } void RegisterPacSampleMCountIfFunctions(ExtensionLoader &loader) { @@ -1306,7 +1309,7 @@ void RegisterPacSampleMCountIfFunctions(ExtensionLoader &loader) { FunctionDescription desc; desc.description = "[INTERNAL] Variable-m PAC counters with per-lane predicates."; info.descriptions.push_back(std::move(desc)); - RegisterCountFunctions(loader, info); + loader.RegisterFunction(std::move(info)); } // ============================================================================ @@ -1358,7 +1361,7 @@ void RegisterPacAvgFunctions(ExtensionLoader &loader) { avg_desc.examples = {"SELECT c_mktsegment, as_noised_avg(priv_hash(hash(c_custkey)), c_acctbal) FROM customer " "GROUP BY c_mktsegment"}; avg_info.descriptions.push_back(std::move(avg_desc)); - RegisterCountFunctions(loader, avg_info); + loader.RegisterFunction(std::move(avg_info)); // as_avg(UBIGINT, value_type[, correction]) → LIST — same implementation as as_count(UBIGINT, ANY) auto list_float_type = LogicalType::LIST(PacFloatLogicalType()); @@ -1371,12 +1374,11 @@ void RegisterPacAvgFunctions(ExtensionLoader &loader) { "as_avg", {LogicalType::UBIGINT, LogicalType::ANY, LogicalType::DOUBLE}, list_float_type, PacCountStateSize, PacCountInitialize, PacCountColumnScatterUpdate, PacCountCombine, PacCountFinalizeCounters, FunctionNullHandling::DEFAULT_NULL_HANDLING, PacCountColumnUpdate, PacCountBind)); - AddPacListAggregateOverload(counters_set, "avg"); CreateAggregateFunctionInfo avg_counters_info(counters_set); FunctionDescription avg_counters_desc; avg_counters_desc.description = "[INTERNAL] Returns 64 PAC subsample counters as LIST for categorical queries."; avg_counters_info.descriptions.push_back(std::move(avg_counters_desc)); - RegisterCountFunctions(loader, avg_counters_info); + loader.RegisterFunction(std::move(avg_counters_info)); } // ============================================================================ diff --git a/src/aggregates/as_min_max.cpp b/src/aggregates/as_min_max.cpp index 6a4a29e4..e4d81a94 100644 --- a/src/aggregates/as_min_max.cpp +++ b/src/aggregates/as_min_max.cpp @@ -72,6 +72,9 @@ struct MinMaxStateCodec { ReadInner(in, *state.EnsureState(allocator)); } state.n_buffered |= buffered; + if (buffered) { + state.FlushBuffer(state, allocator); + } #endif } }; diff --git a/src/aggregates/as_sum_state_export.cpp b/src/aggregates/as_sum_state_export.cpp index 97a479eb..50e4a1ab 100644 --- a/src/aggregates/as_sum_state_export.cpp +++ b/src/aggregates/as_sum_state_export.cpp @@ -69,16 +69,24 @@ struct SumStateExport> { } for (int k = 0; k <= state.max_level_used; k++) { in.Read(state.exact_total[k]); - if (in.ReadFlag()) { - // Retain the inline optimization when all pointer slots fit before it. - if (state.max_level_used < 9) { - state.AllocateLevel(allocator, k); - } else { - state.u.levels[k] = - reinterpret_cast(allocator.Allocate(sizeof(uint64_t) * PAC_APPROX_SWAR_ELEMENTS)); - } - in.ReadData(state.u.levels[k], sizeof(uint64_t) * PAC_APPROX_SWAR_ELEMENTS); +#ifdef PAC_SIGNEDSUM + if (state.exact_total[k] < State::UNDERFLOW_THRESHOLD || state.exact_total[k] > State::OVERFLOW_THRESHOLD) { +#else + if (state.exact_total[k] < 0 || state.exact_total[k] > PacSumIntState::OVERFLOW_THRESHOLD) { +#endif + throw InvalidInputException("Invalid stochastic SUM cascade budget"); + } + if (!in.ReadFlag()) { + throw InvalidInputException("Missing stochastic SUM cascade level"); + } + // Retain the inline optimization when all pointer slots fit before it. + if (state.max_level_used < 9) { + state.AllocateLevel(allocator, k); + } else { + state.u.levels[k] = + reinterpret_cast(allocator.Allocate(sizeof(uint64_t) * PAC_APPROX_SWAR_ELEMENTS)); } + in.ReadData(state.u.levels[k], sizeof(uint64_t) * PAC_APPROX_SWAR_ELEMENTS); } #elif defined(PAC_NOCASCADING) in.Read(state.probabilistic_total128); @@ -165,8 +173,12 @@ struct SumStateExport> { } // namespace void RegisterSumStateExport(AggregateFunction &function, bool double_state) { + auto value_type = function.GetSignature().GetParameter(1).GetType().id(); if (double_state) { AggregateStateExport>::Register(function); + } else if (value_type == LogicalTypeId::UTINYINT || value_type == LogicalTypeId::USMALLINT || + value_type == LogicalTypeId::UINTEGER || value_type == LogicalTypeId::UBIGINT) { + AggregateStateExport, SumStateExport>>::Register(function); } else { AggregateStateExport, SumStateExport>>::Register(function); } diff --git a/src/categorical/pac_categorical.cpp b/src/categorical/pac_categorical.cpp index 9e574a12..245b9aa7 100644 --- a/src/categorical/pac_categorical.cpp +++ b/src/categorical/pac_categorical.cpp @@ -1187,8 +1187,6 @@ static AggregateFunction CreatePacListAggregate(const Identifier &name) { void AddPacListAggregateOverload(AggregateFunctionSet &set, const string &aggr_type) { if (aggr_type == "sum") { set.AddFunction(CreatePacListAggregate(set.name)); - } else if (aggr_type == "avg") { - set.AddFunction(CreatePacListAggregate(set.name)); } else if (aggr_type == "count") { set.AddFunction(CreatePacListAggregate(set.name)); } else if (aggr_type == "min") { diff --git a/src/compiler/privacy_compiler.cpp b/src/compiler/privacy_compiler.cpp index f1ca7f6b..6e1988c6 100644 --- a/src/compiler/privacy_compiler.cpp +++ b/src/compiler/privacy_compiler.cpp @@ -1304,7 +1304,7 @@ static void CompilePacMechanism(const PrivacyCompatibilityResult &check, Optimiz // currently referenced. If we defer decomposition to the post-optimizer // (RewritePacAvgToDiv), the newly-appended count column won't be visible // through those thin projections, causing "Failed to bind column reference" - // errors. User-written as_avg() is decomposed by its separate pre-optimizer hook. + // errors. The post-optimizer still handles user-written as_avg() in SQL. RewritePacAvgToDiv(input, plan); // Clip rewrite: when priv_clip_support is set, refine PAC aggregates to use diff --git a/src/core/privacy_extension.cpp b/src/core/privacy_extension.cpp index e1e167d8..0afc2849 100644 --- a/src/core/privacy_extension.cpp +++ b/src/core/privacy_extension.cpp @@ -303,11 +303,12 @@ static void LoadInternal(ExtensionLoader &loader) { pac_topk_rule.optimizer_info = pac_info; OptimizerExtension::Register(db.config, std::move(pac_topk_rule)); - // Decompose user-written averages before column pruning can hide the new COUNT column. - // Compiler-generated averages are already decomposed by CompilePrivQuery. + // Register as_avg rewrite rule (post-optimizer: decomposes as_noised_avg/as_avg into sum/count + division). + // This post-optimizer handles user-written as_avg() in SQL. Compiler-generated as_noised_avg is already + // decomposed during the pre-optimizer phase (in CompilePrivQuery) so this is a no-op for those. { OptimizerExtension pac_avg_rule; - pac_avg_rule.pre_optimize_function = RewritePacAvgToDiv; + pac_avg_rule.optimize_function = RewritePacAvgToDiv; OptimizerExtension::Register(db.config, std::move(pac_avg_rule)); } diff --git a/src/query_processing/pac_avg_rewriter.cpp b/src/query_processing/pac_avg_rewriter.cpp index e25ea488..a090df3a 100644 --- a/src/query_processing/pac_avg_rewriter.cpp +++ b/src/query_processing/pac_avg_rewriter.cpp @@ -62,9 +62,6 @@ static void RewritePacAvgAggregate(OptimizerExtensionInput &input, unique_ptrCast(); - if (aggr.GetChildren().size() == 1) { - continue; // List AVG already has its own aggregate implementation. - } if (aggr.FunctionMutable().GetName().GetIdentifierName() == "as_noised_avg" || aggr.FunctionMutable().GetName().GetIdentifierName() == "as_avg") { avg_positions.push_back(i); diff --git a/src/utils/privacy_helpers.cpp b/src/utils/privacy_helpers.cpp index a126551f..803fbbfe 100644 --- a/src/utils/privacy_helpers.cpp +++ b/src/utils/privacy_helpers.cpp @@ -221,13 +221,6 @@ void ApplyIndexMapToSubtree(LogicalOperator *node, const std::unordered_maptable_index.index) != map.end()) { cteref_ptr->table_index.index = map.at(cteref_ptr->table_index.index); } - // A copied reader must point at the copied definition, while references - // to definitions outside this subtree retain their original identity. - auto definition = map.find(cteref_ptr->cte_index.index); - if (definition != map.end()) { - cteref_ptr->cte_index.index = definition->second; - } - } else if (auto dg_ptr = dynamic_cast(node)) { if (map.find(dg_ptr->table_index.index) != map.end()) { dg_ptr->table_index.index = map.at(dg_ptr->table_index.index); diff --git a/test/sql/pac_avg.test b/test/sql/pac_avg.test index 6feb0497..16d40abd 100644 --- a/test/sql/pac_avg.test +++ b/test/sql/pac_avg.test @@ -212,44 +212,3 @@ DROP TABLE test_data # Cleanup PAC metadata at end of test statement ok PRAGMA clear_privacy_metadata; - -# Decompose handwritten AVG before HAVING's projection map prunes its new COUNT. -statement ok -SET priv_rewrite=false - -statement ok -SET pac_sample_diversity_check=false - -query III -SELECT g,n,a FROM ( - SELECT i%2 g,as_noised_count(18446744073709551615::UBIGINT,i) n, - as_noised_avg(18446744073709551615::UBIGINT,i) a - FROM range(8)t(i) GROUP BY g - HAVING as_noised_count(18446744073709551615::UBIGINT,i)>0 -) ORDER BY g ----- -0 8 3 -1 8 4 - -# AVG over stochastic intermediate results must preserve each lane (TPC-DS Q65). -query I -WITH a AS ( - SELECT as_avg(list_transform(range(64),lambda j:(i+j)::FLOAT)) lanes FROM range(4)t(i) -) -SELECT count(*) FROM a,range(64)t(j) WHERE lanes[j+1]=j+1.5 ----- -64 - -query I -WITH a AS ( - SELECT as_avg(list_transform(range(64),lambda j:(i+j)::FLOAT)) EXPORT_STATE s FROM range(4)t(i) -) -SELECT count(*) FROM a,range(64)t(j) WHERE finalize(s)[j+1]=j+1.5 ----- -64 - -statement ok -SET pac_sample_diversity_check=true - -statement ok -SET priv_rewrite=true diff --git a/test/sql/pac_categorical_subqueries.test b/test/sql/pac_categorical_subqueries.test index e5503ce2..da665632 100644 --- a/test/sql/pac_categorical_subqueries.test +++ b/test/sql/pac_categorical_subqueries.test @@ -870,62 +870,3 @@ DROP TABLE scalar_case_reason; statement ok DROP TABLE scalar_case_pu; - -# Mixed DISTINCT/non-DISTINCT branches copy DuckDB 2.0 correlated CTEs. -# A copied CTE_SCAN must reference the copied definition (TPC-DS Q94). -statement ok -SET privacy_seed=42 - -statement ok -SET pac_hash_repair=true - -statement ok -SET pac_mi=0 - -statement ok -SET pac_sample_diversity_check=false - -statement ok -SET threads=1 - -statement ok -CREATE PU TABLE cte_person(id BIGINT,PRIVACY_KEY(id)) - -statement ok -CREATE TABLE cte_fact(owner BIGINT,k BIGINT,warehouse BIGINT,v BIGINT) - -statement ok -CREATE TABLE cte_returns(k BIGINT) - -statement ok -ALTER TABLE cte_fact ADD PRIVACY_LINK(owner) REFERENCES cte_person(id) - -statement ok -INSERT INTO cte_person SELECT i FROM range(32)t(i) - -statement ok -INSERT INTO cte_fact SELECT i//2,i//2,i%2,1 FROM range(64)t(i) - -statement ok -INSERT INTO cte_returns VALUES(0),(10),(16) - -# The ordinary world-zero SQL reference gives 15 surviving orders, two facts each; -# both stochastic aggregates apply the existing 2x sampling correction. -query II -SELECT count(DISTINCT f.k),sum(f.v) FROM cte_fact f -WHERE EXISTS(SELECT 1 FROM cte_fact x WHERE x.k=f.k AND x.warehouse<>f.warehouse) -AND NOT EXISTS(SELECT 1 FROM cte_returns r WHERE r.k=f.k) ----- -30 60 - -statement ok -DROP TABLE cte_fact - -statement ok -DROP TABLE cte_returns - -statement ok -DROP TABLE cte_person - -statement ok -SET pac_sample_diversity_check=true From 0b7852db1d57188750bddcf9fdf61418653a3050 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 5 Nov 2026 15:46:25 +0100 Subject: [PATCH 10/32] Align imported stochastic states after variable-size aggregate allocations --- .../workflows/MainDistributionPipeline.yml | 1 + .../aggregates/aggregate_state_export.hpp | 3 +++ test/sql/pac_sum.test | 23 +++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 7b01c057..f7485d1e 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,6 +19,7 @@ jobs: duckdb_version: v2.0.0 ci_tools_version: ebdcc1961780b0ce1272a0d7637dc58447c34885 extension_name: privacy + test_config: '{"test_env_variables": {"T": "\"test/*\" --test-flags=--output-sql=true"}}' code-quality-check: name: Code Quality Check diff --git a/src/include/aggregates/aggregate_state_export.hpp b/src/include/aggregates/aggregate_state_export.hpp index 4993754d..4d30639b 100644 --- a/src/include/aggregates/aggregate_state_export.hpp +++ b/src/include/aggregates/aggregate_state_export.hpp @@ -91,6 +91,9 @@ struct AggregateStateExport { } AggregateStateReader in(value.GetValue()); auto &state = *reinterpret_cast(input.dest_buffer + i * input.layout.total_state_size); + // Other aggregates can leave this shared arena at an odd byte offset. + // Buffered stochastic states store their count in the pointer's low bits. + input.allocator.AlignNext(); CODEC::Read(in, state, input.allocator); in.Finish(); } diff --git a/test/sql/pac_sum.test b/test/sql/pac_sum.test index 97f15376..54b4229b 100644 --- a/test/sql/pac_sum.test +++ b/test/sql/pac_sum.test @@ -612,6 +612,29 @@ SET memory_limit='512MB' statement ok DROP TABLE mixed_spill_expected +# String states allocate variable byte counts in the shared import arena. +# Rebuilding the SUM state must realign its tagged inner-state pointer. +statement ok +SET memory_limit='32MB' + +statement ok +SET debug_force_external=true + +query II +SELECT count(*),count(*) FILTER (WHERE list_min(s)=6 AND list_max(s)=6 AND length(t)=39) FROM ( + SELECT i//3 g,string_agg(repeat('x',13),'') t, + as_sum(18446744073709551615::UBIGINT,1::BIGINT) s + FROM range(300000)t(i) GROUP BY g +) +---- +100000 100000 + +statement ok +SET debug_force_external=false + +statement ok +SET memory_limit='512MB' + query II SELECT finalize(s)=expected,finalize(c)=expected FROM ( SELECT as_noised_count(18446744073709551615::UBIGINT,0.75) EXPORT_STATE s, From 9534f62f3c36757523709e2632e8bd07824ebb76 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 5 Nov 2026 16:04:56 +0100 Subject: [PATCH 11/32] Execute Windows regression tests with statement-level crash tracing --- .github/workflows/MainDistributionPipeline.yml | 1 - test/sql/pac_multi_pu.test | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index f7485d1e..7b01c057 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,7 +19,6 @@ jobs: duckdb_version: v2.0.0 ci_tools_version: ebdcc1961780b0ce1272a0d7637dc58447c34885 extension_name: privacy - test_config: '{"test_env_variables": {"T": "\"test/*\" --test-flags=--output-sql=true"}}' code-quality-check: name: Code Quality Check diff --git a/test/sql/pac_multi_pu.test b/test/sql/pac_multi_pu.test index c65c81a3..cef7aede 100644 --- a/test/sql/pac_multi_pu.test +++ b/test/sql/pac_multi_pu.test @@ -4,6 +4,9 @@ require privacy +# Temporary crash tracing: unlike --output-sql, debug mode executes every query. +mode debug + statement ok PRAGMA clear_privacy_metadata; From 24b7041adfc815f08edbc2750e6a835d6f6b556f Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 5 Nov 2026 16:13:54 +0100 Subject: [PATCH 12/32] Preserve NULL identity semantics when importing stochastic aggregate states --- .../aggregates/aggregate_state_export.hpp | 4 +++- test/sql/pac_sum.test | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/include/aggregates/aggregate_state_export.hpp b/src/include/aggregates/aggregate_state_export.hpp index 4d30639b..ffd33298 100644 --- a/src/include/aggregates/aggregate_state_export.hpp +++ b/src/include/aggregates/aggregate_state_export.hpp @@ -87,7 +87,9 @@ struct AggregateStateExport { for (idx_t i = 0; i < input.input_vec.size(); i++) { auto value = values[i]; if (!value.IsValid()) { - throw InvalidInputException("NULL stochastic aggregate state"); + // Keep the caller-initialized empty state. finalize propagates NULL, + // while combine treats one NULL input as the identity. + continue; } AggregateStateReader in(value.GetValue()); auto &state = *reinterpret_cast(input.dest_buffer + i * input.layout.total_state_size); diff --git a/test/sql/pac_sum.test b/test/sql/pac_sum.test index 54b4229b..c3cd83fd 100644 --- a/test/sql/pac_sum.test +++ b/test/sql/pac_sum.test @@ -644,6 +644,30 @@ SELECT finalize(s)=expected,finalize(c)=expected FROM ( ---- true true +# NULL exported states occur naturally after an outer join. Native finalize +# propagates NULL, and combine keeps the non-NULL input unchanged. +query I +WITH s AS MATERIALIZED ( + SELECT as_sum(18446744073709551615::UBIGINT,1::BIGINT) EXPORT_STATE x, + as_sum(18446744073709551615::UBIGINT,1::BIGINT) expected +) +SELECT finalize(CASE WHEN i=0 THEN x END) IS NOT DISTINCT FROM CASE WHEN i=0 THEN expected END +FROM s,range(2)t(i) ORDER BY i +---- +true +true + +query I +WITH s AS MATERIALIZED ( + SELECT as_sum(18446744073709551615::UBIGINT,1::BIGINT) EXPORT_STATE x, + as_sum(18446744073709551615::UBIGINT,1::BIGINT) expected +) +SELECT finalize(combine(CASE WHEN i=0 THEN x END,CASE WHEN i=1 THEN x END)) IS NOT DISTINCT FROM expected +FROM s,range(2)t(i) ORDER BY i +---- +true +true + statement ok SET pac_sample_diversity_check=true From 4553220f8d2ce7274709a633a8b9e28769b0b275 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 5 Nov 2026 16:21:34 +0100 Subject: [PATCH 13/32] Retain native Windows test diagnostics during crash investigation --- .github/workflows/MainDistributionPipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 7b01c057..fff1393f 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,6 +19,9 @@ jobs: duckdb_version: v2.0.0 ci_tools_version: ebdcc1961780b0ce1272a0d7637dc58447c34885 extension_name: privacy + # Temporary Windows diagnosis: retain the native test output on a crash. + exclude_archs: 'linux_amd64;linux_arm64;osx_arm64;windows_amd64_mingw;wasm_eh;wasm_mvp;wasm_threads' + test_config: '{"test_env_variables": {"TEST_RUNNER_SCRIPT": ".ci/use-native-test-runner", "T": "\"test/*\""}}' code-quality-check: name: Code Quality Check From 39bdeacc95188ddd4a9cbb9d98826cb3e70d0b04 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 5 Nov 2026 16:29:42 +0100 Subject: [PATCH 14/32] Trace the SUM suite identified by the native Windows crash report --- .github/workflows/MainDistributionPipeline.yml | 2 +- test/sql/pac_multi_pu.test | 3 --- test/sql/pac_sum.test | 3 +++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index fff1393f..27d6401a 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -21,7 +21,7 @@ jobs: extension_name: privacy # Temporary Windows diagnosis: retain the native test output on a crash. exclude_archs: 'linux_amd64;linux_arm64;osx_arm64;windows_amd64_mingw;wasm_eh;wasm_mvp;wasm_threads' - test_config: '{"test_env_variables": {"TEST_RUNNER_SCRIPT": ".ci/use-native-test-runner", "T": "\"test/*\""}}' + test_config: '{"test_env_variables": {"TEST_RUNNER_SCRIPT": ".ci/use-native-test-runner", "T": "\"test/sql/pac_sum.test\""}}' code-quality-check: name: Code Quality Check diff --git a/test/sql/pac_multi_pu.test b/test/sql/pac_multi_pu.test index cef7aede..c65c81a3 100644 --- a/test/sql/pac_multi_pu.test +++ b/test/sql/pac_multi_pu.test @@ -4,9 +4,6 @@ require privacy -# Temporary crash tracing: unlike --output-sql, debug mode executes every query. -mode debug - statement ok PRAGMA clear_privacy_metadata; diff --git a/test/sql/pac_sum.test b/test/sql/pac_sum.test index c3cd83fd..56de668e 100644 --- a/test/sql/pac_sum.test +++ b/test/sql/pac_sum.test @@ -4,6 +4,9 @@ require privacy +# Temporary native crash tracing; every statement still executes and is checked. +mode debug + statement ok PRAGMA clear_privacy_metadata; From 1c65642f807988bba5437be2e6783dc9049c72f0 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 5 Nov 2026 17:01:48 +0100 Subject: [PATCH 15/32] Select SUM spill codecs by input type --- src/aggregates/as_sum.cpp | 2 +- src/aggregates/as_sum_state_export.cpp | 15 ++++++++++++--- src/include/aggregates/as_sum_state_export.hpp | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/aggregates/as_sum.cpp b/src/aggregates/as_sum.cpp index 3c3c5747..2a3fe11d 100644 --- a/src/aggregates/as_sum.cpp +++ b/src/aggregates/as_sum.cpp @@ -734,7 +734,7 @@ void PacSumDoubleInitialize(AggregateStateInput &, data_ptr_t *states, idx_t cou // State export is cold-path metadata; the update kernels and native state sizes are unchanged. static AggregateFunction SpillableSum(AggregateFunction function) { - RegisterSumStateExport(function, function.GetStateInitCallback() == PacSumDoubleInitialize); + RegisterSumStateExport(function); return function; } diff --git a/src/aggregates/as_sum_state_export.cpp b/src/aggregates/as_sum_state_export.cpp index 50e4a1ab..3d788f80 100644 --- a/src/aggregates/as_sum_state_export.cpp +++ b/src/aggregates/as_sum_state_export.cpp @@ -172,15 +172,24 @@ struct SumStateExport> { } // namespace -void RegisterSumStateExport(AggregateFunction &function, bool double_state) { +void RegisterSumStateExport(AggregateFunction &function) { auto value_type = function.GetSignature().GetParameter(1).GetType().id(); - if (double_state) { + if (value_type == LogicalTypeId::FLOAT || value_type == LogicalTypeId::DOUBLE || + value_type == LogicalTypeId::UHUGEINT +#ifndef PAC_EXACTSUM + || value_type == LogicalTypeId::HUGEINT +#endif + ) { AggregateStateExport>::Register(function); } else if (value_type == LogicalTypeId::UTINYINT || value_type == LogicalTypeId::USMALLINT || value_type == LogicalTypeId::UINTEGER || value_type == LogicalTypeId::UBIGINT) { AggregateStateExport, SumStateExport>>::Register(function); - } else { + } else if (value_type == LogicalTypeId::BOOLEAN || value_type == LogicalTypeId::TINYINT || + value_type == LogicalTypeId::SMALLINT || value_type == LogicalTypeId::INTEGER || + value_type == LogicalTypeId::BIGINT || value_type == LogicalTypeId::HUGEINT) { AggregateStateExport, SumStateExport>>::Register(function); + } else { + throw InternalException("Unsupported stochastic SUM state type %s", LogicalType(value_type).ToString()); } } diff --git a/src/include/aggregates/as_sum_state_export.hpp b/src/include/aggregates/as_sum_state_export.hpp index 67a0968f..beb3b2bf 100644 --- a/src/include/aggregates/as_sum_state_export.hpp +++ b/src/include/aggregates/as_sum_state_export.hpp @@ -4,6 +4,6 @@ namespace duckdb { -void RegisterSumStateExport(AggregateFunction &function, bool double_state); +void RegisterSumStateExport(AggregateFunction &function); } // namespace duckdb From 1903bf734c5acf3a72935e69c5f1c0d0e326b001 Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 5 Nov 2026 17:09:19 +0100 Subject: [PATCH 16/32] Document SUM spill codec selection --- src/aggregates/as_sum_state_export.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aggregates/as_sum_state_export.cpp b/src/aggregates/as_sum_state_export.cpp index 3d788f80..719bfd36 100644 --- a/src/aggregates/as_sum_state_export.cpp +++ b/src/aggregates/as_sum_state_export.cpp @@ -173,6 +173,7 @@ struct SumStateExport> { } // namespace void RegisterSumStateExport(AggregateFunction &function) { + // Linkers may fold identical initializer functions, so callback addresses do not identify the state layout. auto value_type = function.GetSignature().GetParameter(1).GetType().id(); if (value_type == LogicalTypeId::FLOAT || value_type == LogicalTypeId::DOUBLE || value_type == LogicalTypeId::UHUGEINT From 2bac886e0a5933826e8304885e452f8a1dfa7cec Mon Sep 17 00:00:00 2001 From: ila Date: Thu, 5 Nov 2026 17:15:58 +0100 Subject: [PATCH 17/32] Restore full CI after Windows spill diagnosis --- .github/workflows/MainDistributionPipeline.yml | 3 --- test/sql/pac_sum.test | 3 --- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 27d6401a..7b01c057 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -19,9 +19,6 @@ jobs: duckdb_version: v2.0.0 ci_tools_version: ebdcc1961780b0ce1272a0d7637dc58447c34885 extension_name: privacy - # Temporary Windows diagnosis: retain the native test output on a crash. - exclude_archs: 'linux_amd64;linux_arm64;osx_arm64;windows_amd64_mingw;wasm_eh;wasm_mvp;wasm_threads' - test_config: '{"test_env_variables": {"TEST_RUNNER_SCRIPT": ".ci/use-native-test-runner", "T": "\"test/sql/pac_sum.test\""}}' code-quality-check: name: Code Quality Check diff --git a/test/sql/pac_sum.test b/test/sql/pac_sum.test index 56de668e..c3cd83fd 100644 --- a/test/sql/pac_sum.test +++ b/test/sql/pac_sum.test @@ -4,9 +4,6 @@ require privacy -# Temporary native crash tracing; every statement still executes and is checked. -mode debug - statement ok PRAGMA clear_privacy_metadata; From d93abce6ea73a28eee33fd1bfd7f78dd4cac9b2c Mon Sep 17 00:00:00 2001 From: ila Date: Tue, 10 Nov 2026 15:05:49 +0100 Subject: [PATCH 18/32] Port main UNION and scalar-subquery checks to DuckDB 2.0 --- src/categorical/pac_categorical_lambdas.cpp | 4 ++-- src/categorical/pac_categorical_rewriter.cpp | 6 +++--- src/compiler/privacy_compiler.cpp | 8 ++++---- src/metadata/privacy_compatibility_check.cpp | 2 +- src/query_processing/pac_projection_propagation.cpp | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/categorical/pac_categorical_lambdas.cpp b/src/categorical/pac_categorical_lambdas.cpp index 8bc311e5..a9ba4090 100644 --- a/src/categorical/pac_categorical_lambdas.cpp +++ b/src/categorical/pac_categorical_lambdas.cpp @@ -6,6 +6,7 @@ // Created by ila on 3/13/26. // #include "categorical/pac_categorical_lambdas.hpp" +#include "privacy_debug.hpp" #include "aggregates/as_aggregate.hpp" #include "utils/privacy_helpers.hpp" #include "duckdb/planner/expression/bound_conjunction_expression.hpp" @@ -166,8 +167,7 @@ CloneForLambdaBody(ClientContext &context, Expression *expr, const unordered_map } else if (expr->GetExpressionType() == ExpressionType::CASE_EXPR) { auto &case_expr = expr->Cast(); if (IsScalarSubqueryWrapper(case_expr)) { - return CloneForLambdaBody(context, case_expr.ElseMutable().get(), pac_binding_map, captures, capture_map, - counter_bindings, struct_type); + PRIVACY_DEBUG_PRINT("Categorical lambda: preserving scalar-subquery cardinality check"); } // Regular CASE - recurse into all branches // Start with original return type, will update based on cloned branches diff --git a/src/categorical/pac_categorical_rewriter.cpp b/src/categorical/pac_categorical_rewriter.cpp index 63a41ba1..9c6d2ab4 100644 --- a/src/categorical/pac_categorical_rewriter.cpp +++ b/src/categorical/pac_categorical_rewriter.cpp @@ -174,16 +174,16 @@ static void PrepareFilterPatternSlot(LogicalProjection &proj, idx_t i, if (expr->GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) { proj.types[i] = list_type; } else if (expr->GetExpressionType() == ExpressionType::BOUND_FUNCTION) { - proj.types[i] = expr->return_type; + proj.types[i] = expr->GetReturnType(); } else if (expr->GetExpressionType() == ExpressionType::CASE_EXPR && IsScalarSubqueryWrapper(expr->Cast())) { // first(value) now returns a counter list, but DuckDB's scalar-subquery // cardinality-check CASE still has its original scalar result type. auto &wrapper = expr->Cast(); - if (wrapper.Else().return_type == list_type) { + if (wrapper.Else().GetReturnType() == list_type) { wrapper.CaseChecksMutable()[0].then_expr = BoundCastExpression::AddDefaultCastToType( std::move(wrapper.CaseChecksMutable()[0].then_expr), list_type); - wrapper.return_type = list_type; + wrapper.SetReturnType(list_type); proj.types[i] = list_type; PRIVACY_DEBUG_PRINT("Categorical rewrite: preserving counter lists through scalar-subquery wrapper"); } diff --git a/src/compiler/privacy_compiler.cpp b/src/compiler/privacy_compiler.cpp index 6e1988c6..2a56bf99 100644 --- a/src/compiler/privacy_compiler.cpp +++ b/src/compiler/privacy_compiler.cpp @@ -564,7 +564,7 @@ class PrivateUnionRewriter { vector> found; CollectMemberships(*agg.children[0], found); if (found.empty()) { - return ColumnBinding(DConstants::INVALID_INDEX, DConstants::INVALID_INDEX); + return ColumnBinding(); } if (found.size() != 1 || ContainsOutsideEntity(*agg.children[0])) { throw InvalidInputException("Private UNION cannot combine independent ownership streams above a UNION"); @@ -638,7 +638,7 @@ class PrivateUnionRewriter { for (idx_t i = 0; i < count; i++) { auto &expr = distinct.distinct_targets[i]; if (i >= old_bindings.size() || expr->GetExpressionType() != ExpressionType::BOUND_COLUMN_REF || - expr->Cast().binding != old_bindings[i]) { + expr->Cast().Binding() != old_bindings[i]) { throw InvalidInputException("Private UNION DISTINCT requires the visible output columns"); } } @@ -647,7 +647,7 @@ class PrivateUnionRewriter { // DuckDB's own DISTINCT modifier above UNION DISTINCT is already satisfied. if (annotation != memberships.end() && child->type == LogicalOperatorType::LOGICAL_PROJECTION && child->children[0]->type == LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY && - annotation->second.column_index.index == count) { + annotation->second.column_index == ProjectionIndex(count)) { op = std::move(op->children[0]); return; } @@ -919,7 +919,7 @@ vector BuildPUHashExpressionsForAggregates(OptimizerExten vector result; for (auto *target_agg : target_aggregates) { auto union_mask = unions.FindForAggregate(*target_agg); - if (union_mask.table_index != DConstants::INVALID_INDEX) { + if (union_mask.table_index.index != DConstants::INVALID_INDEX) { result.push_back( {target_agg, make_uniq(LogicalType::UBIGINT, union_mask), union_mask, 1.0}); continue; diff --git a/src/metadata/privacy_compatibility_check.cpp b/src/metadata/privacy_compatibility_check.cpp index 9daf7787..2ee46c0f 100644 --- a/src/metadata/privacy_compatibility_check.cpp +++ b/src/metadata/privacy_compatibility_check.cpp @@ -103,7 +103,7 @@ bool ContainsUnaggregatedEntityInput(const LogicalOperator &op, const PrivacyCom } if (op.type == LogicalOperatorType::LOGICAL_GET) { auto table = op.Cast().GetTable(); - if (table && (check.fk_paths.count(table->name) || + if (table && (check.fk_paths.count(table->name.GetIdentifierName()) || std::find(check.scanned_pu_tables.begin(), check.scanned_pu_tables.end(), table->name) != check.scanned_pu_tables.end())) { return true; diff --git a/src/query_processing/pac_projection_propagation.cpp b/src/query_processing/pac_projection_propagation.cpp index 91048589..2ae22543 100644 --- a/src/query_processing/pac_projection_propagation.cpp +++ b/src/query_processing/pac_projection_propagation.cpp @@ -47,7 +47,7 @@ static bool FindDirectPathToSource(LogicalOperator *current, idx_t target_table_ return true; } } else if (current->type == LogicalOperatorType::LOGICAL_CTE_REF) { - if (current->Cast().table_index == target_table_index) { + if (current->Cast().table_index.index == target_table_index) { return true; } } else if (current->type == LogicalOperatorType::LOGICAL_PROJECTION) { From 96a264c7fc7e180668832c4d7725028d1baf6a77 Mon Sep 17 00:00:00 2001 From: ila Date: Tue, 10 Nov 2026 15:05:50 +0100 Subject: [PATCH 19/32] Port existing handwritten TPC-DS benchmark and validation SQL to DuckDB 2.0 --- benchmark/configs/utility/as_tpcds_sf30.json | 12 + benchmark/tpcds/as_queries/q01.sql | 23 + benchmark/tpcds/as_queries/q03.sql | 14 + benchmark/tpcds/as_queries/q04.sql | 120 +++++ benchmark/tpcds/as_queries/q06.sql | 26 ++ benchmark/tpcds/as_queries/q07.sql | 24 + benchmark/tpcds/as_queries/q09.sql | 8 + benchmark/tpcds/as_queries/q10.sql | 66 +++ benchmark/tpcds/as_queries/q11.sql | 80 ++++ benchmark/tpcds/as_queries/q13.sql | 43 ++ benchmark/tpcds/as_queries/q15.sql | 28 ++ benchmark/tpcds/as_queries/q16.sql | 24 + benchmark/tpcds/as_queries/q17.sql | 48 ++ benchmark/tpcds/as_queries/q18.sql | 50 +++ benchmark/tpcds/as_queries/q19.sql | 31 ++ benchmark/tpcds/as_queries/q23.sql | 83 ++++ benchmark/tpcds/as_queries/q24.sql | 53 +++ benchmark/tpcds/as_queries/q25.sql | 42 ++ benchmark/tpcds/as_queries/q26.sql | 24 + benchmark/tpcds/as_queries/q27.sql | 61 +++ benchmark/tpcds/as_queries/q28.sql | 45 ++ benchmark/tpcds/as_queries/q29.sql | 42 ++ benchmark/tpcds/as_queries/q30.sql | 49 +++ benchmark/tpcds/as_queries/q31.sql | 69 +++ benchmark/tpcds/as_queries/q32.sql | 22 + benchmark/tpcds/as_queries/q33.sql | 64 +++ benchmark/tpcds/as_queries/q34.sql | 42 ++ benchmark/tpcds/as_queries/q35.sql | 61 +++ benchmark/tpcds/as_queries/q40.sql | 27 ++ benchmark/tpcds/as_queries/q42.sql | 15 + benchmark/tpcds/as_queries/q43.sql | 13 + benchmark/tpcds/as_queries/q45.sql | 42 ++ benchmark/tpcds/as_queries/q46.sql | 48 ++ benchmark/tpcds/as_queries/q48.sql | 40 ++ benchmark/tpcds/as_queries/q50.sql | 68 +++ benchmark/tpcds/as_queries/q52.sql | 15 + benchmark/tpcds/as_queries/q55.sql | 14 + benchmark/tpcds/as_queries/q56.sql | 71 +++ benchmark/tpcds/as_queries/q58.sql | 68 +++ benchmark/tpcds/as_queries/q59.sql | 85 ++++ benchmark/tpcds/as_queries/q60.sql | 65 +++ benchmark/tpcds/as_queries/q61.sql | 44 ++ benchmark/tpcds/as_queries/q62.sql | 46 ++ benchmark/tpcds/as_queries/q64.sql | 124 ++++++ benchmark/tpcds/as_queries/q65.sql | 23 + benchmark/tpcds/as_queries/q66.sql | 221 ++++++++++ benchmark/tpcds/as_queries/q68.sql | 45 ++ benchmark/tpcds/as_queries/q69.sql | 52 +++ benchmark/tpcds/as_queries/q72.sql | 40 ++ benchmark/tpcds/as_queries/q73.sql | 42 ++ benchmark/tpcds/as_queries/q74.sql | 64 +++ benchmark/tpcds/as_queries/q77.sql | 91 ++++ benchmark/tpcds/as_queries/q78.sql | 75 ++++ benchmark/tpcds/as_queries/q79.sql | 39 ++ benchmark/tpcds/as_queries/q80.sql | 85 ++++ benchmark/tpcds/as_queries/q81.sql | 55 +++ benchmark/tpcds/as_queries/q83.sql | 68 +++ benchmark/tpcds/as_queries/q85.sql | 60 +++ benchmark/tpcds/as_queries/q88.sql | 139 ++++++ benchmark/tpcds/as_queries/q90.sql | 26 ++ benchmark/tpcds/as_queries/q91.sql | 31 ++ benchmark/tpcds/as_queries/q92.sql | 22 + benchmark/tpcds/as_queries/q93.sql | 20 + benchmark/tpcds/as_queries/q94.sql | 24 + benchmark/tpcds/as_queries/q95.sql | 30 ++ benchmark/tpcds/as_queries/q96.sql | 15 + benchmark/tpcds/as_queries/q99.sql | 45 ++ benchmark/tpcds/as_tpcds_benchmark.cpp | 438 ++++++++++++++++++- benchmark/tpcds/q17_fixture.sql | 22 + benchmark/tpcds/reference_queries/q01.sql | 30 ++ benchmark/tpcds/reference_queries/q03.sql | 30 ++ benchmark/tpcds/reference_queries/q04.sql | 135 ++++++ benchmark/tpcds/reference_queries/q06.sql | 36 ++ benchmark/tpcds/reference_queries/q07.sql | 38 ++ benchmark/tpcds/reference_queries/q09.sql | 17 + benchmark/tpcds/reference_queries/q10.sql | 59 +++ benchmark/tpcds/reference_queries/q11.sql | 93 ++++ benchmark/tpcds/reference_queries/q13.sql | 55 +++ benchmark/tpcds/reference_queries/q15.sql | 39 ++ benchmark/tpcds/reference_queries/q16.sql | 10 + benchmark/tpcds/reference_queries/q17.sql | 67 +++ benchmark/tpcds/reference_queries/q18.sql | 68 +++ benchmark/tpcds/reference_queries/q19.sql | 41 ++ benchmark/tpcds/reference_queries/q23.sql | 96 ++++ benchmark/tpcds/reference_queries/q24.sql | 58 +++ benchmark/tpcds/reference_queries/q25.sql | 54 +++ benchmark/tpcds/reference_queries/q26.sql | 38 ++ benchmark/tpcds/reference_queries/q27.sql | 25 ++ benchmark/tpcds/reference_queries/q28.sql | 9 + benchmark/tpcds/reference_queries/q29.sql | 55 +++ benchmark/tpcds/reference_queries/q30.sql | 68 +++ benchmark/tpcds/reference_queries/q31.sql | 85 ++++ benchmark/tpcds/reference_queries/q32.sql | 14 + benchmark/tpcds/reference_queries/q33.sql | 90 ++++ benchmark/tpcds/reference_queries/q34.sql | 51 +++ benchmark/tpcds/reference_queries/q35.sql | 111 +++++ benchmark/tpcds/reference_queries/q40.sql | 16 + benchmark/tpcds/reference_queries/q42.sql | 31 ++ benchmark/tpcds/reference_queries/q43.sql | 39 ++ benchmark/tpcds/reference_queries/q45.sql | 53 +++ benchmark/tpcds/reference_queries/q46.sql | 61 +++ benchmark/tpcds/reference_queries/q48.sql | 50 +++ benchmark/tpcds/reference_queries/q50.sql | 65 +++ benchmark/tpcds/reference_queries/q52.sql | 31 ++ benchmark/tpcds/reference_queries/q55.sql | 28 ++ benchmark/tpcds/reference_queries/q56.sql | 98 +++++ benchmark/tpcds/reference_queries/q58.sql | 93 ++++ benchmark/tpcds/reference_queries/q59.sql | 94 ++++ benchmark/tpcds/reference_queries/q60.sql | 92 ++++ benchmark/tpcds/reference_queries/q61.sql | 49 +++ benchmark/tpcds/reference_queries/q62.sql | 43 ++ benchmark/tpcds/reference_queries/q64.sql | 133 ++++++ benchmark/tpcds/reference_queries/q65.sql | 17 + benchmark/tpcds/reference_queries/q66.sql | 227 ++++++++++ benchmark/tpcds/reference_queries/q68.sql | 56 +++ benchmark/tpcds/reference_queries/q69.sql | 49 +++ benchmark/tpcds/reference_queries/q72.sql | 20 + benchmark/tpcds/reference_queries/q73.sql | 50 +++ benchmark/tpcds/reference_queries/q74.sql | 74 ++++ benchmark/tpcds/reference_queries/q77.sql | 114 +++++ benchmark/tpcds/reference_queries/q78.sql | 80 ++++ benchmark/tpcds/reference_queries/q79.sql | 53 +++ benchmark/tpcds/reference_queries/q80.sql | 58 +++ benchmark/tpcds/reference_queries/q81.sql | 77 ++++ benchmark/tpcds/reference_queries/q83.sql | 93 ++++ benchmark/tpcds/reference_queries/q85.sql | 68 +++ benchmark/tpcds/reference_queries/q88.sql | 25 ++ benchmark/tpcds/reference_queries/q90.sql | 30 ++ benchmark/tpcds/reference_queries/q91.sql | 44 ++ benchmark/tpcds/reference_queries/q92.sql | 14 + benchmark/tpcds/reference_queries/q93.sql | 12 + benchmark/tpcds/reference_queries/q94.sql | 10 + benchmark/tpcds/reference_queries/q95.sql | 10 + benchmark/tpcds/reference_queries/q96.sql | 24 + benchmark/tpcds/reference_queries/q99.sql | 42 ++ benchmark/tpcds/run.py | 67 ++- 136 files changed, 7538 insertions(+), 35 deletions(-) create mode 100644 benchmark/configs/utility/as_tpcds_sf30.json create mode 100644 benchmark/tpcds/as_queries/q01.sql create mode 100644 benchmark/tpcds/as_queries/q03.sql create mode 100644 benchmark/tpcds/as_queries/q04.sql create mode 100644 benchmark/tpcds/as_queries/q06.sql create mode 100644 benchmark/tpcds/as_queries/q07.sql create mode 100644 benchmark/tpcds/as_queries/q09.sql create mode 100644 benchmark/tpcds/as_queries/q10.sql create mode 100644 benchmark/tpcds/as_queries/q11.sql create mode 100644 benchmark/tpcds/as_queries/q13.sql create mode 100644 benchmark/tpcds/as_queries/q15.sql create mode 100644 benchmark/tpcds/as_queries/q16.sql create mode 100644 benchmark/tpcds/as_queries/q17.sql create mode 100644 benchmark/tpcds/as_queries/q18.sql create mode 100644 benchmark/tpcds/as_queries/q19.sql create mode 100644 benchmark/tpcds/as_queries/q23.sql create mode 100644 benchmark/tpcds/as_queries/q24.sql create mode 100644 benchmark/tpcds/as_queries/q25.sql create mode 100644 benchmark/tpcds/as_queries/q26.sql create mode 100644 benchmark/tpcds/as_queries/q27.sql create mode 100644 benchmark/tpcds/as_queries/q28.sql create mode 100644 benchmark/tpcds/as_queries/q29.sql create mode 100644 benchmark/tpcds/as_queries/q30.sql create mode 100644 benchmark/tpcds/as_queries/q31.sql create mode 100644 benchmark/tpcds/as_queries/q32.sql create mode 100644 benchmark/tpcds/as_queries/q33.sql create mode 100644 benchmark/tpcds/as_queries/q34.sql create mode 100644 benchmark/tpcds/as_queries/q35.sql create mode 100644 benchmark/tpcds/as_queries/q40.sql create mode 100644 benchmark/tpcds/as_queries/q42.sql create mode 100644 benchmark/tpcds/as_queries/q43.sql create mode 100644 benchmark/tpcds/as_queries/q45.sql create mode 100644 benchmark/tpcds/as_queries/q46.sql create mode 100644 benchmark/tpcds/as_queries/q48.sql create mode 100644 benchmark/tpcds/as_queries/q50.sql create mode 100644 benchmark/tpcds/as_queries/q52.sql create mode 100644 benchmark/tpcds/as_queries/q55.sql create mode 100644 benchmark/tpcds/as_queries/q56.sql create mode 100644 benchmark/tpcds/as_queries/q58.sql create mode 100644 benchmark/tpcds/as_queries/q59.sql create mode 100644 benchmark/tpcds/as_queries/q60.sql create mode 100644 benchmark/tpcds/as_queries/q61.sql create mode 100644 benchmark/tpcds/as_queries/q62.sql create mode 100644 benchmark/tpcds/as_queries/q64.sql create mode 100644 benchmark/tpcds/as_queries/q65.sql create mode 100644 benchmark/tpcds/as_queries/q66.sql create mode 100644 benchmark/tpcds/as_queries/q68.sql create mode 100644 benchmark/tpcds/as_queries/q69.sql create mode 100644 benchmark/tpcds/as_queries/q72.sql create mode 100644 benchmark/tpcds/as_queries/q73.sql create mode 100644 benchmark/tpcds/as_queries/q74.sql create mode 100644 benchmark/tpcds/as_queries/q77.sql create mode 100644 benchmark/tpcds/as_queries/q78.sql create mode 100644 benchmark/tpcds/as_queries/q79.sql create mode 100644 benchmark/tpcds/as_queries/q80.sql create mode 100644 benchmark/tpcds/as_queries/q81.sql create mode 100644 benchmark/tpcds/as_queries/q83.sql create mode 100644 benchmark/tpcds/as_queries/q85.sql create mode 100644 benchmark/tpcds/as_queries/q88.sql create mode 100644 benchmark/tpcds/as_queries/q90.sql create mode 100644 benchmark/tpcds/as_queries/q91.sql create mode 100644 benchmark/tpcds/as_queries/q92.sql create mode 100644 benchmark/tpcds/as_queries/q93.sql create mode 100644 benchmark/tpcds/as_queries/q94.sql create mode 100644 benchmark/tpcds/as_queries/q95.sql create mode 100644 benchmark/tpcds/as_queries/q96.sql create mode 100644 benchmark/tpcds/as_queries/q99.sql create mode 100644 benchmark/tpcds/q17_fixture.sql create mode 100644 benchmark/tpcds/reference_queries/q01.sql create mode 100644 benchmark/tpcds/reference_queries/q03.sql create mode 100644 benchmark/tpcds/reference_queries/q04.sql create mode 100644 benchmark/tpcds/reference_queries/q06.sql create mode 100644 benchmark/tpcds/reference_queries/q07.sql create mode 100644 benchmark/tpcds/reference_queries/q09.sql create mode 100644 benchmark/tpcds/reference_queries/q10.sql create mode 100644 benchmark/tpcds/reference_queries/q11.sql create mode 100644 benchmark/tpcds/reference_queries/q13.sql create mode 100644 benchmark/tpcds/reference_queries/q15.sql create mode 100644 benchmark/tpcds/reference_queries/q16.sql create mode 100644 benchmark/tpcds/reference_queries/q17.sql create mode 100644 benchmark/tpcds/reference_queries/q18.sql create mode 100644 benchmark/tpcds/reference_queries/q19.sql create mode 100644 benchmark/tpcds/reference_queries/q23.sql create mode 100644 benchmark/tpcds/reference_queries/q24.sql create mode 100644 benchmark/tpcds/reference_queries/q25.sql create mode 100644 benchmark/tpcds/reference_queries/q26.sql create mode 100644 benchmark/tpcds/reference_queries/q27.sql create mode 100644 benchmark/tpcds/reference_queries/q28.sql create mode 100644 benchmark/tpcds/reference_queries/q29.sql create mode 100644 benchmark/tpcds/reference_queries/q30.sql create mode 100644 benchmark/tpcds/reference_queries/q31.sql create mode 100644 benchmark/tpcds/reference_queries/q32.sql create mode 100644 benchmark/tpcds/reference_queries/q33.sql create mode 100644 benchmark/tpcds/reference_queries/q34.sql create mode 100644 benchmark/tpcds/reference_queries/q35.sql create mode 100644 benchmark/tpcds/reference_queries/q40.sql create mode 100644 benchmark/tpcds/reference_queries/q42.sql create mode 100644 benchmark/tpcds/reference_queries/q43.sql create mode 100644 benchmark/tpcds/reference_queries/q45.sql create mode 100644 benchmark/tpcds/reference_queries/q46.sql create mode 100644 benchmark/tpcds/reference_queries/q48.sql create mode 100644 benchmark/tpcds/reference_queries/q50.sql create mode 100644 benchmark/tpcds/reference_queries/q52.sql create mode 100644 benchmark/tpcds/reference_queries/q55.sql create mode 100644 benchmark/tpcds/reference_queries/q56.sql create mode 100644 benchmark/tpcds/reference_queries/q58.sql create mode 100644 benchmark/tpcds/reference_queries/q59.sql create mode 100644 benchmark/tpcds/reference_queries/q60.sql create mode 100644 benchmark/tpcds/reference_queries/q61.sql create mode 100644 benchmark/tpcds/reference_queries/q62.sql create mode 100644 benchmark/tpcds/reference_queries/q64.sql create mode 100644 benchmark/tpcds/reference_queries/q65.sql create mode 100644 benchmark/tpcds/reference_queries/q66.sql create mode 100644 benchmark/tpcds/reference_queries/q68.sql create mode 100644 benchmark/tpcds/reference_queries/q69.sql create mode 100644 benchmark/tpcds/reference_queries/q72.sql create mode 100644 benchmark/tpcds/reference_queries/q73.sql create mode 100644 benchmark/tpcds/reference_queries/q74.sql create mode 100644 benchmark/tpcds/reference_queries/q77.sql create mode 100644 benchmark/tpcds/reference_queries/q78.sql create mode 100644 benchmark/tpcds/reference_queries/q79.sql create mode 100644 benchmark/tpcds/reference_queries/q80.sql create mode 100644 benchmark/tpcds/reference_queries/q81.sql create mode 100644 benchmark/tpcds/reference_queries/q83.sql create mode 100644 benchmark/tpcds/reference_queries/q85.sql create mode 100644 benchmark/tpcds/reference_queries/q88.sql create mode 100644 benchmark/tpcds/reference_queries/q90.sql create mode 100644 benchmark/tpcds/reference_queries/q91.sql create mode 100644 benchmark/tpcds/reference_queries/q92.sql create mode 100644 benchmark/tpcds/reference_queries/q93.sql create mode 100644 benchmark/tpcds/reference_queries/q94.sql create mode 100644 benchmark/tpcds/reference_queries/q95.sql create mode 100644 benchmark/tpcds/reference_queries/q96.sql create mode 100644 benchmark/tpcds/reference_queries/q99.sql diff --git a/benchmark/configs/utility/as_tpcds_sf30.json b/benchmark/configs/utility/as_tpcds_sf30.json new file mode 100644 index 00000000..ba422483 --- /dev/null +++ b/benchmark/configs/utility/as_tpcds_sf30.json @@ -0,0 +1,12 @@ +{ + "sf": 30, + "db": "benchmark/results/tpcds_sf30/data.db", + "queries_dir": "benchmark/results/tpcds_sf30/queries", + "out_dir": "benchmark/results/tpcds_sf30", + "schema": "benchmark/tpcds/schema.sql", + "threads": 16, + "memory_limit": "30GB", + "repetitions": 5, + "timeout_seconds": 3600, + "extension_directory": "benchmark/results/tpcds_extensions" +} diff --git a/benchmark/tpcds/as_queries/q01.sql b/benchmark/tpcds/as_queries/q01.sql new file mode 100644 index 00000000..1621b57e --- /dev/null +++ b/benchmark/tpcds/as_queries/q01.sql @@ -0,0 +1,23 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- The CTE exports customer totals unchanged; its correlated AVG is stochastic across customers. +WITH customer_total_return AS (SELECT sr_customer_sk AS ctr_customer_sk, + sr_store_sk AS ctr_store_sk, + sum(sr_return_amt) AS ctr_total_return + FROM store_returns, + date_dim + WHERE sr_returned_date_sk = d_date_sk + AND d_year = 2000 + GROUP BY sr_customer_sk, + sr_store_sk) +SELECT c_customer_id +FROM customer_total_return ctr1, + store, + customer +WHERE priv_filter_gt(CAST(CAST(ctr1.ctr_total_return AS DOUBLE) * 0.8333333333333334 AS FLOAT), (SELECT priv_div(as_sum(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return), as_count(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return)) + FROM customer_total_return ctr2 + WHERE ctr1.ctr_store_sk = ctr2.ctr_store_sk)) + AND s_store_sk = ctr1.ctr_store_sk + AND s_state = 'TN' + AND ctr1.ctr_customer_sk = c_customer_sk +ORDER BY c_customer_id +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q03.sql b/benchmark/tpcds/as_queries/q03.sql new file mode 100644 index 00000000..156d7171 --- /dev/null +++ b/benchmark/tpcds/as_queries/q03.sql @@ -0,0 +1,14 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH asa_states AS (SELECT dt.d_year AS d_year, item.i_brand_id AS brand_id, item.i_brand AS brand, as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_sales_price AS DOUBLE)) AS sum_agg FROM date_dim dt, + store_sales, + item +WHERE dt.d_date_sk = store_sales.ss_sold_date_sk + AND store_sales.ss_item_sk = item.i_item_sk + AND item.i_manufact_id = 128 + AND dt.d_moy=11 +GROUP BY dt.d_year, + item.i_brand, + item.i_brand_id +), +asa_ranked AS (SELECT * FROM asa_states ORDER BY d_year, priv_mean(sum_agg) DESC, brand_id LIMIT 100) +SELECT d_year, brand_id, brand, CAST(as_noised(sum_agg) AS DECIMAL(38,2)) AS sum_agg FROM asa_ranked ORDER BY d_year, sum_agg DESC, brand_id; diff --git a/benchmark/tpcds/as_queries/q04.sql b/benchmark/tpcds/as_queries/q04.sql new file mode 100644 index 00000000..b499a2b7 --- /dev/null +++ b/benchmark/tpcds/as_queries/q04.sql @@ -0,0 +1,120 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Keep year_total scalar inside its CTE; outer customer/year joins retain ordinary PAC scalar semantics. +WITH year_total AS (SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + as_noised_sum(priv_hash(hash(c_customer_sk)), ((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total, + 's' sale_type + FROM customer, + store_sales, + date_dim + WHERE c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year + UNION ALL SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + as_noised_sum(priv_hash(hash(c_customer_sk)), (((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2)) year_total, + 'c' sale_type + FROM customer, + catalog_sales, + date_dim + WHERE c_customer_sk = cs_bill_customer_sk + AND cs_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year + UNION ALL SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + as_noised_sum(priv_hash(hash(c_customer_sk)), (((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2)) year_total, + 'w' sale_type + FROM customer, + web_sales, + date_dim + WHERE c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year) +SELECT t_s_secyear.customer_id, + t_s_secyear.customer_first_name, + t_s_secyear.customer_last_name, + t_s_secyear.customer_preferred_cust_flag +FROM year_total t_s_firstyear, + year_total t_s_secyear, + year_total t_c_firstyear, + year_total t_c_secyear, + year_total t_w_firstyear, + year_total t_w_secyear +WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id + AND t_s_firstyear.customer_id = t_c_secyear.customer_id + AND t_s_firstyear.customer_id = t_c_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_secyear.customer_id + AND t_s_firstyear.sale_type = 's' + AND t_c_firstyear.sale_type = 'c' + AND t_w_firstyear.sale_type = 'w' + AND t_s_secyear.sale_type = 's' + AND t_c_secyear.sale_type = 'c' + AND t_w_secyear.sale_type = 'w' + AND t_s_firstyear.dyear = 2001 + AND t_s_secyear.dyear = 2001+1 + AND t_c_firstyear.dyear = 2001 + AND t_c_secyear.dyear = 2001+1 + AND t_w_firstyear.dyear = 2001 + AND t_w_secyear.dyear = 2001+1 + AND t_s_firstyear.year_total > 0 + AND t_c_firstyear.year_total > 0 + AND t_w_firstyear.year_total > 0 + AND CASE + WHEN t_c_firstyear.year_total > 0 THEN t_c_secyear.year_total / t_c_firstyear.year_total + ELSE NULL + END > CASE + WHEN t_s_firstyear.year_total > 0 THEN t_s_secyear.year_total / t_s_firstyear.year_total + ELSE NULL + END + AND CASE + WHEN t_c_firstyear.year_total > 0 THEN t_c_secyear.year_total / t_c_firstyear.year_total + ELSE NULL + END > CASE + WHEN t_w_firstyear.year_total > 0 THEN t_w_secyear.year_total / t_w_firstyear.year_total + ELSE NULL + END +ORDER BY t_s_secyear.customer_id NULLS FIRST, + t_s_secyear.customer_first_name NULLS FIRST, + t_s_secyear.customer_last_name NULLS FIRST, + t_s_secyear.customer_preferred_cust_flag NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q06.sql b/benchmark/tpcds/as_queries/q06.sql new file mode 100644 index 00000000..849ef217 --- /dev/null +++ b/benchmark/tpcds/as_queries/q06.sql @@ -0,0 +1,26 @@ +-- PAC COUNT with scalar HAVING, as in the compiled plan. +SELECT a.ca_state state, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt +FROM customer_address a , + customer c , + store_sales s , + date_dim d , + item i +WHERE a.ca_address_sk = c.c_current_addr_sk + AND c.c_customer_sk = s.ss_customer_sk + AND s.ss_sold_date_sk = d.d_date_sk + AND s.ss_item_sk = i.i_item_sk + AND d.d_month_seq = + (SELECT DISTINCT (d_month_seq) + FROM date_dim + WHERE d_year = 2001 + AND d_moy = 1 ) + AND i.i_current_price > 1.2 * + (SELECT avg(j.i_current_price) + FROM item j + WHERE j.i_category = i.i_category) +GROUP BY a.ca_state +HAVING as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) >= 10 +ORDER BY cnt NULLS FIRST, + a.ca_state NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q07.sql b/benchmark/tpcds/as_queries/q07.sql new file mode 100644 index 00000000..959198df --- /dev/null +++ b/benchmark/tpcds/as_queries/q07.sql @@ -0,0 +1,24 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT i_item_id, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE)) agg1, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_list_price) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_list_price) AS DOUBLE)) agg2, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) AS DOUBLE)) agg3, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_sales_price) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_sales_price) AS DOUBLE)) agg4 +FROM store_sales, + customer_demographics, + date_dim, + item, + promotion +WHERE ss_sold_date_sk = d_date_sk + AND ss_item_sk = i_item_sk + AND ss_cdemo_sk = cd_demo_sk + AND ss_promo_sk = p_promo_sk + AND cd_gender = 'M' + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND (p_channel_email = 'N' + OR p_channel_event = 'N') + AND d_year = 2000 +GROUP BY i_item_id +ORDER BY i_item_id +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q09.sql b/benchmark/tpcds/as_queries/q09.sql new file mode 100644 index 00000000..352ff3cb --- /dev/null +++ b/benchmark/tpcds/as_queries/q09.sql @@ -0,0 +1,8 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Evaluate CASE lane-wise over counter lists and finalize once; all fifteen scalar subqueries remain. +SELECT CAST(as_noised(list_transform(list_zip((SELECT as_count(priv_hash(hash(ss_customer_sk)),1::BIGINT) FROM store_sales WHERE ss_quantity BETWEEN 1 AND 20), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_discount_amt AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_ext_discount_amt)) FROM store_sales WHERE ss_quantity BETWEEN 1 AND 20), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_net_paid AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_net_paid)) FROM store_sales WHERE ss_quantity BETWEEN 1 AND 20)), lambda x: CASE WHEN x[1] > 74129 THEN x[2] ELSE x[3] END)) AS DOUBLE) AS bucket1, +CAST(as_noised(list_transform(list_zip((SELECT as_count(priv_hash(hash(ss_customer_sk)),1::BIGINT) FROM store_sales WHERE ss_quantity BETWEEN 21 AND 40), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_discount_amt AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_ext_discount_amt)) FROM store_sales WHERE ss_quantity BETWEEN 21 AND 40), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_net_paid AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_net_paid)) FROM store_sales WHERE ss_quantity BETWEEN 21 AND 40)), lambda x: CASE WHEN x[1] > 122840 THEN x[2] ELSE x[3] END)) AS DOUBLE) AS bucket2, +CAST(as_noised(list_transform(list_zip((SELECT as_count(priv_hash(hash(ss_customer_sk)),1::BIGINT) FROM store_sales WHERE ss_quantity BETWEEN 41 AND 60), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_discount_amt AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_ext_discount_amt)) FROM store_sales WHERE ss_quantity BETWEEN 41 AND 60), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_net_paid AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_net_paid)) FROM store_sales WHERE ss_quantity BETWEEN 41 AND 60)), lambda x: CASE WHEN x[1] > 56580 THEN x[2] ELSE x[3] END)) AS DOUBLE) AS bucket3, +CAST(as_noised(list_transform(list_zip((SELECT as_count(priv_hash(hash(ss_customer_sk)),1::BIGINT) FROM store_sales WHERE ss_quantity BETWEEN 61 AND 80), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_discount_amt AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_ext_discount_amt)) FROM store_sales WHERE ss_quantity BETWEEN 61 AND 80), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_net_paid AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_net_paid)) FROM store_sales WHERE ss_quantity BETWEEN 61 AND 80)), lambda x: CASE WHEN x[1] > 10097 THEN x[2] ELSE x[3] END)) AS DOUBLE) AS bucket4, +CAST(as_noised(list_transform(list_zip((SELECT as_count(priv_hash(hash(ss_customer_sk)),1::BIGINT) FROM store_sales WHERE ss_quantity BETWEEN 81 AND 100), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_discount_amt AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_ext_discount_amt)) FROM store_sales WHERE ss_quantity BETWEEN 81 AND 100), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_net_paid AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_net_paid)) FROM store_sales WHERE ss_quantity BETWEEN 81 AND 100)), lambda x: CASE WHEN x[1] > 165306 THEN x[2] ELSE x[3] END)) AS DOUBLE) AS bucket5 +FROM reason WHERE r_reason_sk=1; diff --git a/benchmark/tpcds/as_queries/q10.sql b/benchmark/tpcds/as_queries/q10.sql new file mode 100644 index 00000000..df71aa6a --- /dev/null +++ b/benchmark/tpcds/as_queries/q10.sql @@ -0,0 +1,66 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +SELECT cd_gender, + cd_marital_status, + cd_education_status, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt1, + cd_purchase_estimate, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt2, + cd_credit_rating, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt3, + cd_dep_count, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt4, + cd_dep_employed_count, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt5, + cd_dep_college_count, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt6 +FROM customer c, + customer_address ca, + customer_demographics +WHERE c.c_current_addr_sk = ca.ca_address_sk + AND ca_county IN ('Rush County', + 'Toole County', + 'Jefferson County', + 'Dona Ana County', + 'La Porte County') + AND cd_demo_sk = c.c_current_cdemo_sk + AND EXISTS + (SELECT * + FROM store_sales, + date_dim + WHERE c.c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_moy BETWEEN 1 AND 1+3) + AND (EXISTS + (SELECT * + FROM web_sales, + date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_moy BETWEEN 1 AND 1+3) + OR EXISTS + (SELECT * + FROM catalog_sales, + date_dim + WHERE c.c_customer_sk = cs_ship_customer_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_moy BETWEEN 1 AND 1+3)) +GROUP BY cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count +ORDER BY cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q11.sql b/benchmark/tpcds/as_queries/q11.sql new file mode 100644 index 00000000..c7067279 --- /dev/null +++ b/benchmark/tpcds/as_queries/q11.sql @@ -0,0 +1,80 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH year_total AS (SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + as_noised_sum(priv_hash(hash(c_customer_sk)), ss_ext_list_price-ss_ext_discount_amt) year_total, + 's' sale_type + FROM customer, + store_sales, + date_dim + WHERE c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year + UNION ALL SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + as_noised_sum(priv_hash(hash(c_customer_sk)), ws_ext_list_price-ws_ext_discount_amt) year_total, + 'w' sale_type + FROM customer, + web_sales, + date_dim + WHERE c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year) +SELECT t_s_secyear.customer_id, + t_s_secyear.customer_first_name, + t_s_secyear.customer_last_name, + t_s_secyear.customer_preferred_cust_flag +FROM year_total t_s_firstyear, + year_total t_s_secyear, + year_total t_w_firstyear, + year_total t_w_secyear +WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_secyear.customer_id + AND t_s_firstyear.customer_id = t_w_firstyear.customer_id + AND t_s_firstyear.sale_type = 's' + AND t_w_firstyear.sale_type = 'w' + AND t_s_secyear.sale_type = 's' + AND t_w_secyear.sale_type = 'w' + AND t_s_firstyear.dyear = 2001 + AND t_s_secyear.dyear = 2001+1 + AND t_w_firstyear.dyear = 2001 + AND t_w_secyear.dyear = 2001+1 + AND t_s_firstyear.year_total > 0 + AND t_w_firstyear.year_total > 0 + AND CASE + WHEN t_w_firstyear.year_total > 0 THEN (t_w_secyear.year_total*1.0000) / t_w_firstyear.year_total + ELSE 0.0 + END > CASE + WHEN t_s_firstyear.year_total > 0 THEN (t_s_secyear.year_total*1.0000) / t_s_firstyear.year_total + ELSE 0.0 + END +ORDER BY t_s_secyear.customer_id NULLS FIRST, + t_s_secyear.customer_first_name NULLS FIRST, + t_s_secyear.customer_last_name NULLS FIRST, + t_s_secyear.customer_preferred_cust_flag NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q13.sql b/benchmark/tpcds/as_queries/q13.sql new file mode 100644 index 00000000..d666e013 --- /dev/null +++ b/benchmark/tpcds/as_queries/q13.sql @@ -0,0 +1,43 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. + +SELECT (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE)) avg1, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS DOUBLE)) avg2, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_wholesale_cost) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_ext_wholesale_cost) AS DOUBLE)) avg3, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_wholesale_cost) +FROM store_sales , + store , + customer_demographics , + household_demographics , + customer_address , + date_dim +WHERE s_store_sk = ss_store_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2001 and((ss_hdemo_sk=hd_demo_sk + AND cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'M' + AND cd_education_status = 'Advanced Degree' + AND ss_sales_price BETWEEN 100.00 AND 150.00 + AND hd_dep_count = 3) + OR (ss_hdemo_sk=hd_demo_sk + AND cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND ss_sales_price BETWEEN 50.00 AND 100.00 + AND hd_dep_count = 1 ) + OR (ss_hdemo_sk=hd_demo_sk + AND cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'W' + AND cd_education_status = '2 yr Degree' + AND ss_sales_price BETWEEN 150.00 AND 200.00 + AND hd_dep_count = 1)) and((ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('TX', 'OH', 'TX') + AND ss_net_profit BETWEEN 100 AND 200) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('OR', 'NM', 'KY') + AND ss_net_profit BETWEEN 150 AND 300) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('VA', 'TX', 'MS') + AND ss_net_profit BETWEEN 50 AND 250)) ; diff --git a/benchmark/tpcds/as_queries/q15.sql b/benchmark/tpcds/as_queries/q15.sql new file mode 100644 index 00000000..7d7198f6 --- /dev/null +++ b/benchmark/tpcds/as_queries/q15.sql @@ -0,0 +1,28 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT ca_zip, + as_noised_sum(priv_hash(hash(c_customer_sk)), cs_sales_price) +FROM catalog_sales, + customer, + customer_address, + date_dim +WHERE cs_bill_customer_sk = c_customer_sk + AND c_current_addr_sk = ca_address_sk + AND (SUBSTRING(ca_zip, 1, 5) IN ('85669', + '86197', + '88274', + '83405', + '86475', + '85392', + '85460', + '80348', + '81792') + OR ca_state IN ('CA', + 'WA', + 'GA') + OR cs_sales_price > 500) + AND cs_sold_date_sk = d_date_sk + AND d_qoy = 2 + AND d_year = 2001 +GROUP BY ca_zip +ORDER BY ca_zip NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q16.sql b/benchmark/tpcds/as_queries/q16.sql new file mode 100644 index 00000000..757dcb1f --- /dev/null +++ b/benchmark/tpcds/as_queries/q16.sql @@ -0,0 +1,24 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- COUNT(DISTINCT) unions masks per order before counting; SUMs retain their own source scan. +WITH asa_input AS NOT MATERIALIZED (SELECT cs_order_number, cs_ext_ship_cost, cs_net_profit, priv_hash(hash(cs1.cs_bill_customer_sk)) AS membership FROM catalog_sales cs1, + date_dim, + customer_address, + call_center +WHERE d_date BETWEEN '2002-02-01' AND cast('2002-04-02' AS date) + AND cs1.cs_ship_date_sk = d_date_sk + AND cs1.cs_ship_addr_sk = ca_address_sk + AND ca_state = 'GA' + AND cs1.cs_call_center_sk = cc_call_center_sk + AND cc_county = 'Williamson County' + AND EXISTS + (SELECT * + FROM catalog_sales cs2 + WHERE cs1.cs_order_number = cs2.cs_order_number + AND cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) + AND NOT EXISTS + (SELECT * + FROM catalog_returns cr1 + WHERE cs1.cs_order_number = cr1.cr_order_number) +), +asa_distinct AS NOT MATERIALIZED (SELECT cs_order_number, bit_or(membership) FILTER (WHERE cs_order_number IS NOT NULL) AS membership FROM asa_input GROUP BY cs_order_number) +SELECT d.n AS "order count", s.shipping AS "total shipping cost", s.profit AS "total net profit" FROM (SELECT as_noised_count(membership, 1::BIGINT) AS n FROM asa_distinct) d CROSS JOIN (SELECT as_noised_sum(membership, cs_ext_ship_cost) AS shipping, as_noised_sum(membership, cs_net_profit) AS profit FROM asa_input) s ORDER BY "order count" LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q17.sql b/benchmark/tpcds/as_queries/q17.sql new file mode 100644 index 00000000..ea247a6d --- /dev/null +++ b/benchmark/tpcds/as_queries/q17.sql @@ -0,0 +1,48 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT i_item_id, + i_item_desc, + s_state, + as_noised_count(priv_hash(hash(ss_customer_sk)), ss_quantity) AS store_sales_quantitycount, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE)) AS store_sales_quantityave, + as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), ss_quantity) AS store_sales_quantitystdev, + as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), ss_quantity)/(CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE)) AS store_sales_quantitycov, + as_noised_count(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS store_returns_quantitycount, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS DOUBLE)) AS store_returns_quantityave, + as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS store_returns_quantitystdev, + as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), sr_return_quantity)/(CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS DOUBLE)) AS store_returns_quantitycov, + as_noised_count(priv_hash(hash(ss_customer_sk)), cs_quantity) AS catalog_sales_quantitycount, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), cs_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), cs_quantity) AS DOUBLE)) AS catalog_sales_quantityave, + as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), cs_quantity) AS catalog_sales_quantitystdev, + as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), cs_quantity)/(CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), cs_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), cs_quantity) AS DOUBLE)) AS catalog_sales_quantitycov +FROM store_sales, + store_returns, + catalog_sales, + date_dim d1, + date_dim d2, + date_dim d3, + store, + item +WHERE d1.d_quarter_name = '2001Q1' + AND d1.d_date_sk = ss_sold_date_sk + AND i_item_sk = ss_item_sk + AND s_store_sk = ss_store_sk + AND ss_customer_sk = sr_customer_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND sr_returned_date_sk = d2.d_date_sk + AND d2.d_quarter_name IN ('2001Q1', + '2001Q2', + '2001Q3') + AND sr_customer_sk = cs_bill_customer_sk + AND sr_item_sk = cs_item_sk + AND cs_sold_date_sk = d3.d_date_sk + AND d3.d_quarter_name IN ('2001Q1', + '2001Q2', + '2001Q3') +GROUP BY i_item_id, + i_item_desc, + s_state +ORDER BY i_item_id NULLS FIRST, + i_item_desc NULLS FIRST, + s_state NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q18.sql b/benchmark/tpcds/as_queries/q18.sql new file mode 100644 index 00000000..c04011b9 --- /dev/null +++ b/benchmark/tpcds/as_queries/q18.sql @@ -0,0 +1,50 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT i_item_id, + ca_country, + ca_state, + ca_county, + (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cs_quantity AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cs_quantity AS decimal(12, 2))) AS DOUBLE)) agg1, + (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cs_list_price AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cs_list_price AS decimal(12, 2))) AS DOUBLE)) agg2, + (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cs_coupon_amt AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cs_coupon_amt AS decimal(12, 2))) AS DOUBLE)) agg3, + (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cs_sales_price AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cs_sales_price AS decimal(12, 2))) AS DOUBLE)) agg4, + (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cs_net_profit AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cs_net_profit AS decimal(12, 2))) AS DOUBLE)) agg5, + (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(c_birth_year AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(c_birth_year AS decimal(12, 2))) AS DOUBLE)) agg6, + (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cd1.cd_dep_count AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cd1.cd_dep_count AS decimal(12, 2))) AS DOUBLE)) agg7 +FROM catalog_sales, + customer_demographics cd1, + customer_demographics cd2, + customer, + customer_address, + date_dim, + item +WHERE cs_sold_date_sk = d_date_sk + AND cs_item_sk = i_item_sk + AND cs_bill_cdemo_sk = cd1.cd_demo_sk + AND cs_bill_customer_sk = c_customer_sk + AND cd1.cd_gender = 'F' + AND cd1.cd_education_status = 'Unknown' + AND c_current_cdemo_sk = cd2.cd_demo_sk + AND c_current_addr_sk = ca_address_sk + AND c_birth_month IN (1, + 6, + 8, + 9, + 12, + 2) + AND d_year = 1998 + AND ca_state IN ('MS', + 'IN', + 'ND', + 'OK', + 'NM', + 'VA', + 'MS') +GROUP BY ROLLUP (i_item_id, + ca_country, + ca_state, + ca_county) +ORDER BY ca_country NULLS FIRST, + ca_state NULLS FIRST, + ca_county NULLS FIRST, + i_item_id NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q19.sql b/benchmark/tpcds/as_queries/q19.sql new file mode 100644 index 00000000..1488e632 --- /dev/null +++ b/benchmark/tpcds/as_queries/q19.sql @@ -0,0 +1,31 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT i_brand_id brand_id, + i_brand brand, + i_manufact_id, + i_manufact, + as_noised_sum(priv_hash(hash(c_customer_sk)), ss_ext_sales_price) ext_price +FROM date_dim, + store_sales, + item, + customer, + customer_address, + store +WHERE d_date_sk = ss_sold_date_sk + AND ss_item_sk = i_item_sk + AND i_manager_id=8 + AND d_moy=11 + AND d_year=1998 + AND ss_customer_sk = c_customer_sk + AND c_current_addr_sk = ca_address_sk + AND SUBSTRING(ca_zip, 1, 5) <> SUBSTRING(s_zip, 1, 5) + AND ss_store_sk = s_store_sk +GROUP BY i_brand, + i_brand_id, + i_manufact_id, + i_manufact +ORDER BY ext_price DESC, + i_brand, + i_brand_id, + i_manufact_id, + i_manufact +LIMIT 100 ; diff --git a/benchmark/tpcds/as_queries/q23.sql b/benchmark/tpcds/as_queries/q23.sql new file mode 100644 index 00000000..ffebd1d1 --- /dev/null +++ b/benchmark/tpcds/as_queries/q23.sql @@ -0,0 +1,83 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Per-customer inner totals remain ordinary; the aggregate across those totals is stochastic. +WITH frequent_ss_items AS (SELECT itemdesc, + i_item_sk item_sk, + d_date solddate, + as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) cnt + FROM store_sales, + date_dim, + (SELECT SUBSTRING(i_item_desc, 1, 30) itemdesc, + * + FROM item) sq1 + WHERE ss_sold_date_sk = d_date_sk + AND ss_item_sk = i_item_sk + AND d_year IN (2000, + 2000+1, + 2000+2, + 2000+3) + GROUP BY itemdesc, + i_item_sk, + d_date + HAVING as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) >4), +max_store_sales AS (SELECT as_noised_max(priv_hash(hash(c_customer_sk)), csales) tpcds_cmax + FROM + (SELECT c_customer_sk, + sum(ss_quantity*ss_sales_price) csales + FROM store_sales, + customer, + date_dim + WHERE ss_customer_sk = c_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year IN (2000, + 2000+1, + 2000+2, + 2000+3) + GROUP BY c_customer_sk) sq2), +best_ss_customer AS (SELECT c_customer_sk, + as_noised_sum(priv_hash(hash(c_customer_sk)), ss_quantity*ss_sales_price) ssales + FROM store_sales, + customer, + max_store_sales + WHERE ss_customer_sk = c_customer_sk + GROUP BY c_customer_sk + HAVING as_noised_sum(priv_hash(hash(c_customer_sk)), ss_quantity*ss_sales_price) > (50/100.0) * as_noised_max(priv_hash(hash(c_customer_sk)), tpcds_cmax)) +SELECT c_last_name, + c_first_name, + sales +FROM + (SELECT c_last_name, + c_first_name, + as_noised_sum(priv_hash(hash(customer.c_customer_sk)), cs_quantity*cs_list_price) sales + FROM catalog_sales, + customer, + date_dim, + frequent_ss_items, + best_ss_customer + WHERE d_year = 2000 + AND d_moy = 2 + AND cs_sold_date_sk = d_date_sk + AND cs_item_sk = item_sk + AND cs_bill_customer_sk = best_ss_customer.c_customer_sk + AND cs_bill_customer_sk = customer.c_customer_sk + GROUP BY c_last_name, + c_first_name + UNION ALL SELECT c_last_name, + c_first_name, + as_noised_sum(priv_hash(hash(customer.c_customer_sk)), ws_quantity*ws_list_price) sales + FROM web_sales, + customer, + date_dim, + frequent_ss_items, + best_ss_customer + WHERE d_year = 2000 + AND d_moy = 2 + AND ws_sold_date_sk = d_date_sk + AND ws_item_sk = item_sk + AND ws_bill_customer_sk = best_ss_customer.c_customer_sk + AND ws_bill_customer_sk = customer.c_customer_sk + GROUP BY c_last_name, + c_first_name) sq3 +ORDER BY c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + sales NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q24.sql b/benchmark/tpcds/as_queries/q24.sql new file mode 100644 index 00000000..7e61d583 --- /dev/null +++ b/benchmark/tpcds/as_queries/q24.sql @@ -0,0 +1,53 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- The ssales CTE scalarizes its SUM; later SUM/AVG and HAVING operate on those scalar values. +WITH ssales AS (SELECT c_last_name, + c_first_name, + s_store_name, + ca_state, + s_state, + i_color, + i_current_price, + i_manager_id, + i_units, + i_size, + as_noised_sum(priv_hash(hash(c_customer_sk)), ss_net_paid) netpaid + FROM store_sales, + store_returns, + store, + item, + customer, + customer_address + WHERE ss_ticket_number = sr_ticket_number + AND ss_item_sk = sr_item_sk + AND ss_customer_sk = c_customer_sk + AND ss_item_sk = i_item_sk + AND ss_store_sk = s_store_sk + AND c_current_addr_sk = ca_address_sk + AND c_birth_country <> upper(ca_country) + AND s_zip = ca_zip + AND s_market_id=8 + GROUP BY c_last_name, + c_first_name, + s_store_name, + ca_state, + s_state, + i_color, + i_current_price, + i_manager_id, + i_units, + i_size) +SELECT c_last_name, + c_first_name, + s_store_name, + sum(netpaid) paid +FROM ssales +WHERE i_color = 'peach' +GROUP BY c_last_name, + c_first_name, + s_store_name +HAVING sum(netpaid) > + (SELECT 0.05*avg(netpaid) + FROM ssales) +ORDER BY c_last_name, + c_first_name, + s_store_name ; diff --git a/benchmark/tpcds/as_queries/q25.sql b/benchmark/tpcds/as_queries/q25.sql new file mode 100644 index 00000000..b4e038e6 --- /dev/null +++ b/benchmark/tpcds/as_queries/q25.sql @@ -0,0 +1,42 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. + +SELECT i_item_id , + i_item_desc , + s_store_id , + s_store_name , + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) AS store_sales_profit , + as_noised_sum(priv_hash(hash(ss_customer_sk)), sr_net_loss) AS store_returns_loss , + as_noised_sum(priv_hash(hash(ss_customer_sk)), cs_net_profit) AS catalog_sales_profit +FROM store_sales , + store_returns , + catalog_sales , + date_dim d1 , + date_dim d2 , + date_dim d3 , + store , + item +WHERE d1.d_moy = 4 + AND d1.d_year = 2001 + AND d1.d_date_sk = ss_sold_date_sk + AND i_item_sk = ss_item_sk + AND s_store_sk = ss_store_sk + AND ss_customer_sk = sr_customer_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND sr_returned_date_sk = d2.d_date_sk + AND d2.d_moy BETWEEN 4 AND 10 + AND d2.d_year = 2001 + AND sr_customer_sk = cs_bill_customer_sk + AND sr_item_sk = cs_item_sk + AND cs_sold_date_sk = d3.d_date_sk + AND d3.d_moy BETWEEN 4 AND 10 + AND d3.d_year = 2001 +GROUP BY i_item_id , + i_item_desc , + s_store_id , + s_store_name +ORDER BY i_item_id , + i_item_desc , + s_store_id , + s_store_name +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q26.sql b/benchmark/tpcds/as_queries/q26.sql new file mode 100644 index 00000000..36b06fce --- /dev/null +++ b/benchmark/tpcds/as_queries/q26.sql @@ -0,0 +1,24 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT i_item_id, + (CAST(as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(cs_bill_customer_sk)), cs_quantity) AS DOUBLE)) agg1, + (CAST(as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_list_price) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(cs_bill_customer_sk)), cs_list_price) AS DOUBLE)) agg2, + (CAST(as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_coupon_amt) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(cs_bill_customer_sk)), cs_coupon_amt) AS DOUBLE)) agg3, + (CAST(as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_sales_price) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(cs_bill_customer_sk)), cs_sales_price) AS DOUBLE)) agg4 +FROM catalog_sales, + customer_demographics, + date_dim, + item, + promotion +WHERE cs_sold_date_sk = d_date_sk + AND cs_item_sk = i_item_sk + AND cs_bill_cdemo_sk = cd_demo_sk + AND cs_promo_sk = p_promo_sk + AND cd_gender = 'M' + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND (p_channel_email = 'N' + OR p_channel_event = 'N') + AND d_year = 2000 +GROUP BY i_item_id +ORDER BY i_item_id +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q27.sql b/benchmark/tpcds/as_queries/q27.sql new file mode 100644 index 00000000..93f09123 --- /dev/null +++ b/benchmark/tpcds/as_queries/q27.sql @@ -0,0 +1,61 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Carry the source customer key through the row-level CTE into all three UNION aggregate scopes. +WITH results AS (SELECT ss_customer_sk, i_item_id, + s_state, + 0 AS g_state, + ss_quantity agg1, + ss_list_price agg2, + ss_coupon_amt agg3, + ss_sales_price agg4 + FROM store_sales, + customer_demographics, + date_dim, + store, + item + WHERE ss_sold_date_sk = d_date_sk + AND ss_item_sk = i_item_sk + AND ss_store_sk = s_store_sk + AND ss_cdemo_sk = cd_demo_sk + AND cd_gender = 'M' + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND d_year = 2002 + AND s_state = 'TN' ) +SELECT i_item_id, + s_state, + g_state, + agg1, + agg2, + agg3, + agg4 +FROM + ( SELECT i_item_id, + s_state, + 0 AS g_state, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE)) agg1, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE)) agg2, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE)) agg3, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE)) agg4 + FROM results + GROUP BY i_item_id , + s_state + UNION ALL SELECT i_item_id, + NULL AS s_state, + 1 AS g_state, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE)) agg1, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE)) agg2, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE)) agg3, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE)) agg4 + FROM results + GROUP BY i_item_id + UNION ALL SELECT NULL AS i_item_id, + NULL AS s_state, + 1 AS g_state, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE)) agg1, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE)) agg2, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE)) agg3, + (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE)) agg4 + FROM results ) foo +ORDER BY i_item_id NULLS FIRST, + s_state NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q28.sql b/benchmark/tpcds/as_queries/q28.sql new file mode 100644 index 00000000..2a5d29f8 --- /dev/null +++ b/benchmark/tpcds/as_queries/q28.sql @@ -0,0 +1,45 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Each bucket has independent scalar aggregates and a mask-union DISTINCT-count branch. +WITH b1_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales + WHERE ss_quantity BETWEEN 0 AND 5 + AND (ss_list_price BETWEEN 8 AND 8+10 + OR ss_coupon_amt BETWEEN 459 AND 459+1000 + OR ss_wholesale_cost BETWEEN 57 AND 57+20)), +b1_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b1_input GROUP BY ss_list_price), +b1 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B1_LP, as_noised_count(membership,ss_list_price) AS B1_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b1_unique) AS B1_CNTD FROM b1_input), +b2_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales + WHERE ss_quantity BETWEEN 6 AND 10 + AND (ss_list_price BETWEEN 90 AND 90+10 + OR ss_coupon_amt BETWEEN 2323 AND 2323+1000 + OR ss_wholesale_cost BETWEEN 31 AND 31+20)), +b2_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b2_input GROUP BY ss_list_price), +b2 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B2_LP, as_noised_count(membership,ss_list_price) AS B2_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b2_unique) AS B2_CNTD FROM b2_input), +b3_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales + WHERE ss_quantity BETWEEN 11 AND 15 + AND (ss_list_price BETWEEN 142 AND 142+10 + OR ss_coupon_amt BETWEEN 12214 AND 12214+1000 + OR ss_wholesale_cost BETWEEN 79 AND 79+20)), +b3_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b3_input GROUP BY ss_list_price), +b3 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B3_LP, as_noised_count(membership,ss_list_price) AS B3_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b3_unique) AS B3_CNTD FROM b3_input), +b4_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales + WHERE ss_quantity BETWEEN 16 AND 20 + AND (ss_list_price BETWEEN 135 AND 135+10 + OR ss_coupon_amt BETWEEN 6071 AND 6071+1000 + OR ss_wholesale_cost BETWEEN 38 AND 38+20)), +b4_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b4_input GROUP BY ss_list_price), +b4 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B4_LP, as_noised_count(membership,ss_list_price) AS B4_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b4_unique) AS B4_CNTD FROM b4_input), +b5_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales + WHERE ss_quantity BETWEEN 21 AND 25 + AND (ss_list_price BETWEEN 122 AND 122+10 + OR ss_coupon_amt BETWEEN 836 AND 836+1000 + OR ss_wholesale_cost BETWEEN 17 AND 17+20)), +b5_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b5_input GROUP BY ss_list_price), +b5 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B5_LP, as_noised_count(membership,ss_list_price) AS B5_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b5_unique) AS B5_CNTD FROM b5_input), +b6_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales + WHERE ss_quantity BETWEEN 26 AND 30 + AND (ss_list_price BETWEEN 154 AND 154+10 + OR ss_coupon_amt BETWEEN 7326 AND 7326+1000 + OR ss_wholesale_cost BETWEEN 7 AND 7+20)), +b6_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b6_input GROUP BY ss_list_price), +b6 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B6_LP, as_noised_count(membership,ss_list_price) AS B6_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b6_unique) AS B6_CNTD FROM b6_input) +SELECT * FROM b1, b2, b3, b4, b5, b6 LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q29.sql b/benchmark/tpcds/as_queries/q29.sql new file mode 100644 index 00000000..a58132ca --- /dev/null +++ b/benchmark/tpcds/as_queries/q29.sql @@ -0,0 +1,42 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT i_item_id, + i_item_desc, + s_store_id, + s_store_name, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) AS store_sales_quantity, + as_noised_sum(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS store_returns_quantity, + as_noised_sum(priv_hash(hash(ss_customer_sk)), cs_quantity) AS catalog_sales_quantity +FROM store_sales, + store_returns, + catalog_sales, + date_dim d1, + date_dim d2, + date_dim d3, + store, + item +WHERE d1.d_moy = 9 + AND d1.d_year = 1999 + AND d1.d_date_sk = ss_sold_date_sk + AND i_item_sk = ss_item_sk + AND s_store_sk = ss_store_sk + AND ss_customer_sk = sr_customer_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND sr_returned_date_sk = d2.d_date_sk + AND d2.d_moy BETWEEN 9 AND 9 + 3 + AND d2.d_year = 1999 + AND sr_customer_sk = cs_bill_customer_sk + AND sr_item_sk = cs_item_sk + AND cs_sold_date_sk = d3.d_date_sk + AND d3.d_year IN (1999, + 1999+1, + 1999+2) +GROUP BY i_item_id, + i_item_desc, + s_store_id, + s_store_name +ORDER BY i_item_id, + i_item_desc, + s_store_id, + s_store_name +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q30.sql b/benchmark/tpcds/as_queries/q30.sql new file mode 100644 index 00000000..7c835ab5 --- /dev/null +++ b/benchmark/tpcds/as_queries/q30.sql @@ -0,0 +1,49 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Per-customer CTE totals stay scalar; only the correlated cross-customer AVG supplies a counter list. +WITH customer_total_return AS (SELECT wr_returning_customer_sk AS ctr_customer_sk, + ca_state AS ctr_state, + sum(wr_return_amt) AS ctr_total_return + FROM web_returns, + date_dim, + customer_address + WHERE wr_returned_date_sk = d_date_sk + AND d_year = 2002 + AND wr_returning_addr_sk = ca_address_sk + GROUP BY wr_returning_customer_sk, + ca_state) +SELECT c_customer_id, + c_salutation, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_day, + c_birth_month, + c_birth_year, + c_birth_country, + c_login, + c_email_address, + c_last_review_date_sk, + ctr_total_return +FROM customer_total_return ctr1, + customer_address, + customer +WHERE priv_filter_gt(CAST(CAST(ctr1.ctr_total_return AS DOUBLE) * 0.8333333333333334 AS FLOAT), (SELECT priv_div(as_sum(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return), as_count(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return)) + FROM customer_total_return ctr2 + WHERE ctr1.ctr_state = ctr2.ctr_state)) + AND ca_address_sk = c_current_addr_sk + AND ca_state = 'GA' + AND ctr1.ctr_customer_sk = c_customer_sk +ORDER BY c_customer_id NULLS FIRST, + c_salutation NULLS FIRST, + c_first_name NULLS FIRST, + c_last_name NULLS FIRST, + c_preferred_cust_flag NULLS FIRST, + c_birth_day NULLS FIRST, + c_birth_month NULLS FIRST, + c_birth_year NULLS FIRST, + c_birth_country NULLS FIRST, + c_login NULLS FIRST, + c_email_address NULLS FIRST, + c_last_review_date_sk NULLS FIRST, + ctr_total_return NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q31.sql b/benchmark/tpcds/as_queries/q31.sql new file mode 100644 index 00000000..a232e042 --- /dev/null +++ b/benchmark/tpcds/as_queries/q31.sql @@ -0,0 +1,69 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH ss AS (SELECT ca_county, + d_qoy, + d_year, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS store_sales + FROM store_sales, + date_dim, + customer_address + WHERE ss_sold_date_sk = d_date_sk + AND ss_addr_sk=ca_address_sk + GROUP BY ca_county, + d_qoy, + d_year), +ws AS (SELECT ca_county, + d_qoy, + d_year, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) AS web_sales + FROM web_sales, + date_dim, + customer_address + WHERE ws_sold_date_sk = d_date_sk + AND ws_bill_addr_sk=ca_address_sk + GROUP BY ca_county, + d_qoy, + d_year) +SELECT ss1.ca_county , + ss1.d_year , + (ws2.web_sales*1.0000)/ws1.web_sales web_q1_q2_increase , + (ss2.store_sales*1.0000)/ss1.store_sales store_q1_q2_increase , + (ws3.web_sales*1.0000)/ws2.web_sales web_q2_q3_increase , + (ss3.store_sales*1.0000)/ss2.store_sales store_q2_q3_increase +FROM ss ss1 , + ss ss2 , + ss ss3 , + ws ws1 , + ws ws2 , + ws ws3 +WHERE ss1.d_qoy = 1 + AND ss1.d_year = 2000 + AND ss1.ca_county = ss2.ca_county + AND ss2.d_qoy = 2 + AND ss2.d_year = 2000 + AND ss2.ca_county = ss3.ca_county + AND ss3.d_qoy = 3 + AND ss3.d_year = 2000 + AND ss1.ca_county = ws1.ca_county + AND ws1.d_qoy = 1 + AND ws1.d_year = 2000 + AND ws1.ca_county = ws2.ca_county + AND ws2.d_qoy = 2 + AND ws2.d_year = 2000 + AND ws1.ca_county = ws3.ca_county + AND ws3.d_qoy = 3 + AND ws3.d_year = 2000 + AND CASE + WHEN ws1.web_sales > 0 THEN (ws2.web_sales*1.0000)/ws1.web_sales + ELSE NULL + END > CASE + WHEN ss1.store_sales > 0 THEN (ss2.store_sales*1.0000)/ss1.store_sales + ELSE NULL + END + AND CASE + WHEN ws2.web_sales > 0 THEN (ws3.web_sales*1.0000)/ws2.web_sales + ELSE NULL + END > CASE + WHEN ss2.store_sales > 0 THEN (ss3.store_sales*1.0000)/ss2.store_sales + ELSE NULL + END +ORDER BY ss1.ca_county; diff --git a/benchmark/tpcds/as_queries/q32.sql b/benchmark/tpcds/as_queries/q32.sql new file mode 100644 index 00000000..9c4ae6b7 --- /dev/null +++ b/benchmark/tpcds/as_queries/q32.sql @@ -0,0 +1,22 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- The correlated threshold narrows the tuple mask before the outer stochastic SUM. +SELECT as_noised_sum(priv_select_gt(priv_hash(hash(cs_bill_customer_sk)), CAST(cs_ext_discount_amt AS FLOAT), ( SELECT list_transform(priv_div(as_sum(priv_hash(hash(cs_bill_customer_sk)), CAST(cs_ext_discount_amt AS DOUBLE)), as_count(priv_hash(hash(cs_bill_customer_sk)), cs_ext_discount_amt)), lambda x: CAST(1.3 * x AS FLOAT)) + FROM catalog_sales , + date_dim + WHERE cs_item_sk = i_item_sk + AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) + AND d_date_sk = cs_sold_date_sk )), cs_ext_discount_amt) AS "excess discount amount" +FROM catalog_sales , + item , + date_dim +WHERE i_manufact_id = 977 + AND i_item_sk = cs_item_sk + AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) + AND d_date_sk = cs_sold_date_sk + AND priv_select_gt(priv_hash(hash(cs_bill_customer_sk)), CAST(cs_ext_discount_amt AS FLOAT), ( SELECT list_transform(priv_div(as_sum(priv_hash(hash(cs_bill_customer_sk)), CAST(cs_ext_discount_amt AS DOUBLE)), as_count(priv_hash(hash(cs_bill_customer_sk)), cs_ext_discount_amt)), lambda x: CAST(1.3 * x AS FLOAT)) + FROM catalog_sales , + date_dim + WHERE cs_item_sk = i_item_sk + AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) + AND d_date_sk = cs_sold_date_sk )) <> 0 +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q33.sql b/benchmark/tpcds/as_queries/q33.sql new file mode 100644 index 00000000..c4288e8c --- /dev/null +++ b/benchmark/tpcds/as_queries/q33.sql @@ -0,0 +1,64 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH ss AS ( SELECT i_manufact_id, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales + FROM store_sales, + date_dim, + customer_address, + item + WHERE i_manufact_id IN + (SELECT i_manufact_id + FROM item + WHERE i_category IN ('Electronics')) + AND ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 5 + AND ss_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_manufact_id), +cs AS ( SELECT i_manufact_id, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales + FROM catalog_sales, + date_dim, + customer_address, + item + WHERE i_manufact_id IN + (SELECT i_manufact_id + FROM item + WHERE i_category IN ('Electronics')) + AND cs_item_sk = i_item_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 5 + AND cs_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_manufact_id), +ws AS ( SELECT i_manufact_id, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales + FROM web_sales, + date_dim, + customer_address, + item + WHERE i_manufact_id IN + (SELECT i_manufact_id + FROM item + WHERE i_category IN ('Electronics')) + AND ws_item_sk = i_item_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 5 + AND ws_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_manufact_id) +SELECT i_manufact_id, + sum(total_sales) total_sales +FROM + (SELECT * + FROM ss + UNION ALL SELECT * + FROM cs + UNION ALL SELECT * + FROM ws) tmp1 +GROUP BY i_manufact_id +ORDER BY total_sales +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q34.sql b/benchmark/tpcds/as_queries/q34.sql new file mode 100644 index 00000000..e7cdd84b --- /dev/null +++ b/benchmark/tpcds/as_queries/q34.sql @@ -0,0 +1,42 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- The terminal BETWEEN uses two separate PAC majority comparisons, not one per-world conjunction. +SELECT c_last_name , + c_first_name , + c_salutation , + c_preferred_cust_flag , + ss_ticket_number , + CAST(as_noised(cnt) AS BIGINT) AS cnt +FROM + (SELECT ss_ticket_number , + ss_customer_sk , + as_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) cnt + FROM store_sales, + date_dim, + store, + household_demographics + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND (date_dim.d_dom BETWEEN 1 AND 3 + OR date_dim.d_dom BETWEEN 25 AND 28) + AND (household_demographics.hd_buy_potential = '>10000' + OR household_demographics.hd_buy_potential = 'Unknown') + AND household_demographics.hd_vehicle_count > 0 + AND (CASE + WHEN household_demographics.hd_vehicle_count > 0 THEN (household_demographics.hd_dep_count*1.000)/ household_demographics.hd_vehicle_count + ELSE NULL + END) > 1.2 + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_county = 'Williamson County' + GROUP BY ss_ticket_number, + ss_customer_sk) dn, + customer +WHERE ss_customer_sk = c_customer_sk + AND priv_filter_lte(15::FLOAT, cnt) AND priv_filter_gte(20::FLOAT, cnt) +ORDER BY c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + c_salutation NULLS FIRST, + c_preferred_cust_flag DESC NULLS FIRST, + ss_ticket_number NULLS FIRST; diff --git a/benchmark/tpcds/as_queries/q35.sql b/benchmark/tpcds/as_queries/q35.sql new file mode 100644 index 00000000..a6a41304 --- /dev/null +++ b/benchmark/tpcds/as_queries/q35.sql @@ -0,0 +1,61 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +SELECT ca_state, + cd_gender, + cd_marital_status, + cd_dep_count, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt1, + as_noised_min(priv_hash(hash(c.c_customer_sk)), cd_dep_count) min1, + as_noised_max(priv_hash(hash(c.c_customer_sk)), cd_dep_count) max1, + (CAST(as_noised_sum(priv_hash(hash(c.c_customer_sk)), cd_dep_count) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c.c_customer_sk)), cd_dep_count) AS DOUBLE)) avg1, + cd_dep_employed_count, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt2, + as_noised_min(priv_hash(hash(c.c_customer_sk)), cd_dep_employed_count) min2, + as_noised_max(priv_hash(hash(c.c_customer_sk)), cd_dep_employed_count) max2, + (CAST(as_noised_sum(priv_hash(hash(c.c_customer_sk)), cd_dep_employed_count) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c.c_customer_sk)), cd_dep_employed_count) AS DOUBLE)) avg2, + cd_dep_college_count, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt3, + as_noised_min(priv_hash(hash(c.c_customer_sk)), cd_dep_college_count), + as_noised_max(priv_hash(hash(c.c_customer_sk)), cd_dep_college_count), + (CAST(as_noised_sum(priv_hash(hash(c.c_customer_sk)), cd_dep_college_count) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c.c_customer_sk)), cd_dep_college_count) AS DOUBLE)) +FROM customer c, + customer_address ca, + customer_demographics +WHERE c.c_current_addr_sk = ca.ca_address_sk + AND cd_demo_sk = c.c_current_cdemo_sk + AND EXISTS + (SELECT * + FROM store_sales, + date_dim + WHERE c.c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) + AND (EXISTS + (SELECT * + FROM web_sales, + date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) + OR EXISTS + (SELECT * + FROM catalog_sales, + date_dim + WHERE c.c_customer_sk = cs_ship_customer_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4)) +GROUP BY ca_state, + cd_gender, + cd_marital_status, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count +ORDER BY ca_state NULLS FIRST, + cd_gender NULLS FIRST, + cd_marital_status NULLS FIRST, + cd_dep_count NULLS FIRST, + cd_dep_employed_count NULLS FIRST, + cd_dep_college_count NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q40.sql b/benchmark/tpcds/as_queries/q40.sql new file mode 100644 index 00000000..18e49653 --- /dev/null +++ b/benchmark/tpcds/as_queries/q40.sql @@ -0,0 +1,27 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- LEFT JOIN uses the catalog-sales owner once; do not intersect the return customer mask. +SELECT w_state, + i_item_id, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE + WHEN (cast(d_date AS date) < CAST ('2000-03-11' AS date)) THEN cs_sales_price - coalesce(cr_refunded_cash,0) + ELSE 0 + END) AS sales_before, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE + WHEN (cast(d_date AS date) >= CAST ('2000-03-11' AS date)) THEN cs_sales_price - coalesce(cr_refunded_cash,0) + ELSE 0 + END) AS sales_after +FROM catalog_sales +LEFT OUTER JOIN catalog_returns ON (cs_order_number = cr_order_number + AND cs_item_sk = cr_item_sk) ,warehouse, + item, + date_dim +WHERE i_current_price BETWEEN 0.99 AND 1.49 + AND i_item_sk = cs_item_sk + AND cs_warehouse_sk = w_warehouse_sk + AND cs_sold_date_sk = d_date_sk + AND d_date BETWEEN CAST ('2000-02-10' AS date) AND CAST ('2000-04-10' AS date) +GROUP BY w_state, + i_item_id +ORDER BY w_state, + i_item_id +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q42.sql b/benchmark/tpcds/as_queries/q42.sql new file mode 100644 index 00000000..c0bc6259 --- /dev/null +++ b/benchmark/tpcds/as_queries/q42.sql @@ -0,0 +1,15 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH asa_states AS (SELECT dt.d_year AS d_year, item.i_category_id AS i_category_id, item.i_category AS i_category, as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_sales_price AS DOUBLE)) AS sum_agg FROM date_dim dt, + store_sales, + item +WHERE dt.d_date_sk = store_sales.ss_sold_date_sk + AND store_sales.ss_item_sk = item.i_item_sk + AND item.i_manager_id = 1 + AND dt.d_moy=11 + AND dt.d_year=2000 +GROUP BY dt.d_year, + item.i_category_id, + item.i_category +), +asa_ranked AS (SELECT * FROM asa_states ORDER BY priv_mean(sum_agg) DESC, d_year, i_category_id, i_category LIMIT 100) +SELECT d_year, i_category_id, i_category, CAST(as_noised(sum_agg) AS DECIMAL(38,2)) AS sum_agg FROM asa_ranked ORDER BY sum_agg DESC, d_year, i_category_id, i_category; diff --git a/benchmark/tpcds/as_queries/q43.sql b/benchmark/tpcds/as_queries/q43.sql new file mode 100644 index 00000000..caafc1e3 --- /dev/null +++ b/benchmark/tpcds/as_queries/q43.sql @@ -0,0 +1,13 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH asa_states AS (SELECT s_store_name, s_store_id, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Sunday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS sun_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Monday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS mon_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Tuesday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS tue_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Wednesday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS wed_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Thursday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS thu_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Friday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS fri_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Saturday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS sat_sales FROM date_dim, + store_sales, + store +WHERE d_date_sk = ss_sold_date_sk + AND s_store_sk = ss_store_sk + AND s_gmt_offset = -5 + AND d_year = 2000 +GROUP BY s_store_name, + s_store_id +), +asa_ranked AS (SELECT * FROM asa_states ORDER BY s_store_name, s_store_id, priv_mean(sun_sales), priv_mean(mon_sales), priv_mean(tue_sales), priv_mean(wed_sales), priv_mean(thu_sales), priv_mean(fri_sales), priv_mean(sat_sales) LIMIT 100) +SELECT s_store_name, s_store_id, CAST(as_noised(sun_sales) AS DECIMAL(38,2)) AS sun_sales, CAST(as_noised(mon_sales) AS DECIMAL(38,2)) AS mon_sales, CAST(as_noised(tue_sales) AS DECIMAL(38,2)) AS tue_sales, CAST(as_noised(wed_sales) AS DECIMAL(38,2)) AS wed_sales, CAST(as_noised(thu_sales) AS DECIMAL(38,2)) AS thu_sales, CAST(as_noised(fri_sales) AS DECIMAL(38,2)) AS fri_sales, CAST(as_noised(sat_sales) AS DECIMAL(38,2)) AS sat_sales FROM asa_ranked ORDER BY s_store_name, s_store_id, sun_sales, mon_sales, tue_sales, wed_sales, thu_sales, fri_sales, sat_sales; diff --git a/benchmark/tpcds/as_queries/q45.sql b/benchmark/tpcds/as_queries/q45.sql new file mode 100644 index 00000000..1fa8f442 --- /dev/null +++ b/benchmark/tpcds/as_queries/q45.sql @@ -0,0 +1,42 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT ca_zip, + ca_city, + as_noised_sum(priv_hash(hash(c_customer_sk)), ws_sales_price) +FROM web_sales, + customer, + customer_address, + date_dim, + item +WHERE ws_bill_customer_sk = c_customer_sk + AND c_current_addr_sk = ca_address_sk + AND ws_item_sk = i_item_sk + AND (SUBSTRING(ca_zip,1,5) IN ('85669', + '86197', + '88274', + '83405', + '86475', + '85392', + '85460', + '80348', + '81792') + OR i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_item_sk IN (2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29) )) + AND ws_sold_date_sk = d_date_sk + AND d_qoy = 2 + AND d_year = 2001 +GROUP BY ca_zip, + ca_city +ORDER BY ca_zip, + ca_city +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q46.sql b/benchmark/tpcds/as_queries/q46.sql new file mode 100644 index 00000000..839b21fa --- /dev/null +++ b/benchmark/tpcds/as_queries/q46.sql @@ -0,0 +1,48 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. + +SELECT c_last_name, + c_first_name, + ca_city, + bought_city, + ss_ticket_number, + amt, + profit +FROM + (SELECT ss_ticket_number, + ss_customer_sk, + ca_city bought_city, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) amt, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) profit + FROM store_sales, + date_dim, + store, + household_demographics, + customer_address + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND store_sales.ss_addr_sk = customer_address.ca_address_sk + AND (household_demographics.hd_dep_count = 4 + OR household_demographics.hd_vehicle_count= 3) + AND date_dim.d_dow IN (6, + 0) + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_city IN ('Fairview', + 'Midway') + GROUP BY ss_ticket_number, + ss_customer_sk, + ss_addr_sk, + ca_city) dn, + customer, + customer_address current_addr +WHERE ss_customer_sk = c_customer_sk + AND customer.c_current_addr_sk = current_addr.ca_address_sk + AND current_addr.ca_city <> bought_city +ORDER BY c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + ca_city NULLS FIRST, + bought_city NULLS FIRST, + ss_ticket_number NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q48.sql b/benchmark/tpcds/as_queries/q48.sql new file mode 100644 index 00000000..b2796c71 --- /dev/null +++ b/benchmark/tpcds/as_queries/q48.sql @@ -0,0 +1,40 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) +FROM store_sales, + store, + customer_demographics, + customer_address, + date_dim +WHERE s_store_sk = ss_store_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2000 + AND ((cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'M' + AND cd_education_status = '4 yr Degree' + AND ss_sales_price BETWEEN 100.00 AND 150.00) + OR (cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'D' + AND cd_education_status = '2 yr Degree' + AND ss_sales_price BETWEEN 50.00 AND 100.00) + OR (cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND ss_sales_price BETWEEN 150.00 AND 200.00)) + AND ((ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('CO', + 'OH', + 'TX') + AND ss_net_profit BETWEEN 0 AND 2000) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('OR', + 'MN', + 'KY') + AND ss_net_profit BETWEEN 150 AND 3000) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('VA', + 'CA', + 'MS') + AND ss_net_profit BETWEEN 50 AND 25000)) ; diff --git a/benchmark/tpcds/as_queries/q50.sql b/benchmark/tpcds/as_queries/q50.sql new file mode 100644 index 00000000..07fc3b8b --- /dev/null +++ b/benchmark/tpcds/as_queries/q50.sql @@ -0,0 +1,68 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT s_store_name, + s_company_id, + s_street_number, + s_street_name, + s_street_type, + s_suite_number, + s_city, + s_county, + s_state, + s_zip, + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (sr_returned_date_sk - ss_sold_date_sk <= 30) THEN 1 + ELSE 0 + END) AS "30 days", + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (sr_returned_date_sk - ss_sold_date_sk > 30) + AND (sr_returned_date_sk - ss_sold_date_sk <= 60) THEN 1 + ELSE 0 + END) AS "31-60 days", + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (sr_returned_date_sk - ss_sold_date_sk > 60) + AND (sr_returned_date_sk - ss_sold_date_sk <= 90) THEN 1 + ELSE 0 + END) AS "61-90 days", + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (sr_returned_date_sk - ss_sold_date_sk > 90) + AND (sr_returned_date_sk - ss_sold_date_sk <= 120) THEN 1 + ELSE 0 + END) AS "91-120 days", + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (sr_returned_date_sk - ss_sold_date_sk > 120) THEN 1 + ELSE 0 + END) AS ">120 days" +FROM store_sales, + store_returns, + store, + date_dim d1, + date_dim d2 +WHERE d2.d_year = 2001 + AND d2.d_moy = 8 + AND ss_ticket_number = sr_ticket_number + AND ss_item_sk = sr_item_sk + AND ss_sold_date_sk = d1.d_date_sk + AND sr_returned_date_sk = d2.d_date_sk + AND ss_customer_sk = sr_customer_sk + AND ss_store_sk = s_store_sk +GROUP BY s_store_name, + s_company_id, + s_street_number, + s_street_name, + s_street_type, + s_suite_number, + s_city, + s_county, + s_state, + s_zip +ORDER BY s_store_name, + s_company_id, + s_street_number, + s_street_name, + s_street_type, + s_suite_number, + s_city, + s_county, + s_state, + s_zip +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q52.sql b/benchmark/tpcds/as_queries/q52.sql new file mode 100644 index 00000000..adad0f2a --- /dev/null +++ b/benchmark/tpcds/as_queries/q52.sql @@ -0,0 +1,15 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH asa_states AS (SELECT dt.d_year AS d_year, item.i_brand_id AS brand_id, item.i_brand AS brand, as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_sales_price AS DOUBLE)) AS ext_price FROM date_dim dt, + store_sales, + item +WHERE dt.d_date_sk = store_sales.ss_sold_date_sk + AND store_sales.ss_item_sk = item.i_item_sk + AND item.i_manager_id = 1 + AND dt.d_moy=11 + AND dt.d_year=2000 +GROUP BY dt.d_year, + item.i_brand, + item.i_brand_id +), +asa_ranked AS (SELECT * FROM asa_states ORDER BY d_year, priv_mean(ext_price) DESC, brand_id LIMIT 100) +SELECT d_year, brand_id, brand, CAST(as_noised(ext_price) AS DECIMAL(38,2)) AS ext_price FROM asa_ranked ORDER BY d_year, ext_price DESC, brand_id; diff --git a/benchmark/tpcds/as_queries/q55.sql b/benchmark/tpcds/as_queries/q55.sql new file mode 100644 index 00000000..a93b0782 --- /dev/null +++ b/benchmark/tpcds/as_queries/q55.sql @@ -0,0 +1,14 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH asa_states AS (SELECT i_brand_id AS brand_id, i_brand AS brand, as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_sales_price AS DOUBLE)) AS ext_price FROM date_dim, + store_sales, + item +WHERE d_date_sk = ss_sold_date_sk + AND ss_item_sk = i_item_sk + AND i_manager_id=28 + AND d_moy=11 + AND d_year=1999 +GROUP BY i_brand, + i_brand_id +), +asa_ranked AS (SELECT * FROM asa_states ORDER BY priv_mean(ext_price) DESC, brand_id LIMIT 100) +SELECT brand_id, brand, CAST(as_noised(ext_price) AS DECIMAL(38,2)) AS ext_price FROM asa_ranked ORDER BY ext_price DESC, brand_id; diff --git a/benchmark/tpcds/as_queries/q56.sql b/benchmark/tpcds/as_queries/q56.sql new file mode 100644 index 00000000..7e10f771 --- /dev/null +++ b/benchmark/tpcds/as_queries/q56.sql @@ -0,0 +1,71 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH ss AS (SELECT i_item_id, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales + FROM store_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_color IN ('slate', + 'blanched', + 'burnished')) + AND ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 2 + AND ss_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id), +cs AS (SELECT i_item_id, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales + FROM catalog_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_color IN ('slate', + 'blanched', + 'burnished')) + AND cs_item_sk = i_item_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 2 + AND cs_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id), +ws AS (SELECT i_item_id, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales + FROM web_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_color IN ('slate', + 'blanched', + 'burnished')) + AND ws_item_sk = i_item_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 2 + AND ws_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id) +SELECT i_item_id, + sum(total_sales) total_sales +FROM + (SELECT * + FROM ss + UNION ALL SELECT * + FROM cs + UNION ALL SELECT * + FROM ws) tmp1 +GROUP BY i_item_id +ORDER BY total_sales NULLS FIRST, + i_item_id NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q58.sql b/benchmark/tpcds/as_queries/q58.sql new file mode 100644 index 00000000..cbc64b88 --- /dev/null +++ b/benchmark/tpcds/as_queries/q58.sql @@ -0,0 +1,68 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH ss_items AS (SELECT i_item_id item_id, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) ss_item_rev + FROM store_sales, + item, + date_dim + WHERE ss_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq = + (SELECT d_week_seq + FROM date_dim + WHERE d_date = '2000-01-03')) + AND ss_sold_date_sk = d_date_sk + GROUP BY i_item_id), +cs_items AS (SELECT i_item_id item_id, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) cs_item_rev + FROM catalog_sales, + item, + date_dim + WHERE cs_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq = + (SELECT d_week_seq + FROM date_dim + WHERE d_date = '2000-01-03')) + AND cs_sold_date_sk = d_date_sk + GROUP BY i_item_id), +ws_items AS (SELECT i_item_id item_id, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) ws_item_rev + FROM web_sales, + item, + date_dim + WHERE ws_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq = + (SELECT d_week_seq + FROM date_dim + WHERE d_date = '2000-01-03')) + AND ws_sold_date_sk = d_date_sk + GROUP BY i_item_id) +SELECT ss_items.item_id, + ss_item_rev, + ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev, + cs_item_rev, + cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev, + ws_item_rev, + ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev, + (ss_item_rev+cs_item_rev+ws_item_rev)/3 average +FROM ss_items, + cs_items, + ws_items +WHERE ss_items.item_id=cs_items.item_id + AND ss_items.item_id=ws_items.item_id + AND ss_item_rev BETWEEN 0.9 * cs_item_rev AND 1.1 * cs_item_rev + AND ss_item_rev BETWEEN 0.9 * ws_item_rev AND 1.1 * ws_item_rev + AND cs_item_rev BETWEEN 0.9 * ss_item_rev AND 1.1 * ss_item_rev + AND cs_item_rev BETWEEN 0.9 * ws_item_rev AND 1.1 * ws_item_rev + AND ws_item_rev BETWEEN 0.9 * ss_item_rev AND 1.1 * ss_item_rev + AND ws_item_rev BETWEEN 0.9 * cs_item_rev AND 1.1 * cs_item_rev +ORDER BY ss_items.item_id NULLS FIRST, + ss_item_rev NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q59.sql b/benchmark/tpcds/as_queries/q59.sql new file mode 100644 index 00000000..4dd90e46 --- /dev/null +++ b/benchmark/tpcds/as_queries/q59.sql @@ -0,0 +1,85 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH wss AS (SELECT d_week_seq, + ss_store_sk, + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Sunday') THEN ss_sales_price + ELSE NULL + END) sun_sales, + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Monday') THEN ss_sales_price + ELSE NULL + END) mon_sales, + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Tuesday') THEN ss_sales_price + ELSE NULL + END) tue_sales, + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Wednesday') THEN ss_sales_price + ELSE NULL + END) wed_sales, + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Thursday') THEN ss_sales_price + ELSE NULL + END) thu_sales, + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Friday') THEN ss_sales_price + ELSE NULL + END) fri_sales, + as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Saturday') THEN ss_sales_price + ELSE NULL + END) sat_sales + FROM store_sales, + date_dim + WHERE d_date_sk = ss_sold_date_sk + GROUP BY d_week_seq, + ss_store_sk) +SELECT s_store_name1, + s_store_id1, + d_week_seq1, + sun_sales1/sun_sales2 AS sun_sales_ratio, + mon_sales1/mon_sales2 AS mon_sales_ratio, + tue_sales1/tue_sales2 AS tue_sales_ratio, + wed_sales1/wed_sales2 AS wed_sales_ratio, + thu_sales1/thu_sales2 AS thu_sales_ratio, + fri_sales1/fri_sales2 AS fri_sales_ratio, + sat_sales1/sat_sales2 AS sat_sales_ratio +FROM + (SELECT s_store_name s_store_name1, + wss.d_week_seq d_week_seq1, + s_store_id s_store_id1, + sun_sales sun_sales1, + mon_sales mon_sales1, + tue_sales tue_sales1, + wed_sales wed_sales1, + thu_sales thu_sales1, + fri_sales fri_sales1, + sat_sales sat_sales1 + FROM wss, + store, + date_dim d + WHERE d.d_week_seq = wss.d_week_seq + AND ss_store_sk = s_store_sk + AND d_month_seq BETWEEN 1212 AND 1212 + 11) y, + (SELECT s_store_name s_store_name2, + wss.d_week_seq d_week_seq2, + s_store_id s_store_id2, + sun_sales sun_sales2, + mon_sales mon_sales2, + tue_sales tue_sales2, + wed_sales wed_sales2, + thu_sales thu_sales2, + fri_sales fri_sales2, + sat_sales sat_sales2 + FROM wss, + store, + date_dim d + WHERE d.d_week_seq = wss.d_week_seq + AND ss_store_sk = s_store_sk + AND d_month_seq BETWEEN 1212 + 12 AND 1212 + 23) x +WHERE s_store_id1=s_store_id2 + AND d_week_seq1=d_week_seq2-52 +ORDER BY s_store_name1 NULLS FIRST, + s_store_id1 NULLS FIRST, + d_week_seq1 NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q60.sql b/benchmark/tpcds/as_queries/q60.sql new file mode 100644 index 00000000..e2430099 --- /dev/null +++ b/benchmark/tpcds/as_queries/q60.sql @@ -0,0 +1,65 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH ss AS (SELECT i_item_id, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales + FROM store_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_category ='Music') + AND ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 9 + AND ss_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id), +cs AS (SELECT i_item_id, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales + FROM catalog_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_category ='Music') + AND cs_item_sk = i_item_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 9 + AND cs_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id), +ws AS (SELECT i_item_id, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales + FROM web_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_category = 'Music') + AND ws_item_sk = i_item_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 9 + AND ws_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id) +SELECT i_item_id, + sum(total_sales) total_sales +FROM + (SELECT * + FROM ss + UNION ALL SELECT * + FROM cs + UNION ALL SELECT * + FROM ws) tmp1 +GROUP BY i_item_id +ORDER BY i_item_id, + total_sales +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q61.sql b/benchmark/tpcds/as_queries/q61.sql new file mode 100644 index 00000000..4f924fc8 --- /dev/null +++ b/benchmark/tpcds/as_queries/q61.sql @@ -0,0 +1,44 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Ratio arithmetic is lane-wise; only final outputs are scalarized, matching categorical compilation. +SELECT CAST(as_noised(promotions) AS DECIMAL(38,2)) AS promotions, CAST(as_noised(total) AS DECIMAL(38,2)) AS total, CAST(as_noised(list_transform(list_zip(promotions,total), lambda x: CAST(x[1] AS DECIMAL(15,4)) / CAST(x[2] AS DECIMAL(15,4)) * 100)::FLOAT[]) AS DOUBLE) +FROM (SELECT as_sum(priv_hash(hash(c_customer_sk)), ss_ext_sales_price) promotions + FROM store_sales, + store, + promotion, + date_dim, + customer, + customer_address, + item + WHERE ss_sold_date_sk = d_date_sk + AND ss_store_sk = s_store_sk + AND ss_promo_sk = p_promo_sk + AND ss_customer_sk= c_customer_sk + AND ca_address_sk = c_current_addr_sk + AND ss_item_sk = i_item_sk + AND ca_gmt_offset = -5 + AND i_category = 'Jewelry' + AND (p_channel_dmail = 'Y' + OR p_channel_email = 'Y' + OR p_channel_tv = 'Y') + AND s_gmt_offset = -5 + AND d_year = 1998 + AND d_moy = 11) promotional_sales, (SELECT as_sum(priv_hash(hash(c_customer_sk)), ss_ext_sales_price) total + FROM store_sales, + store, + date_dim, + customer, + customer_address, + item + WHERE ss_sold_date_sk = d_date_sk + AND ss_store_sk = s_store_sk + AND ss_customer_sk= c_customer_sk + AND ca_address_sk = c_current_addr_sk + AND ss_item_sk = i_item_sk + AND ca_gmt_offset = -5 + AND i_category = 'Jewelry' + AND s_gmt_offset = -5 + AND d_year = 1998 + AND d_moy = 11) all_sales +ORDER BY promotions, + total +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q62.sql b/benchmark/tpcds/as_queries/q62.sql new file mode 100644 index 00000000..a95aa874 --- /dev/null +++ b/benchmark/tpcds/as_queries/q62.sql @@ -0,0 +1,46 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT w_substr, + sm_type, + web_name, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), CASE + WHEN (ws_ship_date_sk - ws_sold_date_sk <= 30) THEN 1 + ELSE 0 + END) AS "30 days", + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), CASE + WHEN (ws_ship_date_sk - ws_sold_date_sk > 30) + AND (ws_ship_date_sk - ws_sold_date_sk <= 60) THEN 1 + ELSE 0 + END) AS "31-60 days", + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), CASE + WHEN (ws_ship_date_sk - ws_sold_date_sk > 60) + AND (ws_ship_date_sk - ws_sold_date_sk <= 90) THEN 1 + ELSE 0 + END) AS "61-90 days", + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), CASE + WHEN (ws_ship_date_sk - ws_sold_date_sk > 90) + AND (ws_ship_date_sk - ws_sold_date_sk <= 120) THEN 1 + ELSE 0 + END) AS "91-120 days", + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), CASE + WHEN (ws_ship_date_sk - ws_sold_date_sk > 120) THEN 1 + ELSE 0 + END) AS ">120 days" +FROM web_sales, + (SELECT SUBSTRING(w_warehouse_name,1,20) w_substr, + * + FROM warehouse) sq1, + ship_mode, + web_site, + date_dim +WHERE d_month_seq BETWEEN 1200 AND 1200 + 11 + AND ws_ship_date_sk = d_date_sk + AND ws_warehouse_sk = w_warehouse_sk + AND ws_ship_mode_sk = sm_ship_mode_sk + AND ws_web_site_sk = web_site_sk +GROUP BY w_substr, + sm_type, + web_name +ORDER BY 1 NULLS FIRST, + 2 NULLS FIRST, + 3 NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q64.sql b/benchmark/tpcds/as_queries/q64.sql new file mode 100644 index 00000000..bed6a7ea --- /dev/null +++ b/benchmark/tpcds/as_queries/q64.sql @@ -0,0 +1,124 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Each CTE retains scalar aggregates and HAVING; source ownership differs between its two scopes. +WITH cs_ui AS (SELECT cs_item_sk, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_list_price) AS sale, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cr_refunded_cash+cr_reversed_charge+cr_store_credit) AS refund + FROM catalog_sales, + catalog_returns + WHERE cs_item_sk = cr_item_sk + AND cs_order_number = cr_order_number + GROUP BY cs_item_sk + HAVING as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_list_price)>2*as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cr_refunded_cash+cr_reversed_charge+cr_store_credit)), +cross_sales AS (SELECT i_product_name product_name, + i_item_sk item_sk, + s_store_name store_name, + s_zip store_zip, + ad1.ca_street_number b_street_number, + ad1.ca_street_name b_street_name, + ad1.ca_city b_city, + ad1.ca_zip b_zip, + ad2.ca_street_number c_street_number, + ad2.ca_street_name c_street_name, + ad2.ca_city c_city, + ad2.ca_zip c_zip, + d1.d_year AS syear, + d2.d_year AS fsyear, + d3.d_year s2year, + as_noised_count(priv_hash(hash(c_customer_sk)), 1::BIGINT) cnt, + as_noised_sum(priv_hash(hash(c_customer_sk)), ss_wholesale_cost) s1, + as_noised_sum(priv_hash(hash(c_customer_sk)), ss_list_price) s2, + as_noised_sum(priv_hash(hash(c_customer_sk)), ss_coupon_amt) s3 + FROM store_sales, + store_returns, + cs_ui, + date_dim d1, + date_dim d2, + date_dim d3, + store, + customer, + customer_demographics cd1, + customer_demographics cd2, + promotion, + household_demographics hd1, + household_demographics hd2, + customer_address ad1, + customer_address ad2, + income_band ib1, + income_band ib2, + item + WHERE ss_store_sk = s_store_sk + AND ss_sold_date_sk = d1.d_date_sk + AND ss_customer_sk = c_customer_sk + AND ss_cdemo_sk= cd1.cd_demo_sk + AND ss_hdemo_sk = hd1.hd_demo_sk + AND ss_addr_sk = ad1.ca_address_sk + AND ss_item_sk = i_item_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND ss_item_sk = cs_ui.cs_item_sk + AND c_current_cdemo_sk = cd2.cd_demo_sk + AND c_current_hdemo_sk = hd2.hd_demo_sk + AND c_current_addr_sk = ad2.ca_address_sk + AND c_first_sales_date_sk = d2.d_date_sk + AND c_first_shipto_date_sk = d3.d_date_sk + AND ss_promo_sk = p_promo_sk + AND hd1.hd_income_band_sk = ib1.ib_income_band_sk + AND hd2.hd_income_band_sk = ib2.ib_income_band_sk + AND cd1.cd_marital_status <> cd2.cd_marital_status + AND i_color IN ('purple', + 'burlywood', + 'indian', + 'spring', + 'floral', + 'medium') + AND i_current_price BETWEEN 64 AND 64 + 10 + AND i_current_price BETWEEN 64 + 1 AND 64 + 15 + GROUP BY i_product_name, + i_item_sk, + s_store_name, + s_zip, + ad1.ca_street_number, + ad1.ca_street_name, + ad1.ca_city, + ad1.ca_zip, + ad2.ca_street_number, + ad2.ca_street_name, + ad2.ca_city, + ad2.ca_zip, + d1.d_year, + d2.d_year, + d3.d_year) +SELECT cs1.product_name, + cs1.store_name, + cs1.store_zip, + cs1.b_street_number, + cs1.b_street_name, + cs1.b_city, + cs1.b_zip, + cs1.c_street_number, + cs1.c_street_name, + cs1.c_city, + cs1.c_zip, + cs1.syear cs1syear, + cs1.cnt cs1cnt, + cs1.s1 AS s11, + cs1.s2 AS s21, + cs1.s3 AS s31, + cs2.s1 AS s12, + cs2.s2 AS s22, + cs2.s3 AS s32, + cs2.syear, + cs2.cnt +FROM cross_sales cs1, + cross_sales cs2 +WHERE cs1.item_sk=cs2.item_sk + AND cs1.syear = 1999 + AND cs2.syear = 1999 + 1 + AND cs2.cnt <= cs1.cnt + AND cs1.store_name = cs2.store_name + AND cs1.store_zip = cs2.store_zip +ORDER BY cs1.product_name, + cs1.store_name, + cs2.cnt, + cs1.s1, + cs2.s1; diff --git a/benchmark/tpcds/as_queries/q65.sql b/benchmark/tpcds/as_queries/q65.sql new file mode 100644 index 00000000..e774e277 --- /dev/null +++ b/benchmark/tpcds/as_queries/q65.sql @@ -0,0 +1,23 @@ +-- Hand transcription of Q65's compiled PAC dataflow, for auditing (priv_rewrite=false). +-- This intentionally retains the scalarized inner SUM and majority filter from the compiler. +SELECT s_store_name, i_item_desc, + CAST(as_noised(sc.revenue) AS DECIMAL(38,2)) AS revenue, + i_current_price, i_wholesale_cost, i_brand +FROM main.store, main.item, + (SELECT ss_store_sk, avg(revenue) AS ave + FROM (SELECT ss_store_sk, ss_item_sk, + as_noised_sum(priv_hash(hash(ss_customer_sk)),ss_sales_price) AS revenue + FROM main.store_sales, main.date_dim + WHERE ss_sold_date_sk=d_date_sk AND d_month_seq BETWEEN 1176 AND 1187 + GROUP BY ss_store_sk,ss_item_sk) sa + GROUP BY ss_store_sk) sb, + (SELECT ss_store_sk,ss_item_sk, + as_sum(priv_hash(hash(ss_customer_sk)),ss_sales_price) AS revenue + FROM main.store_sales, main.date_dim + WHERE ss_sold_date_sk=d_date_sk AND d_month_seq BETWEEN 1176 AND 1187 + GROUP BY ss_store_sk,ss_item_sk) sc +WHERE sb.ss_store_sk=sc.ss_store_sk + AND priv_filter_gte(CAST(0.1*sb.ave AS FLOAT),sc.revenue) + AND s_store_sk=sc.ss_store_sk AND i_item_sk=sc.ss_item_sk +ORDER BY s_store_name NULLS FIRST,i_item_desc NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q66.sql b/benchmark/tpcds/as_queries/q66.sql new file mode 100644 index 00000000..7c9e669f --- /dev/null +++ b/benchmark/tpcds/as_queries/q66.sql @@ -0,0 +1,221 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Only the two leaf UNION branches are stochastic; the outer monthly rollup remains ordinary. +select + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,ship_carriers + ,year_ + ,sum(jan_sales) as jan_sales + ,sum(feb_sales) as feb_sales + ,sum(mar_sales) as mar_sales + ,sum(apr_sales) as apr_sales + ,sum(may_sales) as may_sales + ,sum(jun_sales) as jun_sales + ,sum(jul_sales) as jul_sales + ,sum(aug_sales) as aug_sales + ,sum(sep_sales) as sep_sales + ,sum(oct_sales) as oct_sales + ,sum(nov_sales) as nov_sales + ,sum(dec_sales) as dec_sales + ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot + ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot + ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot + ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot + ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot + ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot + ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot + ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot + ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot + ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot + ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot + ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot + ,sum(jan_net) as jan_net + ,sum(feb_net) as feb_net + ,sum(mar_net) as mar_net + ,sum(apr_net) as apr_net + ,sum(may_net) as may_net + ,sum(jun_net) as jun_net + ,sum(jul_net) as jul_net + ,sum(aug_net) as aug_net + ,sum(sep_net) as sep_net + ,sum(oct_net) as oct_net + ,sum(nov_net) as nov_net + ,sum(dec_net) as dec_net + from ( + select + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,'DHL,BARIAN' as ship_carriers + ,d_year as year_ + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 1 + then ws_ext_sales_price* ws_quantity else 0 end) as jan_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 2 + then ws_ext_sales_price* ws_quantity else 0 end) as feb_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 3 + then ws_ext_sales_price* ws_quantity else 0 end) as mar_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 4 + then ws_ext_sales_price* ws_quantity else 0 end) as apr_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 5 + then ws_ext_sales_price* ws_quantity else 0 end) as may_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 6 + then ws_ext_sales_price* ws_quantity else 0 end) as jun_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 7 + then ws_ext_sales_price* ws_quantity else 0 end) as jul_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 8 + then ws_ext_sales_price* ws_quantity else 0 end) as aug_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 9 + then ws_ext_sales_price* ws_quantity else 0 end) as sep_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 10 + then ws_ext_sales_price* ws_quantity else 0 end) as oct_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 11 + then ws_ext_sales_price* ws_quantity else 0 end) as nov_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 12 + then ws_ext_sales_price* ws_quantity else 0 end) as dec_sales + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 1 + then ws_net_paid * ws_quantity else 0 end) as jan_net + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 2 + then ws_net_paid * ws_quantity else 0 end) as feb_net + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 3 + then ws_net_paid * ws_quantity else 0 end) as mar_net + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 4 + then ws_net_paid * ws_quantity else 0 end) as apr_net + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 5 + then ws_net_paid * ws_quantity else 0 end) as may_net + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 6 + then ws_net_paid * ws_quantity else 0 end) as jun_net + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 7 + then ws_net_paid * ws_quantity else 0 end) as jul_net + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 8 + then ws_net_paid * ws_quantity else 0 end) as aug_net + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 9 + then ws_net_paid * ws_quantity else 0 end) as sep_net + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 10 + then ws_net_paid * ws_quantity else 0 end) as oct_net + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 11 + then ws_net_paid * ws_quantity else 0 end) as nov_net + ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 12 + then ws_net_paid * ws_quantity else 0 end) as dec_net + from + web_sales + ,warehouse + ,date_dim + ,time_dim + ,ship_mode + where + ws_warehouse_sk = w_warehouse_sk + and ws_sold_date_sk = d_date_sk + and ws_sold_time_sk = t_time_sk + and ws_ship_mode_sk = sm_ship_mode_sk + and d_year = 2001 + and t_time between 30838 and 30838+28800 + and sm_carrier in ('DHL','BARIAN') + group by + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,d_year + +UNION ALL + + select + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,'DHL,BARIAN' as ship_carriers + ,d_year as year_ + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 1 + then cs_sales_price* cs_quantity else 0 end) as jan_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 2 + then cs_sales_price* cs_quantity else 0 end) as feb_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 3 + then cs_sales_price* cs_quantity else 0 end) as mar_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 4 + then cs_sales_price* cs_quantity else 0 end) as apr_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 5 + then cs_sales_price* cs_quantity else 0 end) as may_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 6 + then cs_sales_price* cs_quantity else 0 end) as jun_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 7 + then cs_sales_price* cs_quantity else 0 end) as jul_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 8 + then cs_sales_price* cs_quantity else 0 end) as aug_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 9 + then cs_sales_price* cs_quantity else 0 end) as sep_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 10 + then cs_sales_price* cs_quantity else 0 end) as oct_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 11 + then cs_sales_price* cs_quantity else 0 end) as nov_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 12 + then cs_sales_price* cs_quantity else 0 end) as dec_sales + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 1 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as jan_net + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 2 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as feb_net + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 3 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as mar_net + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 4 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as apr_net + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 5 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as may_net + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 6 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as jun_net + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 7 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as jul_net + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 8 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as aug_net + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 9 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as sep_net + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 10 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as oct_net + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 11 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as nov_net + ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 12 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as dec_net + from + catalog_sales + ,warehouse + ,date_dim + ,time_dim + ,ship_mode + where + cs_warehouse_sk = w_warehouse_sk + and cs_sold_date_sk = d_date_sk + and cs_sold_time_sk = t_time_sk + and cs_ship_mode_sk = sm_ship_mode_sk + and d_year = 2001 + and t_time between 30838 AND 30838+28800 + and sm_carrier in ('DHL','BARIAN') + group by + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,d_year + ) x + group by + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,ship_carriers + ,year_ + order by w_warehouse_name NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q68.sql b/benchmark/tpcds/as_queries/q68.sql new file mode 100644 index 00000000..7cf7f5c7 --- /dev/null +++ b/benchmark/tpcds/as_queries/q68.sql @@ -0,0 +1,45 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +SELECT c_last_name, + c_first_name, + ca_city, + bought_city, + ss_ticket_number, + extended_price, + extended_tax, + list_price +FROM + (SELECT ss_ticket_number, + ss_customer_sk, + ca_city bought_city, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) extended_price, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_list_price) list_price, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_tax) extended_tax + FROM store_sales, + date_dim, + store, + household_demographics, + customer_address + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND store_sales.ss_addr_sk = customer_address.ca_address_sk + AND date_dim.d_dom BETWEEN 1 AND 2 + AND (household_demographics.hd_dep_count = 4 + OR household_demographics.hd_vehicle_count= 3) + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_city IN ('Fairview', + 'Midway') + GROUP BY ss_ticket_number, + ss_customer_sk, + ss_addr_sk, + ca_city) dn, + customer, + customer_address current_addr +WHERE ss_customer_sk = c_customer_sk + AND customer.c_current_addr_sk = current_addr.ca_address_sk + AND current_addr.ca_city <> bought_city +ORDER BY c_last_name NULLS FIRST, + ss_ticket_number NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q69.sql b/benchmark/tpcds/as_queries/q69.sql new file mode 100644 index 00000000..58d35272 --- /dev/null +++ b/benchmark/tpcds/as_queries/q69.sql @@ -0,0 +1,52 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +SELECT cd_gender, + cd_marital_status, + cd_education_status, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt1, + cd_purchase_estimate, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt2, + cd_credit_rating, + as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt3 +FROM customer c, + customer_address ca, + customer_demographics +WHERE c.c_current_addr_sk = ca.ca_address_sk + AND ca_state IN ('KY', + 'GA', + 'NM') + AND cd_demo_sk = c.c_current_cdemo_sk + AND EXISTS + (SELECT * + FROM store_sales, + date_dim + WHERE c.c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy BETWEEN 4 AND 4+2) + AND (NOT EXISTS + (SELECT * + FROM web_sales, + date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy BETWEEN 4 AND 4+2) + AND NOT EXISTS + (SELECT * + FROM catalog_sales, + date_dim + WHERE c.c_customer_sk = cs_ship_customer_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy BETWEEN 4 AND 4+2)) +GROUP BY cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating +ORDER BY cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q72.sql b/benchmark/tpcds/as_queries/q72.sql new file mode 100644 index 00000000..fb593e23 --- /dev/null +++ b/benchmark/tpcds/as_queries/q72.sql @@ -0,0 +1,40 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- One catalog-sales mask drives the states after the original joins, including LEFT JOINs. +SELECT i_item_desc, + w_warehouse_name, + d1.d_week_seq, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE + WHEN p_promo_sk IS NULL THEN 1 + ELSE 0 + END) no_promo, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE + WHEN p_promo_sk IS NOT NULL THEN 1 + ELSE 0 + END) promo, + as_noised_count(priv_hash(hash(cs_bill_customer_sk)), 1::BIGINT) total_cnt +FROM catalog_sales +JOIN inventory ON (cs_item_sk = inv_item_sk) +JOIN warehouse ON (w_warehouse_sk=inv_warehouse_sk) +JOIN item ON (i_item_sk = cs_item_sk) +JOIN customer_demographics ON (cs_bill_cdemo_sk = cd_demo_sk) +JOIN household_demographics ON (cs_bill_hdemo_sk = hd_demo_sk) +JOIN date_dim d1 ON (cs_sold_date_sk = d1.d_date_sk) +JOIN date_dim d2 ON (inv_date_sk = d2.d_date_sk) +JOIN date_dim d3 ON (cs_ship_date_sk = d3.d_date_sk) +LEFT OUTER JOIN promotion ON (cs_promo_sk=p_promo_sk) +LEFT OUTER JOIN catalog_returns ON (cr_item_sk = cs_item_sk + AND cr_order_number = cs_order_number) +WHERE d1.d_week_seq = d2.d_week_seq + AND inv_quantity_on_hand < cs_quantity + AND d3.d_date > d1.d_date + 5 -- SQL Server: DATEADD(day, 5, d1.d_date) + AND hd_buy_potential = '>10000' + AND d1.d_year = 1999 + AND cd_marital_status = 'D' +GROUP BY i_item_desc, + w_warehouse_name, + d1.d_week_seq +ORDER BY total_cnt DESC NULLS FIRST, + i_item_desc NULLS FIRST, + w_warehouse_name NULLS FIRST, + d1.d_week_seq NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q73.sql b/benchmark/tpcds/as_queries/q73.sql new file mode 100644 index 00000000..c6eb962c --- /dev/null +++ b/benchmark/tpcds/as_queries/q73.sql @@ -0,0 +1,42 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Terminal range comparisons are separate majority decisions; output cnt is finalized afterwards. + +SELECT c_last_name, + c_first_name, + c_salutation, + c_preferred_cust_flag, + ss_ticket_number, + CAST(as_noised(cnt) AS BIGINT) AS cnt +FROM + (SELECT ss_ticket_number, + ss_customer_sk, + as_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) cnt + FROM store_sales, + date_dim, + store, + household_demographics + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND date_dim.d_dom BETWEEN 1 AND 2 + AND (household_demographics.hd_buy_potential = 'Unknown' + OR household_demographics.hd_buy_potential = '>10000') + AND household_demographics.hd_vehicle_count > 0 + AND CASE + WHEN household_demographics.hd_vehicle_count > 0 THEN (household_demographics.hd_dep_count*1.000)/ household_demographics.hd_vehicle_count + ELSE NULL + END > 1 + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_county IN ('Orange County', + 'Bronx County', + 'Franklin Parish', + 'Williamson County') + GROUP BY ss_ticket_number, + ss_customer_sk) dj, + customer +WHERE ss_customer_sk = c_customer_sk + AND priv_filter_lte(1::FLOAT, cnt) AND priv_filter_gte(5::FLOAT, cnt) +ORDER BY cnt DESC, + c_last_name ASC; diff --git a/benchmark/tpcds/as_queries/q74.sql b/benchmark/tpcds/as_queries/q74.sql new file mode 100644 index 00000000..1a7ee67d --- /dev/null +++ b/benchmark/tpcds/as_queries/q74.sql @@ -0,0 +1,64 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH year_total AS (SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + d_year AS year_, + as_noised_sum(priv_hash(hash(c_customer_sk)), ss_net_paid) year_total, + 's' sale_type + FROM customer, + store_sales, + date_dim + WHERE c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year IN (2001, + 2001+1) + GROUP BY c_customer_id, + c_first_name, + c_last_name, + d_year + UNION ALL SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + d_year AS year_, + as_noised_sum(priv_hash(hash(c_customer_sk)), ws_net_paid) year_total, + 'w' sale_type + FROM customer, + web_sales, + date_dim + WHERE c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year IN (2001, + 2001+1) + GROUP BY c_customer_id, + c_first_name, + c_last_name, + d_year) +SELECT t_s_secyear.customer_id, + t_s_secyear.customer_first_name, + t_s_secyear.customer_last_name +FROM year_total t_s_firstyear, + year_total t_s_secyear, + year_total t_w_firstyear, + year_total t_w_secyear +WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_secyear.customer_id + AND t_s_firstyear.customer_id = t_w_firstyear.customer_id + AND t_s_firstyear.sale_type = 's' + AND t_w_firstyear.sale_type = 'w' + AND t_s_secyear.sale_type = 's' + AND t_w_secyear.sale_type = 'w' + AND t_s_firstyear.year_ = 2001 + AND t_s_secyear.year_ = 2001+1 + AND t_w_firstyear.year_ = 2001 + AND t_w_secyear.year_ = 2001+1 + AND t_s_firstyear.year_total > 0 + AND t_w_firstyear.year_total > 0 + AND CASE + WHEN t_w_firstyear.year_total > 0 THEN t_w_secyear.year_total / t_w_firstyear.year_total + ELSE NULL + END > CASE + WHEN t_s_firstyear.year_total > 0 THEN t_s_secyear.year_total / t_s_firstyear.year_total + ELSE NULL + END +ORDER BY 1 NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q77.sql b/benchmark/tpcds/as_queries/q77.sql new file mode 100644 index 00000000..c29146cd --- /dev/null +++ b/benchmark/tpcds/as_queries/q77.sql @@ -0,0 +1,91 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Keep all six original branches, including the original catalog Cartesian product. +WITH ss AS (SELECT s_store_sk, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS sales, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) AS profit + FROM store_sales, + date_dim, + store + WHERE ss_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND ss_store_sk = s_store_sk + GROUP BY s_store_sk), +sr AS (SELECT s_store_sk, + as_noised_sum(priv_hash(hash(sr_customer_sk)), sr_return_amt) AS returns_, + as_noised_sum(priv_hash(hash(sr_customer_sk)), sr_net_loss) AS profit_loss + FROM store_returns, + date_dim, + store + WHERE sr_returned_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND sr_store_sk = s_store_sk + GROUP BY s_store_sk), +cs AS (SELECT cs_call_center_sk, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) AS sales, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_net_profit) AS profit + FROM catalog_sales, + date_dim + WHERE cs_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + GROUP BY cs_call_center_sk), +cr AS (SELECT cr_call_center_sk, + as_noised_sum(priv_hash(hash(cr_returning_customer_sk)), cr_return_amount) AS returns_, + as_noised_sum(priv_hash(hash(cr_returning_customer_sk)), cr_net_loss) AS profit_loss + FROM catalog_returns, + date_dim + WHERE cr_returned_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + GROUP BY cr_call_center_sk ), +ws AS (SELECT wp_web_page_sk, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) AS sales, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_net_profit) AS profit + FROM web_sales, + date_dim, + web_page + WHERE ws_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND ws_web_page_sk = wp_web_page_sk + GROUP BY wp_web_page_sk), +wr AS (SELECT wp_web_page_sk, + as_noised_sum(priv_hash(hash(wr_returning_customer_sk)), wr_return_amt) AS returns_, + as_noised_sum(priv_hash(hash(wr_returning_customer_sk)), wr_net_loss) AS profit_loss + FROM web_returns, + date_dim, + web_page + WHERE wr_returned_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND wr_web_page_sk = wp_web_page_sk + GROUP BY wp_web_page_sk) +SELECT channel , + id , + sum(sales) AS sales , + sum(returns_) AS returns_ , + sum(profit) AS profit +FROM + (SELECT 'store channel' AS channel , + ss.s_store_sk AS id , + sales , + coalesce(returns_, 0) AS returns_ , + (profit - coalesce(profit_loss,0)) AS profit + FROM ss + LEFT JOIN sr ON ss.s_store_sk = sr.s_store_sk + UNION ALL SELECT 'catalog channel' AS channel , + cs_call_center_sk AS id , + sales , + returns_ , + (profit - profit_loss) AS profit + FROM cs , + cr + UNION ALL SELECT 'web channel' AS channel , + ws.wp_web_page_sk AS id , + sales , + coalesce(returns_, 0) returns_ , + (profit - coalesce(profit_loss,0)) AS profit + FROM ws + LEFT JOIN wr ON ws.wp_web_page_sk = wr.wp_web_page_sk ) x +GROUP BY ROLLUP (channel, + id) +ORDER BY channel NULLS FIRST, + id NULLS FIRST, + returns_ DESC +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q78.sql b/benchmark/tpcds/as_queries/q78.sql new file mode 100644 index 00000000..d91eeae1 --- /dev/null +++ b/benchmark/tpcds/as_queries/q78.sql @@ -0,0 +1,75 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Preserve original sales/returns outer joins; ownership comes from each sales branch. +WITH ws AS (SELECT d_year AS ws_sold_year, + ws_item_sk, + ws_bill_customer_sk ws_customer_sk, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_quantity) ws_qty, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_wholesale_cost) ws_wc, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_sales_price) ws_sp + FROM web_sales + LEFT JOIN web_returns ON wr_order_number=ws_order_number + AND ws_item_sk=wr_item_sk + JOIN date_dim ON ws_sold_date_sk = d_date_sk + WHERE wr_order_number IS NULL + GROUP BY d_year, + ws_item_sk, + ws_bill_customer_sk ), +cs AS (SELECT d_year AS cs_sold_year, + cs_item_sk, + cs_bill_customer_sk cs_customer_sk, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_quantity) cs_qty, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_wholesale_cost) cs_wc, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_sales_price) cs_sp + FROM catalog_sales + LEFT JOIN catalog_returns ON cr_order_number=cs_order_number + AND cs_item_sk=cr_item_sk + JOIN date_dim ON cs_sold_date_sk = d_date_sk + WHERE cr_order_number IS NULL + GROUP BY d_year, + cs_item_sk, + cs_bill_customer_sk ), +ss AS (SELECT d_year AS ss_sold_year, + ss_item_sk, + ss_customer_sk, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) ss_qty, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_wholesale_cost) ss_wc, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_sales_price) ss_sp + FROM store_sales + LEFT JOIN store_returns ON sr_ticket_number=ss_ticket_number + AND ss_item_sk=sr_item_sk + JOIN date_dim ON ss_sold_date_sk = d_date_sk + WHERE sr_ticket_number IS NULL + GROUP BY d_year, + ss_item_sk, + ss_customer_sk ) +SELECT ss_sold_year, + ss_item_sk, + ss_customer_sk, + round((ss_qty*1.00)/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, + ss_qty store_qty, + ss_wc store_wholesale_cost, + ss_sp store_sales_price, + coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, + coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, + coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price +FROM ss +LEFT JOIN ws ON (ws_sold_year=ss_sold_year + AND ws_item_sk=ss_item_sk + AND ws_customer_sk=ss_customer_sk) +LEFT JOIN cs ON (cs_sold_year=ss_sold_year + AND cs_item_sk=ss_item_sk + AND cs_customer_sk=ss_customer_sk) +WHERE (coalesce(ws_qty,0)>0 + OR coalesce(cs_qty, 0)>0) + AND ss_sold_year=2000 +ORDER BY ss_sold_year, + ss_item_sk, + ss_customer_sk, + ss_qty DESC, + ss_wc DESC, + ss_sp DESC, + other_chan_qty, + other_chan_wholesale_cost, + other_chan_sales_price, + ratio +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q79.sql b/benchmark/tpcds/as_queries/q79.sql new file mode 100644 index 00000000..670f5494 --- /dev/null +++ b/benchmark/tpcds/as_queries/q79.sql @@ -0,0 +1,39 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +SELECT c_last_name, + c_first_name, + SUBSTRING(s_city,1,30), + ss_ticket_number, + amt, + profit +FROM + (SELECT ss_ticket_number , + ss_customer_sk , + store.s_city , + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) amt , + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) profit + FROM store_sales, + date_dim, + store, + household_demographics + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND (household_demographics.hd_dep_count = 6 + OR household_demographics.hd_vehicle_count > 2) + AND date_dim.d_dow = 1 + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_number_employees BETWEEN 200 AND 295 + GROUP BY ss_ticket_number, + ss_customer_sk, + ss_addr_sk, + store.s_city) ms, + customer +WHERE ss_customer_sk = c_customer_sk +ORDER BY c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + SUBSTRING(s_city,1,30) NULLS FIRST, + profit NULLS FIRST, + ss_ticket_number +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q80.sql b/benchmark/tpcds/as_queries/q80.sql new file mode 100644 index 00000000..ae118341 --- /dev/null +++ b/benchmark/tpcds/as_queries/q80.sql @@ -0,0 +1,85 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Stochastic sales/returns expressions share the sales-side owner; outer ROLLUP stays scalar. +WITH ssr AS (SELECT s_store_id AS store_id, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS sales, + as_noised_sum(priv_hash(hash(ss_customer_sk)), coalesce(sr_return_amt, 0)) AS returns_, + as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit - coalesce(sr_net_loss, 0)) AS profit + FROM store_sales + LEFT OUTER JOIN store_returns ON (ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number), date_dim, + store, + item, + promotion + WHERE ss_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND ss_store_sk = s_store_sk + AND ss_item_sk = i_item_sk + AND i_current_price > 50 + AND ss_promo_sk = p_promo_sk + AND p_channel_tv = 'N' + GROUP BY s_store_id), +csr AS (SELECT cp_catalog_page_id AS catalog_page_id, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) AS sales, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), coalesce(cr_return_amount, 0)) AS returns_, + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_net_profit - coalesce(cr_net_loss, 0)) AS profit + FROM catalog_sales + LEFT OUTER JOIN catalog_returns ON (cs_item_sk = cr_item_sk + AND cs_order_number = cr_order_number), date_dim, + catalog_page, + item, + promotion + WHERE cs_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND cs_catalog_page_sk = cp_catalog_page_sk + AND cs_item_sk = i_item_sk + AND i_current_price > 50 + AND cs_promo_sk = p_promo_sk + AND p_channel_tv = 'N' + GROUP BY cp_catalog_page_id), +wsr AS (SELECT web_site_id, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) AS sales, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), coalesce(wr_return_amt, 0)) AS returns_, + as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_net_profit - coalesce(wr_net_loss, 0)) AS profit + FROM web_sales + LEFT OUTER JOIN web_returns ON (ws_item_sk = wr_item_sk + AND ws_order_number = wr_order_number), date_dim, + web_site, + item, + promotion + WHERE ws_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND ws_web_site_sk = web_site_sk + AND ws_item_sk = i_item_sk + AND i_current_price > 50 + AND ws_promo_sk = p_promo_sk + AND p_channel_tv = 'N' + GROUP BY web_site_id) +SELECT channel , + id , + sum(sales) AS sales , + sum(returns_) AS returns_ , + sum(profit) AS profit +FROM + (SELECT 'store channel' AS channel , + concat('store', store_id) AS id , + sales , + returns_ , + profit + FROM ssr + UNION ALL SELECT 'catalog channel' AS channel , + concat('catalog_page', catalog_page_id) AS id , + sales , + returns_ , + profit + FROM csr + UNION ALL SELECT 'web channel' AS channel , + concat('web_site', web_site_id) AS id , + sales , + returns_ , + profit + FROM wsr ) x +GROUP BY ROLLUP (channel, + id) +ORDER BY channel NULLS FIRST, + id NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q81.sql b/benchmark/tpcds/as_queries/q81.sql new file mode 100644 index 00000000..7549b3f6 --- /dev/null +++ b/benchmark/tpcds/as_queries/q81.sql @@ -0,0 +1,55 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Per-customer CTE totals stay scalar; only the correlated cross-customer AVG supplies a counter list. +WITH customer_total_return AS (SELECT cr_returning_customer_sk AS ctr_customer_sk , + ca_state AS ctr_state, + sum(cr_return_amt_inc_tax) AS ctr_total_return + FROM catalog_returns , + date_dim , + customer_address + WHERE cr_returned_date_sk = d_date_sk + AND d_year = 2000 + AND cr_returning_addr_sk = ca_address_sk + GROUP BY cr_returning_customer_sk , + ca_state) +SELECT c_customer_id, + c_salutation, + c_first_name, + c_last_name, + ca_street_number, + ca_street_name , + ca_street_type, + ca_suite_number, + ca_city, + ca_county, + ca_state, + ca_zip, + ca_country, + ca_gmt_offset , + ca_location_type, + ctr_total_return +FROM customer_total_return ctr1 , + customer_address , + customer +WHERE priv_filter_gt(CAST(CAST(ctr1.ctr_total_return AS DOUBLE) * 0.8333333333333334 AS FLOAT), (SELECT priv_div(as_sum(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return), as_count(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return)) + FROM customer_total_return ctr2 + WHERE ctr1.ctr_state = ctr2.ctr_state)) + AND ca_address_sk = c_current_addr_sk + AND ca_state = 'GA' + AND ctr1.ctr_customer_sk = c_customer_sk +ORDER BY c_customer_id, + c_salutation, + c_first_name, + c_last_name, + ca_street_number, + ca_street_name , + ca_street_type, + ca_suite_number, + ca_city, + ca_county, + ca_state, + ca_zip, + ca_country, + ca_gmt_offset , + ca_location_type, + ctr_total_return +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q83.sql b/benchmark/tpcds/as_queries/q83.sql new file mode 100644 index 00000000..9f365214 --- /dev/null +++ b/benchmark/tpcds/as_queries/q83.sql @@ -0,0 +1,68 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +WITH sr_items AS (SELECT i_item_id item_id, + as_noised_sum(priv_hash(hash(sr_customer_sk)), sr_return_quantity) sr_item_qty + FROM store_returns, + item, + date_dim + WHERE sr_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq IN + (SELECT d_week_seq + FROM date_dim + WHERE d_date IN ('2000-06-30', + '2000-09-27', + '2000-11-17'))) + AND sr_returned_date_sk = d_date_sk + GROUP BY i_item_id), +cr_items AS (SELECT i_item_id item_id, + as_noised_sum(priv_hash(hash(cr_returning_customer_sk)), cr_return_quantity) cr_item_qty + FROM catalog_returns, + item, + date_dim + WHERE cr_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq IN + (SELECT d_week_seq + FROM date_dim + WHERE d_date IN ('2000-06-30', + '2000-09-27', + '2000-11-17'))) + AND cr_returned_date_sk = d_date_sk + GROUP BY i_item_id), +wr_items AS (SELECT i_item_id item_id, + as_noised_sum(priv_hash(hash(wr_returning_customer_sk)), wr_return_quantity) wr_item_qty + FROM web_returns, + item, + date_dim + WHERE wr_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq IN + (SELECT d_week_seq + FROM date_dim + WHERE d_date IN ('2000-06-30', + '2000-09-27', + '2000-11-17'))) + AND wr_returned_date_sk = d_date_sk + GROUP BY i_item_id) +SELECT sr_items.item_id , + sr_item_qty , + (sr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 sr_dev , + cr_item_qty , + (cr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 cr_dev , + wr_item_qty , + (wr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 wr_dev , + (sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average +FROM sr_items , + cr_items , + wr_items +WHERE sr_items.item_id=cr_items.item_id + AND sr_items.item_id=wr_items.item_id +ORDER BY sr_items.item_id NULLS FIRST, + sr_item_qty NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q85.sql b/benchmark/tpcds/as_queries/q85.sql new file mode 100644 index 00000000..47ba8f73 --- /dev/null +++ b/benchmark/tpcds/as_queries/q85.sql @@ -0,0 +1,60 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +-- The compiler uses one accessible web-sales customer mask, not a second-mask intersection. + +SELECT SUBSTRING(r_reason_desc,1,20) , + (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), ws_quantity) AS DOUBLE)) avg1, + (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), wr_refunded_cash) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), wr_refunded_cash) AS DOUBLE)) avg2, + (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), wr_fee) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), wr_fee) AS DOUBLE)) +FROM web_sales, + web_returns, + web_page, + customer_demographics cd1, + customer_demographics cd2, + customer_address, + date_dim, + reason +WHERE ws_web_page_sk = wp_web_page_sk + AND ws_item_sk = wr_item_sk + AND ws_order_number = wr_order_number + AND ws_sold_date_sk = d_date_sk + AND d_year = 2000 + AND cd1.cd_demo_sk = wr_refunded_cdemo_sk + AND cd2.cd_demo_sk = wr_returning_cdemo_sk + AND ca_address_sk = wr_refunded_addr_sk + AND r_reason_sk = wr_reason_sk + AND ( ( cd1.cd_marital_status = 'M' + AND cd1.cd_marital_status = cd2.cd_marital_status + AND cd1.cd_education_status = 'Advanced Degree' + AND cd1.cd_education_status = cd2.cd_education_status + AND ws_sales_price BETWEEN 100.00 AND 150.00 ) + OR ( cd1.cd_marital_status = 'S' + AND cd1.cd_marital_status = cd2.cd_marital_status + AND cd1.cd_education_status = 'College' + AND cd1.cd_education_status = cd2.cd_education_status + AND ws_sales_price BETWEEN 50.00 AND 100.00 ) + OR ( cd1.cd_marital_status = 'W' + AND cd1.cd_marital_status = cd2.cd_marital_status + AND cd1.cd_education_status = '2 yr Degree' + AND cd1.cd_education_status = cd2.cd_education_status + AND ws_sales_price BETWEEN 150.00 AND 200.00 ) ) + AND ( ( ca_country = 'United States' + AND ca_state IN ('IN', + 'OH', + 'NJ') + AND ws_net_profit BETWEEN 100 AND 200) + OR ( ca_country = 'United States' + AND ca_state IN ('WI', + 'CT', + 'KY') + AND ws_net_profit BETWEEN 150 AND 300) + OR ( ca_country = 'United States' + AND ca_state IN ('LA', + 'IA', + 'AR') + AND ws_net_profit BETWEEN 50 AND 250) ) +GROUP BY r_reason_desc +ORDER BY SUBSTRING(r_reason_desc,1,20) , + (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), ws_quantity) AS DOUBLE)) , + (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), wr_refunded_cash) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), wr_refunded_cash) AS DOUBLE)) , + (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), wr_fee) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), wr_fee) AS DOUBLE)) +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q88.sql b/benchmark/tpcds/as_queries/q88.sql new file mode 100644 index 00000000..5d178052 --- /dev/null +++ b/benchmark/tpcds/as_queries/q88.sql @@ -0,0 +1,139 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +SELECT * +FROM + (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h8_30_to_9 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 8 + AND time_dim.t_minute >= 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s1, + (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h9_to_9_30 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 9 + AND time_dim.t_minute < 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s2, + (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h9_30_to_10 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 9 + AND time_dim.t_minute >= 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s3, + (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h10_to_10_30 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 10 + AND time_dim.t_minute < 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s4, + (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h10_30_to_11 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 10 + AND time_dim.t_minute >= 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s5, + (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h11_to_11_30 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 11 + AND time_dim.t_minute < 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s6, + (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h11_30_to_12 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 11 + AND time_dim.t_minute >= 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s7, + (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h12_to_12_30 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 12 + AND time_dim.t_minute < 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s8 ; diff --git a/benchmark/tpcds/as_queries/q90.sql b/benchmark/tpcds/as_queries/q90.sql new file mode 100644 index 00000000..2de65cf8 --- /dev/null +++ b/benchmark/tpcds/as_queries/q90.sql @@ -0,0 +1,26 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- Evaluate the NULL guard and DECIMAL ratio lane-wise before FLOAT-list finalization. +SELECT CAST(as_noised(list_transform(list_zip(amc,pmc), lambda x: CASE WHEN x[2]=0 THEN NULL ELSE CAST(x[1] AS DECIMAL(15,4))/CAST(x[2] AS DECIMAL(15,4)) END)::FLOAT[]) AS DOUBLE) AS am_pm_ratio +FROM (SELECT as_count(priv_hash(hash(ws_bill_customer_sk)), 1::BIGINT) amc + FROM web_sales, + household_demographics, + time_dim, + web_page + WHERE ws_sold_time_sk = time_dim.t_time_sk + AND ws_ship_hdemo_sk = household_demographics.hd_demo_sk + AND ws_web_page_sk = web_page.wp_web_page_sk + AND time_dim.t_hour BETWEEN 8 AND 8+1 + AND household_demographics.hd_dep_count = 6 + AND web_page.wp_char_count BETWEEN 5000 AND 5200) at_, (SELECT as_count(priv_hash(hash(ws_bill_customer_sk)), 1::BIGINT) pmc + FROM web_sales, + household_demographics, + time_dim, + web_page + WHERE ws_sold_time_sk = time_dim.t_time_sk + AND ws_ship_hdemo_sk = household_demographics.hd_demo_sk + AND ws_web_page_sk = web_page.wp_web_page_sk + AND time_dim.t_hour BETWEEN 19 AND 19+1 + AND household_demographics.hd_dep_count = 6 + AND web_page.wp_char_count BETWEEN 5000 AND 5200) pt +ORDER BY am_pm_ratio +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q91.sql b/benchmark/tpcds/as_queries/q91.sql new file mode 100644 index 00000000..cc0f31e5 --- /dev/null +++ b/benchmark/tpcds/as_queries/q91.sql @@ -0,0 +1,31 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT cc_call_center_id Call_Center, + cc_name Call_Center_Name, + cc_manager Manager, + as_noised_sum(priv_hash(hash(c_customer_sk)), cr_net_loss) Returns_Loss +FROM call_center, + catalog_returns, + date_dim, + customer, + customer_address, + customer_demographics, + household_demographics +WHERE cr_call_center_sk = cc_call_center_sk + AND cr_returned_date_sk = d_date_sk + AND cr_returning_customer_sk= c_customer_sk + AND cd_demo_sk = c_current_cdemo_sk + AND hd_demo_sk = c_current_hdemo_sk + AND ca_address_sk = c_current_addr_sk + AND d_year = 1998 + AND d_moy = 11 + AND ((cd_marital_status = 'M' + AND cd_education_status = 'Unknown') or(cd_marital_status = 'W' + AND cd_education_status = 'Advanced Degree')) + AND hd_buy_potential LIKE 'Unknown%' + AND ca_gmt_offset = -7 +GROUP BY cc_call_center_id, + cc_name, + cc_manager, + cd_marital_status, + cd_education_status +ORDER BY as_noised_sum(priv_hash(hash(c_customer_sk)), cr_net_loss) DESC; diff --git a/benchmark/tpcds/as_queries/q92.sql b/benchmark/tpcds/as_queries/q92.sql new file mode 100644 index 00000000..6f041b12 --- /dev/null +++ b/benchmark/tpcds/as_queries/q92.sql @@ -0,0 +1,22 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +SELECT as_noised_sum(priv_select_gt(priv_hash(hash(ws_bill_customer_sk)), CAST(ws_ext_discount_amt AS FLOAT), (SELECT list_transform(priv_div(as_sum(priv_hash(hash(ws_bill_customer_sk)), CAST(ws_ext_discount_amt AS DOUBLE)), as_count(priv_hash(hash(ws_bill_customer_sk)), ws_ext_discount_amt)), lambda x: CAST(1.3 * x AS FLOAT)) + FROM web_sales, + date_dim + WHERE ws_item_sk = i_item_sk + AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) + AND d_date_sk = ws_sold_date_sk )), ws_ext_discount_amt) AS "Excess Discount Amount" +FROM web_sales, + item, + date_dim +WHERE i_manufact_id = 350 + AND i_item_sk = ws_item_sk + AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) + AND d_date_sk = ws_sold_date_sk + AND priv_select_gt(priv_hash(hash(ws_bill_customer_sk)), CAST(ws_ext_discount_amt AS FLOAT), (SELECT list_transform(priv_div(as_sum(priv_hash(hash(ws_bill_customer_sk)), CAST(ws_ext_discount_amt AS DOUBLE)), as_count(priv_hash(hash(ws_bill_customer_sk)), ws_ext_discount_amt)), lambda x: CAST(1.3 * x AS FLOAT)) + FROM web_sales, + date_dim + WHERE ws_item_sk = i_item_sk + AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) + AND d_date_sk = ws_sold_date_sk )) <> 0 +ORDER BY "Excess Discount Amount" +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q93.sql b/benchmark/tpcds/as_queries/q93.sql new file mode 100644 index 00000000..8be9d191 --- /dev/null +++ b/benchmark/tpcds/as_queries/q93.sql @@ -0,0 +1,20 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +SELECT ss_customer_sk, + as_noised_sum(priv_hash(hash(ss_customer_sk)), act_sales) sumsales +FROM + (SELECT ss_item_sk, + ss_ticket_number, + ss_customer_sk, + CASE + WHEN sr_return_quantity IS NOT NULL THEN (ss_quantity-sr_return_quantity)*ss_sales_price + ELSE (ss_quantity*ss_sales_price) + END act_sales + FROM store_sales + LEFT OUTER JOIN store_returns ON (sr_item_sk = ss_item_sk + AND sr_ticket_number = ss_ticket_number) ,reason + WHERE sr_reason_sk = r_reason_sk + AND r_reason_desc = 'reason 28') t +GROUP BY ss_customer_sk +ORDER BY sumsales NULLS FIRST, + ss_customer_sk NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q94.sql b/benchmark/tpcds/as_queries/q94.sql new file mode 100644 index 00000000..3b2858b8 --- /dev/null +++ b/benchmark/tpcds/as_queries/q94.sql @@ -0,0 +1,24 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- COUNT(DISTINCT) unions membership by order; EXISTS/NOT EXISTS retain their original SQL semantics. +WITH asa_input AS NOT MATERIALIZED (SELECT ws_order_number, ws_ext_ship_cost, ws_net_profit, priv_hash(hash(ws1.ws_bill_customer_sk)) AS membership FROM web_sales ws1 , + date_dim , + customer_address , + web_site +WHERE d_date BETWEEN '1999-02-01' AND cast('1999-04-02' AS date) + AND ws1.ws_ship_date_sk = d_date_sk + AND ws1.ws_ship_addr_sk = ca_address_sk + AND ca_state = 'IL' + AND ws1.ws_web_site_sk = web_site_sk + AND web_company_name = 'pri' + AND EXISTS + (SELECT * + FROM web_sales ws2 + WHERE ws1.ws_order_number = ws2.ws_order_number + AND ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) + AND NOT exists + (SELECT * + FROM web_returns wr1 + WHERE ws1.ws_order_number = wr1.wr_order_number) +), +asa_distinct AS NOT MATERIALIZED (SELECT ws_order_number, bit_or(membership) FILTER (WHERE ws_order_number IS NOT NULL) AS membership FROM asa_input GROUP BY ws_order_number) +SELECT d.n AS "order count", s.shipping AS "total shipping cost", s.profit AS "total net profit" FROM (SELECT as_noised_count(membership, 1::BIGINT) AS n FROM asa_distinct) d CROSS JOIN (SELECT as_noised_sum(membership, ws_ext_ship_cost) AS shipping, as_noised_sum(membership, ws_net_profit) AS profit FROM asa_input) s ORDER BY "order count" LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q95.sql b/benchmark/tpcds/as_queries/q95.sql new file mode 100644 index 00000000..32c7aedd --- /dev/null +++ b/benchmark/tpcds/as_queries/q95.sql @@ -0,0 +1,30 @@ +-- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. +-- The ws_wh CTE stays ordinary; order membership unions and SUMs use the outer web-sales owner. +WITH ws_wh AS NOT MATERIALIZED (SELECT ws1.ws_order_number, + ws1.ws_warehouse_sk wh1, + ws2.ws_warehouse_sk wh2 + FROM web_sales ws1, + web_sales ws2 + WHERE ws1.ws_order_number = ws2.ws_order_number + AND ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk), +asa_input AS NOT MATERIALIZED (SELECT ws_order_number, ws_ext_ship_cost, ws_net_profit, priv_hash(hash(ws1.ws_bill_customer_sk)) AS membership FROM web_sales ws1 , + date_dim , + customer_address , + web_site +WHERE d_date BETWEEN '1999-02-01' AND cast('1999-04-02' AS date) + AND ws1.ws_ship_date_sk = d_date_sk + AND ws1.ws_ship_addr_sk = ca_address_sk + AND ca_state = 'IL' + AND ws1.ws_web_site_sk = web_site_sk + AND web_company_name = 'pri' + AND ws1.ws_order_number IN + (SELECT ws_order_number + FROM ws_wh) + AND ws1.ws_order_number IN + (SELECT wr_order_number + FROM web_returns, + ws_wh + WHERE wr_order_number = ws_wh.ws_order_number) +), +asa_distinct AS NOT MATERIALIZED (SELECT ws_order_number, bit_or(membership) FILTER (WHERE ws_order_number IS NOT NULL) AS membership FROM asa_input GROUP BY ws_order_number) +SELECT d.n AS "order count", s.shipping AS "total shipping cost", s.profit AS "total net profit" FROM (SELECT as_noised_count(membership, 1::BIGINT) AS n FROM asa_distinct) d CROSS JOIN (SELECT as_noised_sum(membership, ws_ext_ship_cost) AS shipping, as_noised_sum(membership, ws_net_profit) AS profit FROM asa_input) s ORDER BY "order count" LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q96.sql b/benchmark/tpcds/as_queries/q96.sql new file mode 100644 index 00000000..32da38b4 --- /dev/null +++ b/benchmark/tpcds/as_queries/q96.sql @@ -0,0 +1,15 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) +FROM store_sales , + household_demographics, + time_dim, + store +WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 20 + AND time_dim.t_minute >= 30 + AND household_demographics.hd_dep_count = 7 + AND store.s_store_name = 'ese' +ORDER BY as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) +LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q99.sql b/benchmark/tpcds/as_queries/q99.sql new file mode 100644 index 00000000..4b680ee8 --- /dev/null +++ b/benchmark/tpcds/as_queries/q99.sql @@ -0,0 +1,45 @@ +-- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. +SELECT w_substr , + sm_type , + LOWER(cc_name) cc_name_lower , + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE + WHEN (cs_ship_date_sk - cs_sold_date_sk <= 30) THEN 1 + ELSE 0 + END) AS "30 days", + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE + WHEN (cs_ship_date_sk - cs_sold_date_sk > 30) + AND (cs_ship_date_sk - cs_sold_date_sk <= 60) THEN 1 + ELSE 0 + END) AS "31-60 days", + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE + WHEN (cs_ship_date_sk - cs_sold_date_sk > 60) + AND (cs_ship_date_sk - cs_sold_date_sk <= 90) THEN 1 + ELSE 0 + END) AS "61-90 days", + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE + WHEN (cs_ship_date_sk - cs_sold_date_sk > 90) + AND (cs_ship_date_sk - cs_sold_date_sk <= 120) THEN 1 + ELSE 0 + END) AS "91-120 days", + as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE + WHEN (cs_ship_date_sk - cs_sold_date_sk > 120) THEN 1 + ELSE 0 + END) AS ">120 days" +FROM catalog_sales , + (SELECT SUBSTRING(w_warehouse_name,1,20) w_substr, * + FROM warehouse) AS sq1 , + ship_mode , + call_center , + date_dim +WHERE d_month_seq BETWEEN 1200 AND 1200 + 11 + AND cs_ship_date_sk = d_date_sk + AND cs_warehouse_sk = w_warehouse_sk + AND cs_ship_mode_sk = sm_ship_mode_sk + AND cs_call_center_sk = cc_call_center_sk +GROUP BY w_substr , + sm_type , + cc_name +ORDER BY w_substr NULLS FIRST, + sm_type NULLS FIRST, + cc_name_lower NULLS FIRST +LIMIT 100; diff --git a/benchmark/tpcds/as_tpcds_benchmark.cpp b/benchmark/tpcds/as_tpcds_benchmark.cpp index df757b4f..d622a741 100644 --- a/benchmark/tpcds/as_tpcds_benchmark.cpp +++ b/benchmark/tpcds/as_tpcds_benchmark.cpp @@ -1,10 +1,14 @@ -// TPC-DS coverage and performance use the original SQL and the automatic rewriter. -// Execution success is not an independent validation of per-world correctness. +// Performance/coverage follows TPC-H's PAC output semantics: 64 internal worlds and scalar/categorical outputs. +// pac_mi=0 disables additive noise; priv_filter still uses PAC's deterministic majority decision. +// The timed AS path uses handwritten PAC SQL, with automatic compilation recorded separately for coverage. +// Per-world AS references are separate diagnostics, not compiler-equivalence or timing oracles. #include "duckdb.hpp" #include "benchmark_json.hpp" #include "benchmark_paths.hpp" +#include #include +#include #include #include #include @@ -39,7 +43,7 @@ static string Csv(const string &value) { return out + "\""; } -static unique_ptr Query(Connection &con, const string &sql) { +static unique_ptr Query(Connection &con, const string &sql) { auto result = con.Query(sql); if (!result || result->HasError()) { throw std::runtime_error(result ? result->GetError() : "No result"); @@ -53,10 +57,363 @@ static string Name(int query) { return name.str(); } +// Separate per-world reference queries expand reduced aggregate results, never base tuples. +using WorldRows = vector>>; + +static void SetHandwrittenAggregates(Connection &con, bool conditional, idx_t first_world, idx_t world_count) { + for (auto &operation : + {string("sum"), string("count"), string("avg"), string("stddev_samp"), string("min"), string("max")}) { + string expression; + if (conditional) { + expression = "["; + for (idx_t world = 0; world < 64; world++) { + if (world) { + expression += ","; + } + if (world < first_world || world >= first_world + world_count) { + expression += "NULL"; + } else { + expression += operation + "(v) FILTER (WHERE (m & " + std::to_string(uint64_t(1) << world) + + "::UBIGINT) <> 0)"; + } + } + expression += "]"; + } else if (operation == "stddev_samp") { + expression = "as_stddev_samp(m, v)"; + } else if (operation == "count") { + expression = "list_transform(as_count(m, v), lambda x: x / 2.0)"; + } else { + auto aggregate = operation == "avg" ? "sum" : operation; + auto value = operation == "avg" ? "x[1] / x[2]" : operation == "sum" ? "x[1] / 2.0" : "x[1]"; + // MIN/MAX dispatch on their SQL input type during binding, including macro definition. + auto argument = operation == "min" || operation == "max" ? "CAST(v AS DOUBLE)" : "v"; + expression = "list_transform(list_zip(as_" + string(aggregate) + "(m, " + argument + + "), as_count(m, v)), lambda x: CASE WHEN x[2] = 0 THEN NULL ELSE " + value + " END)"; + } + Query(con, "CREATE OR REPLACE TEMP MACRO tpcds_" + operation + "(m, v) AS " + expression); + } +} + +static WorldRows CollectHandwritten(QueryResult &result) { + WorldRows rows(64); + for (idx_t r = 0; r < result.RowCount(); r++) { + auto world = result.Collection().GetValue(0, r).GetValue(); + if (world < 0 || world >= 64) { + throw std::runtime_error("Invalid handwritten world index"); + } + vector row; + for (idx_t col = 1; col < result.ColumnCount(); col++) { + row.push_back(result.Collection().GetValue(col, r)); + } + rows[world].push_back(std::move(row)); + } + return rows; +} + +static void InstallSampledViews(Connection &con) { + Query(con, "SET VARIABLE tpcds_world=0"); + Query(con, "CREATE SCHEMA IF NOT EXISTS tpcds_sampled"); + Query(con, "CREATE OR REPLACE VIEW tpcds_sampled.customer AS SELECT * FROM main.customer " + "WHERE ((priv_hash(hash(c_customer_sk)) >> getvariable('tpcds_world')) & 1) <> 0"); + for (const auto &owner : vector> {{"store_sales", "ss_customer_sk"}, + {"store_returns", "sr_customer_sk"}, + {"catalog_sales", "cs_bill_customer_sk"}, + {"catalog_returns", "cr_returning_customer_sk"}, + {"web_sales", "ws_bill_customer_sk"}, + {"web_returns", "wr_returning_customer_sk"}}) { + Query(con, "CREATE OR REPLACE VIEW tpcds_sampled." + owner.first + " AS SELECT * FROM main." + owner.first + + " WHERE " + owner.second + " IN (SELECT c_customer_sk FROM tpcds_sampled.customer)"); + } +} + +struct ValidationComparison { + idx_t rows = 0; + idx_t values = 0; + idx_t exact_mismatches = 0; + idx_t float_mismatches = 0; + idx_t empty_worlds = 0; + double max_absolute_error = 0; + double max_relative_error = 0; +}; + +// SQL ORDER BY may leave ties unresolved. Compare result multisets, retaining the SQL's per-world LIMIT. +static vector CanonicalRowOrder(const vector> &rows, const vector &keys) { + vector order; + vector labels; + for (idx_t row = 0; row < rows.size(); row++) { + order.push_back(row); + string key; + for (idx_t col = 0; col < keys.size(); col++) { + if (keys[col]) { + auto value = rows[row][col].IsNull() ? string("NULL") : rows[row][col].ToString(); + key += std::to_string(value.size()) + ":" + value; + } + } + labels.push_back(std::move(key)); + } + std::sort(order.begin(), order.end(), [&](idx_t a, idx_t b) { + if (labels[a] != labels[b]) { + return labels[a] < labels[b]; + } + for (idx_t col = 0; col < keys.size(); col++) { + if (keys[col]) { + continue; + } + auto &x = rows[a][col]; + auto &y = rows[b][col]; + if (x.IsNull() != y.IsNull()) { + return x.IsNull(); + } + if (!x.IsNull()) { + auto xv = x.GetValue(), yv = y.GetValue(); + if (std::isnan(xv) != std::isnan(yv)) { + return std::isnan(xv); + } + if (xv != yv && !(std::isnan(xv) && std::isnan(yv))) { + return xv < yv; + } + } + } + return false; + }); + return order; +} + +static ValidationComparison CompareWorldRows(const WorldRows &expected, const WorldRows &actual, idx_t world_count = 64, + bool integral_keys = true) { + ValidationComparison comparison; + for (idx_t world = 0; world < world_count; world++) { + if (expected[world].size() != actual[world].size()) { + throw std::runtime_error("World " + std::to_string(world) + " row count: expected " + + std::to_string(expected[world].size()) + ", got " + + std::to_string(actual[world].size())); + } + comparison.empty_worlds += expected[world].empty(); + comparison.rows += expected[world].size(); + if (expected[world].empty()) { + continue; + } + if (expected[world][0].size() != actual[world][0].size()) { + throw std::runtime_error("Handwritten output column count differs"); + } + vector keys; + for (idx_t col = 0; col < expected[world][0].size(); col++) { + auto &e = expected[world][0][col].type(); + auto &a = actual[world][0][col].type(); + keys.push_back( + !e.IsNumeric() || !a.IsNumeric() || + (integral_keys && e == a && e.id() != LogicalTypeId::DOUBLE && e.id() != LogicalTypeId::FLOAT)); + } + auto eo = CanonicalRowOrder(expected[world], keys); + auto ao = CanonicalRowOrder(actual[world], keys); + for (idx_t row = 0; row < expected[world].size(); row++) { + if (expected[world][eo[row]].size() != actual[world][ao[row]].size()) { + throw std::runtime_error("Handwritten output column count differs"); + } + for (idx_t col = 0; col < expected[world][row].size(); col++) { + auto &e = expected[world][eo[row]][col]; + auto &a = actual[world][ao[row]][col]; + auto location = "world=" + std::to_string(world) + ", row=" + std::to_string(row) + + ", column=" + std::to_string(col); + if (e.IsNull() != a.IsNull()) { + throw std::runtime_error("NULL mismatch: " + location); + } + if (e.IsNull()) { + continue; + } + if (!e.type().IsNumeric() || !a.type().IsNumeric()) { + if (e.ToString() != a.ToString()) { + throw std::runtime_error("Value/order mismatch: " + location); + } + continue; + } + double x = e.GetValue(), y = a.GetValue(); + if (x == y || (std::isnan(x) && std::isnan(y))) { + comparison.values++; + continue; + } + if (!std::isfinite(x) || !std::isfinite(y)) { + throw std::runtime_error("Non-finite value mismatch: " + location); + } + comparison.exact_mismatches++; + comparison.float_mismatches += static_cast(x) != static_cast(y); + comparison.values++; + auto error = std::abs(x - y); + comparison.max_absolute_error = std::max(comparison.max_absolute_error, error); + comparison.max_relative_error = + std::max(comparison.max_relative_error, error / std::max(1.0, std::abs(x))); + } + } + } + return comparison; +} + +static void ValidateHandwritten(Connection &con, const JsonValue &config, int id) { + auto name = Name(id); + auto output = JsonString(config, "out_dir"); + auto directory = JsonString(config, "reference_queries_dir", "benchmark/tpcds/reference_queries"); + auto handwritten = Read(directory + "/" + name + ".sql"); + auto original = Read(JsonString(config, "queries_dir") + "/" + name + ".sql"); + Query(con, "SET priv_rewrite=false"); + InstallSampledViews(con); + Query(con, "SET search_path='tpcds_sampled,main'"); + WorldRows expected(64); + for (idx_t world = 0; world < 64; world++) { + Query(con, "SET VARIABLE tpcds_world=" + std::to_string(world)); + auto result = Query(con, original); + for (idx_t row = 0; row < result->RowCount(); row++) { + vector values; + for (idx_t col = 0; col < result->ColumnCount(); col++) { + values.push_back(result->Collection().GetValue(col, row)); + } + expected[world].push_back(std::move(values)); + } + } + std::ofstream csv(output + "/" + name + "_validation.csv"); + csv << "query,method,status,worlds,rows,numeric_values,exact_mismatches,float_rounded_mismatches," + "max_absolute_error,max_relative_error,empty_worlds,error\n"; + for (bool conditional : {true, false}) { + auto method = conditional ? "conditional_sql" : "simd_asa_handwritten"; + try { + WorldRows actual(64); + // Bound the ordinary conditional oracle's state size; this is validation, not timing. + auto block_size = conditional ? idx_t(8) : idx_t(64); + for (idx_t first = 0; first < 64; first += block_size) { + SetHandwrittenAggregates(con, conditional, first, block_size); + auto result = Query(con, handwritten); + auto block = CollectHandwritten(*result); + for (idx_t world = first; world < first + block_size; world++) { + actual[world] = std::move(block[world]); + } + } + auto comparison = CompareWorldRows(expected, actual); + auto status = comparison.exact_mismatches ? "numeric_difference" + : comparison.empty_worlds == 64 ? "empty_only" + : "exact_match"; + csv << name << ',' << method << ',' << status << ",64," << comparison.rows << ',' << comparison.values + << ',' << comparison.exact_mismatches << ',' << comparison.float_mismatches << ',' + << std::setprecision(17) << comparison.max_absolute_error << ',' << comparison.max_relative_error << ',' + << comparison.empty_worlds << ",\"\"\n"; + } catch (const std::exception &error) { + csv << name << ',' << method << ",error,64,,,,,,,," << Csv(error.what()) << '\n'; + } + csv.flush(); + } + Query(con, "SET search_path='main'"); + Query(con, "DROP SCHEMA tpcds_sampled CASCADE"); +} + +// TPC-H-style equivalence check: original SQL through the compiler versus explicit PAC SQL. +static void ValidatePacQuery(Connection &con, const JsonValue &config, int id) { + auto name = Name(id); + auto output = JsonString(config, "out_dir"); + auto original = Read(JsonString(config, "queries_dir") + "/" + name + ".sql"); + auto handwritten = Read(JsonString(config, "as_queries_dir", "benchmark/tpcds/as_queries") + "/" + name + ".sql"); + std::ofstream csv(output + "/" + name + "_pac_validation.csv"); + csv << "query,status,compiled_rows,handwritten_rows,numeric_values,exact_mismatches,float_rounded_mismatches," + "max_absolute_error,max_relative_error,error\n"; + idx_t compiled_rows = 0, handwritten_rows = 0; + try { + WorldRows expected(64), actual(64); + for (bool automatic : {true, false}) { + Query(con, "SET priv_rewrite=" + string(automatic ? "true" : "false")); + auto &sql = automatic ? original : handwritten; + auto plan = Query(con, "EXPLAIN " + sql); + std::ofstream plan_file(output + "/" + name + (automatic ? "_compiled.plan.txt" : "_handwritten.plan.txt")); + for (idx_t row = 0; row < plan->RowCount(); row++) { + plan_file << plan->Collection().GetValue(1, row).ToString() << '\n'; + } + auto result = Query(con, sql); + if (automatic) { + compiled_rows = result->RowCount(); + } else { + handwritten_rows = result->RowCount(); + } + auto &rows = automatic ? expected[0] : actual[0]; + for (idx_t row = 0; row < result->RowCount(); row++) { + vector values; + for (idx_t col = 0; col < result->ColumnCount(); col++) { + values.push_back(result->Collection().GetValue(col, row)); + } + rows.push_back(std::move(values)); + } + } + auto comparison = CompareWorldRows(expected, actual, 1, false); + auto status = comparison.exact_mismatches ? "numeric_difference" + : comparison.empty_worlds ? "empty_only" + : "exact_match"; + csv << name << ',' << status << ',' << compiled_rows << ',' << handwritten_rows << ',' << comparison.values + << ',' << comparison.exact_mismatches << ',' << comparison.float_mismatches << ',' << std::setprecision(17) + << comparison.max_absolute_error << ',' << comparison.max_relative_error << ",\"\"\n"; + } catch (const std::exception &error) { + csv << name << ",error," << compiled_rows << ',' << handwritten_rows << ",,,,,," << Csv(error.what()) << '\n'; + } + Query(con, "SET priv_rewrite=false"); +} + +// Output-semantics diagnostic only: ordinary world-zero SQL is not an equivalence oracle for PAC output. +// With mi=0, scalar terminals select lane 0, but PAC terminal filters may use majority decisions and SUM/COUNT +// retain their scaling. Differences recorded here must not be interpreted as compiler-correctness failures. +static void ValidateCompiled(Connection &con, const JsonValue &config, int id) { + auto name = Name(id); + auto output = JsonString(config, "out_dir"); + auto original = Read(JsonString(config, "queries_dir") + "/" + name + ".sql"); + Query(con, "SET priv_rewrite=false"); + InstallSampledViews(con); + Query(con, "SET search_path='tpcds_sampled,main'"); + auto reference = Query(con, original); + WorldRows expected(64), actual(64); + for (idx_t row = 0; row < reference->RowCount(); row++) { + vector values; + for (idx_t col = 0; col < reference->ColumnCount(); col++) { + values.push_back(reference->Collection().GetValue(col, row)); + } + expected[0].push_back(std::move(values)); + } + Query(con, "SET search_path='main'; SET priv_rewrite=true"); + std::ofstream csv(output + "/" + name + "_compiled_validation.csv"); + csv << "query,status,reference_world,reference_rows,compiled_rows,numeric_values,exact_mismatches," + "float_rounded_mismatches,max_absolute_error,max_relative_error,error\n"; + idx_t compiled_rows = 0; + bool executed = false; + try { + auto explain = Query(con, "EXPLAIN " + original); + std::ofstream plan(output + "/" + name + "_compiled_validation.plan.txt"); + for (idx_t row = 0; row < explain->RowCount(); row++) { + plan << explain->Collection().GetValue(1, row).ToString() << '\n'; + } + auto compiled = Query(con, original); + compiled_rows = compiled->RowCount(); + executed = true; + for (idx_t row = 0; row < compiled_rows; row++) { + vector values; + for (idx_t col = 0; col < compiled->ColumnCount(); col++) { + values.push_back(compiled->Collection().GetValue(col, row)); + } + actual[0].push_back(std::move(values)); + } + auto comparison = CompareWorldRows(expected, actual, 1, false); + auto status = comparison.exact_mismatches ? "numeric_difference" + : comparison.empty_worlds ? "empty_only" + : "exact_match"; + csv << name << ',' << status << ",0," << reference->RowCount() << ',' << compiled_rows << ',' + << comparison.values << ',' << comparison.exact_mismatches << ',' << comparison.float_mismatches << ',' + << std::setprecision(17) << comparison.max_absolute_error << ',' << comparison.max_relative_error + << ",\"\"\n"; + } catch (const std::exception &error) { + csv << name << (executed ? ",result_mismatch,0," : ",execution_error,0,") << reference->RowCount() << ',' + << compiled_rows << ",,,,,," << Csv(error.what()) << '\n'; + } + Query(con, "SET priv_rewrite=false; SET search_path='main'"); + Query(con, "DROP SCHEMA tpcds_sampled CASCADE"); +} + int main(int argc, char **argv) { try { - if (argc != 3) { - throw std::runtime_error("Usage: as_tpcds_benchmark CONFIG --prepare|QUERY_NUMBER"); + if (argc != 3 && !(argc == 4 && (string(argv[3]) == "--validate" || string(argv[3]) == "--validate-worlds" || + string(argv[3]) == "--validate-compiled"))) { + throw std::runtime_error("Usage: as_tpcds_benchmark CONFIG --prepare|QUERY_NUMBER " + "[--validate|--validate-worlds|--validate-compiled]"); } auto config = JsonParser(Read(argv[1])).Parse(); auto db_path = JsonString(config, "db"); @@ -77,7 +434,7 @@ int main(int argc, char **argv) { Query(con, "SET memory_limit=" + Quote(JsonString(config, "memory_limit", "4GB"))); // privacy_noise=false bypasses PAC compilation; mi=0 disables additive noise without bypassing it. Query(con, "SET priv_rewrite=false; SET pac_mi=0; SET privacy_noise=true; SET privacy_seed=42; " - "SET pac_sample_diversity_check=false; SET pac_conservative_mode=true;"); + "SET pac_sample_diversity_check=false; SET pac_conservative_mode=true; SET priv_check=false;"); if (string(argv[2]) == "--prepare") { auto extension_dir = JsonString(config, "extension_directory"); if (!extension_dir.empty()) { @@ -86,7 +443,7 @@ int main(int argc, char **argv) { Query(con, "INSTALL tpcds; LOAD tpcds;"); auto tables = Query(con, "SELECT count(*) FROM duckdb_tables() WHERE schema_name='main' " "AND table_name IN ('customer','store_sales','catalog_sales','web_sales')"); - auto present = tables->GetValue(0, 0).GetValue(); + auto present = tables->Collection().GetValue(0, 0).GetValue(); auto sf_value = JsonNumber(config, "sf", 1); if (sf_value <= 0) { throw std::runtime_error("Scale factor must be positive"); @@ -103,15 +460,15 @@ int main(int argc, char **argv) { throw std::runtime_error("Expected all 99 TPC-DS queries"); } for (idx_t row = 0; row < workload->RowCount(); row++) { - auto id = workload->GetValue(0, row).GetValue(); + auto id = workload->Collection().GetValue(0, row).GetValue(); std::ofstream file(queries + "/" + Name(id) + ".sql"); if (!file) { throw std::runtime_error("Cannot export original query"); } - file << workload->GetValue(1, row).ToString(); + file << workload->Collection().GetValue(1, row).ToString(); } std::ofstream metadata(output + "/environment.txt"); - metadata << "DuckDB: " << Query(con, "SELECT version()")->GetValue(0, 0).ToString() << '\n' + metadata << "DuckDB: " << Query(con, "SELECT version()")->Collection().GetValue(0, 0).ToString() << '\n' << "Config:\n" << Read(argv[1]) << "\nOwnership:\n" << Read(schema); @@ -124,7 +481,7 @@ int main(int argc, char **argv) { metadata << "SUM: approximate\n"; #endif metadata << "Settings: m=64, privacy_seed=42, pac_mi=0, privacy_noise=true, " - "pac_sample_diversity_check=false, pac_conservative_mode=true\n"; + "pac_sample_diversity_check=false, pac_conservative_mode=true, priv_check=false\n"; Query(con, "CHECKPOINT"); return 0; } @@ -144,46 +501,83 @@ int main(int argc, char **argv) { Query(con, declaration); } } + if (argc == 4) { + if (string(argv[3]) == "--validate-compiled") { + ValidateCompiled(con, config, id); + return 0; + } + if (string(argv[3]) == "--validate-worlds") { + ValidateHandwritten(con, config, id); + } else { + ValidatePacQuery(con, config, id); + } + return 0; + } std::ofstream csv(output + "/" + name + ".csv"); if (!csv) { throw std::runtime_error("Cannot create result CSV"); } - csv << "query,mode,phase,status,rewritten,repetition,runtime_ms,rows,error\n"; - for (auto mode : {string("duckdb"), string("simd_asa_auto")}) { - Query(con, "SET priv_rewrite=" + string(mode == "duckdb" ? "false" : "true")); + csv << "query,mode,phase,status,rewritten,repetition,runtime_ms,rows,world_multiplier,error\n"; + for (auto mode : + {string("duckdb"), string("simd_asa_handwritten"), string("simd_asa_auto"), string("naive_as")}) { + Query(con, "SET priv_rewrite=" + string(mode == "simd_asa_auto" ? "true" : "false")); + string query_sql = sql; + if (mode == "simd_asa_handwritten") { + auto path = JsonString(config, "as_queries_dir", "benchmark/tpcds/as_queries") + "/" + name + ".sql"; + std::ifstream handwritten(path); + if (!handwritten) { + csv << name << ',' << mode << ",setup,not_available,0,0,,,,\"\"\n"; + continue; + } + query_sql = Read(path); + } + auto multiplier = mode == "naive_as" ? 64 : 1; bool rewritten = false; string phase = "explain"; try { - auto explain = Query(con, "EXPLAIN " + sql); + if (mode == "naive_as") { + // Match TPC-H: prepare once outside timing, time one sampled-world execution, + // and multiply by 64. Views apply customer sampling at query time to every fact scan. + InstallSampledViews(con); + Query(con, "SET search_path='tpcds_sampled,main'"); + Query(con, "PREPARE tpcds_naive AS " + sql); + query_sql = "EXECUTE tpcds_naive"; + } + auto explain = Query(con, "EXPLAIN " + query_sql); string plan; for (idx_t row = 0; row < explain->RowCount(); row++) { - plan += explain->GetValue(1, row).ToString() + "\n"; + plan += explain->Collection().GetValue(1, row).ToString() + "\n"; } std::ofstream plan_file(output + "/" + name + "_" + mode + ".plan.txt"); plan_file << plan; rewritten = plan.find("as_noised_") != string::npos || plan.find("as_sum(") != string::npos || plan.find("as_count(") != string::npos || plan.find("priv_select") != string::npos; - if (mode != "duckdb" && !rewritten) { - csv << name << ',' << mode << ",explain,no_stochastic_operator,0,0,,,\"\"\n"; + if ((mode == "simd_asa_auto" || mode == "simd_asa_handwritten") && !rewritten) { + csv << name << ',' << mode << ",explain,no_stochastic_operator,0,0,,,,\"\"\n"; csv.flush(); continue; } phase = "warmup"; - Query(con, sql); + Query(con, query_sql); phase = "measure"; for (int run = 1; run <= repetitions; run++) { auto start = std::chrono::steady_clock::now(); - auto result = Query(con, sql); + auto result = Query(con, query_sql); auto elapsed = std::chrono::duration(std::chrono::steady_clock::now() - start); csv << name << ',' << mode << ",measure,ok," << rewritten << ',' << run << ',' - << std::setprecision(12) << elapsed.count() << ',' << result->RowCount() << ",\"\"\n"; + << std::setprecision(12) << elapsed.count() * multiplier << ',' << result->RowCount() << ',' + << multiplier << ",\"\"\n"; csv.flush(); } } catch (const std::exception &error) { - csv << name << ',' << mode << ',' << phase << ",error," << rewritten << ",0,,," << Csv(error.what()) + csv << name << ',' << mode << ',' << phase << ",error," << rewritten << ",0,,,," << Csv(error.what()) << '\n'; csv.flush(); } + if (mode == "naive_as") { + Query(con, "SET search_path='main'"); + Query(con, "DROP SCHEMA IF EXISTS tpcds_sampled CASCADE"); + } } return 0; } catch (const std::exception &error) { diff --git a/benchmark/tpcds/q17_fixture.sql b/benchmark/tpcds/q17_fixture.sql new file mode 100644 index 00000000..2beeac68 --- /dev/null +++ b/benchmark/tpcds/q17_fixture.sql @@ -0,0 +1,22 @@ +-- Synthetic nonempty correctness fixture for the unchanged TPC-DS Q17 SQL, not a performance dataset. +-- Run against a fresh database. Empty owned tables allow the standard ownership setup to be reused. +SET priv_rewrite=false; +CREATE TABLE customer AS SELECT i::INTEGER AS c_customer_sk FROM range(128) r(i); +CREATE TABLE store_sales AS +SELECT c_customer_sk AS ss_customer_sk, 1 AS ss_item_sk, c_customer_sk AS ss_ticket_number, + 1 AS ss_sold_date_sk, 1 AS ss_store_sk, c_customer_sk % 10 + 1 AS ss_quantity +FROM customer; +CREATE TABLE store_returns AS +SELECT c_customer_sk AS sr_customer_sk, 1 AS sr_item_sk, c_customer_sk AS sr_ticket_number, + 1 AS sr_returned_date_sk, c_customer_sk % 4 + 1 AS sr_return_quantity +FROM customer; +CREATE TABLE catalog_sales AS +SELECT c_customer_sk AS cs_bill_customer_sk, 1 AS cs_item_sk, 1 AS cs_sold_date_sk, + c_customer_sk % 13 + 1 AS cs_quantity +FROM customer; +CREATE TABLE catalog_returns(cr_returning_customer_sk INTEGER); +CREATE TABLE web_sales(ws_bill_customer_sk INTEGER); +CREATE TABLE web_returns(wr_returning_customer_sk INTEGER); +CREATE TABLE date_dim AS SELECT 1 AS d_date_sk, '2001Q1' AS d_quarter_name; +CREATE TABLE store AS SELECT 1 AS s_store_sk, 'CA' AS s_state; +CREATE TABLE item AS SELECT 1 AS i_item_sk, 'ITEM1' AS i_item_id, 'test item' AS i_item_desc; diff --git a/benchmark/tpcds/reference_queries/q01.sql b/benchmark/tpcds/reference_queries/q01.sql new file mode 100644 index 00000000..5509f88e --- /dev/null +++ b/benchmark/tpcds/reference_queries/q01.sql @@ -0,0 +1,30 @@ +-- Correlated thresholds use only the matching world of the reduced return aggregates. +WITH return_stats AS ( +SELECT sr_customer_sk AS ctr_customer_sk, + sr_store_sk AS ctr_store_sk, + tpcds_sum(priv_hash(hash(sr_customer_sk)), sr_return_amt) AS ctr_total_return, + tpcds_count(priv_hash(hash(sr_customer_sk)),1) AS presence +FROM main.store_returns, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE sr_customer_sk=asa_owner.asa_key AND sr_returned_date_sk = d_date_sk + AND d_year = 2000 + GROUP BY sr_customer_sk, + sr_store_sk +), customer_total_return AS ( +SELECT world, ctr_customer_sk, ctr_store_sk, ctr_total_return[world+1] AS ctr_total_return +FROM return_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT ctr1.world AS world, c_customer_id +FROM customer_total_return ctr1, + main.store, + main.customer +WHERE ctr1.ctr_total_return > + (SELECT avg(ctr_total_return)*1.2 + FROM customer_total_return ctr2 + WHERE ctr1.world=ctr2.world AND ctr1.ctr_store_sk = ctr2.ctr_store_sk) + AND s_store_sk = ctr1.ctr_store_sk + AND s_state = 'TN' + AND ctr1.ctr_customer_sk = c_customer_sk +QUALIFY row_number() OVER (PARTITION BY ctr1.world ORDER BY c_customer_id) <= 100 +ORDER BY ctr1.world, c_customer_id; diff --git a/benchmark/tpcds/reference_queries/q03.sql b/benchmark/tpcds/reference_queries/q03.sql new file mode 100644 index 00000000..06c8dada --- /dev/null +++ b/benchmark/tpcds/reference_queries/q03.sql @@ -0,0 +1,30 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT dt.d_year AS d_year, + item.i_brand_id AS brand_id, + item.i_brand AS brand, + tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS sum_agg, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.date_dim dt, + main.store_sales, + main.item, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND dt.d_date_sk = store_sales.ss_sold_date_sk + AND store_sales.ss_item_sk = item.i_item_sk + AND item.i_manufact_id = 128 + AND dt.d_moy=11 +GROUP BY dt.d_year, + main.item.i_brand, + main.item.i_brand_id +), answers AS ( + SELECT world, d_year, + brand_id, + brand, + sum_agg[world+1] AS sum_agg + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY d_year, sum_agg DESC, brand_id) <= 100 +ORDER BY world, d_year, sum_agg DESC, brand_id; diff --git a/benchmark/tpcds/reference_queries/q04.sql b/benchmark/tpcds/reference_queries/q04.sql new file mode 100644 index 00000000..68905a47 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q04.sql @@ -0,0 +1,135 @@ +-- Shared AS states per channel/customer/year; joins align the world index. +WITH year_total_stats AS ( +SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + tpcds_sum(priv_hash(hash(c_customer_sk)), ((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total, + 's' sale_type, + tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence +FROM main.customer, + main.store_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year + +UNION ALL + SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + tpcds_sum(priv_hash(hash(c_customer_sk)), (((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2)) year_total, + 'c' sale_type, + tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence +FROM main.customer, + main.catalog_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = cs_bill_customer_sk + AND cs_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year + +UNION ALL + SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + tpcds_sum(priv_hash(hash(c_customer_sk)), (((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2)) year_total, + 'w' sale_type, + tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence +FROM main.customer, + main.web_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year +), year_total AS ( +SELECT world, customer_id, customer_first_name, customer_last_name, customer_preferred_cust_flag, customer_birth_country, customer_login, customer_email_address, dyear, year_total[world+1] AS year_total, sale_type +FROM year_total_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT t_s_firstyear.world AS world, t_s_secyear.customer_id, + t_s_secyear.customer_first_name, + t_s_secyear.customer_last_name, + t_s_secyear.customer_preferred_cust_flag +FROM year_total t_s_firstyear, + year_total t_s_secyear, + year_total t_c_firstyear, + year_total t_c_secyear, + year_total t_w_firstyear, + year_total t_w_secyear +WHERE t_s_firstyear.world=t_s_secyear.world AND t_s_firstyear.world=t_c_firstyear.world AND t_s_firstyear.world=t_c_secyear.world AND t_s_firstyear.world=t_w_firstyear.world AND t_s_firstyear.world=t_w_secyear.world AND t_s_secyear.customer_id = t_s_firstyear.customer_id + AND t_s_firstyear.customer_id = t_c_secyear.customer_id + AND t_s_firstyear.customer_id = t_c_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_secyear.customer_id + AND t_s_firstyear.sale_type = 's' + AND t_c_firstyear.sale_type = 'c' + AND t_w_firstyear.sale_type = 'w' + AND t_s_secyear.sale_type = 's' + AND t_c_secyear.sale_type = 'c' + AND t_w_secyear.sale_type = 'w' + AND t_s_firstyear.dyear = 2001 + AND t_s_secyear.dyear = 2001+1 + AND t_c_firstyear.dyear = 2001 + AND t_c_secyear.dyear = 2001+1 + AND t_w_firstyear.dyear = 2001 + AND t_w_secyear.dyear = 2001+1 + AND t_s_firstyear.year_total > 0 + AND t_c_firstyear.year_total > 0 + AND t_w_firstyear.year_total > 0 + AND CASE + WHEN t_c_firstyear.year_total > 0 THEN t_c_secyear.year_total / t_c_firstyear.year_total + ELSE NULL + END > CASE + WHEN t_s_firstyear.year_total > 0 THEN t_s_secyear.year_total / t_s_firstyear.year_total + ELSE NULL + END + AND CASE + WHEN t_c_firstyear.year_total > 0 THEN t_c_secyear.year_total / t_c_firstyear.year_total + ELSE NULL + END > CASE + WHEN t_w_firstyear.year_total > 0 THEN t_w_secyear.year_total / t_w_firstyear.year_total + ELSE NULL + END +QUALIFY row_number() OVER (PARTITION BY t_s_firstyear.world ORDER BY t_s_secyear.customer_id NULLS FIRST, + t_s_secyear.customer_first_name NULLS FIRST, + t_s_secyear.customer_last_name NULLS FIRST, + t_s_secyear.customer_preferred_cust_flag NULLS FIRST) <= 100 +ORDER BY t_s_firstyear.world, t_s_secyear.customer_id NULLS FIRST, + t_s_secyear.customer_first_name NULLS FIRST, + t_s_secyear.customer_last_name NULLS FIRST, + t_s_secyear.customer_preferred_cust_flag NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q06.sql b/benchmark/tpcds/reference_queries/q06.sql new file mode 100644 index 00000000..25dfcac0 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q06.sql @@ -0,0 +1,36 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT a.ca_state AS state, + tpcds_count((priv_hash(hash(s.ss_customer_sk))), 1) AS cnt, + tpcds_count((priv_hash(hash(s.ss_customer_sk))), 1) AS presence + FROM main.customer_address a , + main.customer c , + main.store_sales s , + main.date_dim d , + main.item i, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE s.ss_customer_sk = asa_owner0.asa_key0 + AND a.ca_address_sk = c.c_current_addr_sk + AND c.c_customer_sk = s.ss_customer_sk + AND s.ss_sold_date_sk = d.d_date_sk + AND s.ss_item_sk = i.i_item_sk + AND d.d_month_seq = + (SELECT DISTINCT (d_month_seq) + FROM main.date_dim + WHERE d_year = 2001 + AND d_moy = 1 ) + AND i.i_current_price > 1.2 * + (SELECT avg(j.i_current_price) + FROM main.item j + WHERE j.i_category = i.i_category) +GROUP BY a.ca_state +), answers AS ( + SELECT world, state, + cnt[world+1] AS cnt + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +WHERE cnt >= 10 +QUALIFY row_number() OVER (PARTITION BY world ORDER BY cnt NULLS FIRST, state NULLS FIRST) <= 100 +ORDER BY world, cnt NULLS FIRST, state NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q07.sql b/benchmark/tpcds/reference_queries/q07.sql new file mode 100644 index 00000000..43f73f96 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q07.sql @@ -0,0 +1,38 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT i_item_id, + tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_quantity) AS agg1, + tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_list_price) AS agg2, + tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_coupon_amt) AS agg3, + tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_sales_price) AS agg4, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.store_sales, + main.customer_demographics, + main.date_dim, + main.item, + main.promotion, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND ss_sold_date_sk = d_date_sk + AND ss_item_sk = i_item_sk + AND ss_cdemo_sk = cd_demo_sk + AND ss_promo_sk = p_promo_sk + AND cd_gender = 'M' + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND (p_channel_email = 'N' + OR p_channel_event = 'N') + AND d_year = 2000 +GROUP BY i_item_id +), answers AS ( + SELECT world, i_item_id, + agg1[world+1] AS agg1, + agg2[world+1] AS agg2, + agg3[world+1] AS agg3, + agg4[world+1] AS agg4 + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id) <= 100 +ORDER BY world, i_item_id; diff --git a/benchmark/tpcds/reference_queries/q09.sql b/benchmark/tpcds/reference_queries/q09.sql new file mode 100644 index 00000000..3cb2df1b --- /dev/null +++ b/benchmark/tpcds/reference_queries/q09.sql @@ -0,0 +1,17 @@ +-- Each CASE selects between its world-specific aggregate results. +WITH stats AS (SELECT tpcds_count(CASE WHEN ss_quantity BETWEEN 1 AND 20 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS n1, +tpcds_avg(CASE WHEN ss_quantity BETWEEN 1 AND 20 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_ext_discount_amt) AS discount1, +tpcds_avg(CASE WHEN ss_quantity BETWEEN 1 AND 20 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_net_paid) AS paid1, +tpcds_count(CASE WHEN ss_quantity BETWEEN 21 AND 40 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS n2, +tpcds_avg(CASE WHEN ss_quantity BETWEEN 21 AND 40 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_ext_discount_amt) AS discount2, +tpcds_avg(CASE WHEN ss_quantity BETWEEN 21 AND 40 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_net_paid) AS paid2, +tpcds_count(CASE WHEN ss_quantity BETWEEN 41 AND 60 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS n3, +tpcds_avg(CASE WHEN ss_quantity BETWEEN 41 AND 60 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_ext_discount_amt) AS discount3, +tpcds_avg(CASE WHEN ss_quantity BETWEEN 41 AND 60 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_net_paid) AS paid3, +tpcds_count(CASE WHEN ss_quantity BETWEEN 61 AND 80 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS n4, +tpcds_avg(CASE WHEN ss_quantity BETWEEN 61 AND 80 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_ext_discount_amt) AS discount4, +tpcds_avg(CASE WHEN ss_quantity BETWEEN 61 AND 80 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_net_paid) AS paid4, +tpcds_count(CASE WHEN ss_quantity BETWEEN 81 AND 100 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS n5, +tpcds_avg(CASE WHEN ss_quantity BETWEEN 81 AND 100 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_ext_discount_amt) AS discount5, +tpcds_avg(CASE WHEN ss_quantity BETWEEN 81 AND 100 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_net_paid) AS paid5 FROM main.store_sales JOIN main.customer ON ss_customer_sk=c_customer_sk) +SELECT world, CASE WHEN n1[world+1]>74129 THEN discount1[world+1] ELSE paid1[world+1] END AS bucket1, CASE WHEN n2[world+1]>122840 THEN discount2[world+1] ELSE paid2[world+1] END AS bucket2, CASE WHEN n3[world+1]>56580 THEN discount3[world+1] ELSE paid3[world+1] END AS bucket3, CASE WHEN n4[world+1]>10097 THEN discount4[world+1] ELSE paid4[world+1] END AS bucket4, CASE WHEN n5[world+1]>165306 THEN discount5[world+1] ELSE paid5[world+1] END AS bucket5 FROM stats CROSS JOIN range(64) worlds(world) CROSS JOIN main.reason WHERE r_reason_sk=1 ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q10.sql b/benchmark/tpcds/reference_queries/q10.sql new file mode 100644 index 00000000..590bc867 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q10.sql @@ -0,0 +1,59 @@ +-- Same-owner EXISTS is static after entity inclusion; shipping-customer EXISTS needs the bill-owner mask. +WITH catalog_presence AS ( +SELECT cs_ship_customer_sk,bit_or(priv_hash(hash(cs_bill_customer_sk))) AS membership + FROM main.catalog_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_moy BETWEEN 1 AND 1+3 +GROUP BY cs_ship_customer_sk +), stats AS (SELECT cd_gender, +cd_marital_status, +cd_education_status, +cd_purchase_estimate, +cd_credit_rating, +cd_dep_count, +cd_dep_employed_count, +cd_dep_college_count, +tpcds_count((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_moy BETWEEN 1 AND 1+3) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),1) AS n +FROM main.customer c CROSS JOIN + main.customer_address ca CROSS JOIN + main.customer_demographics LEFT JOIN catalog_presence ON c.c_customer_sk=cs_ship_customer_sk +WHERE c.c_current_addr_sk = ca.ca_address_sk + AND ca_county IN ('Rush County', + 'Toole County', + 'Jefferson County', + 'Dona Ana County', + 'La Porte County') + AND cd_demo_sk = c.c_current_cdemo_sk + +AND EXISTS(SELECT * + FROM main.store_sales, + main.date_dim + WHERE c.c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_moy BETWEEN 1 AND 1+3) +GROUP BY cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count +), answers AS (SELECT world, cd_gender, cd_marital_status, cd_education_status, n[world+1] AS cnt1, cd_purchase_estimate, n[world+1] AS cnt2, cd_credit_rating, n[world+1] AS cnt3, cd_dep_count, n[world+1] AS cnt4, cd_dep_employed_count, n[world+1] AS cnt5, cd_dep_college_count, n[world+1] AS cnt6 FROM stats CROSS JOIN range(64) worlds(world) WHERE n[world+1]>0) +SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count)<=100 ORDER BY world,cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count; diff --git a/benchmark/tpcds/reference_queries/q11.sql b/benchmark/tpcds/reference_queries/q11.sql new file mode 100644 index 00000000..07e84dc3 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q11.sql @@ -0,0 +1,93 @@ +-- Shared AS states per channel/customer/year; joins align the world index. +WITH year_total_stats AS ( +SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + tpcds_sum(priv_hash(hash(c_customer_sk)), ss_ext_list_price-ss_ext_discount_amt) year_total, + 's' sale_type, + tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence +FROM main.customer, + main.store_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year + +UNION ALL + SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + tpcds_sum(priv_hash(hash(c_customer_sk)), ws_ext_list_price-ws_ext_discount_amt) year_total, + 'w' sale_type, + tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence +FROM main.customer, + main.web_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year +), year_total AS ( +SELECT world, customer_id, customer_first_name, customer_last_name, customer_preferred_cust_flag, customer_birth_country, customer_login, customer_email_address, dyear, year_total[world+1] AS year_total, sale_type +FROM year_total_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT t_s_firstyear.world AS world, t_s_secyear.customer_id, + t_s_secyear.customer_first_name, + t_s_secyear.customer_last_name, + t_s_secyear.customer_preferred_cust_flag +FROM year_total t_s_firstyear, + year_total t_s_secyear, + year_total t_w_firstyear, + year_total t_w_secyear +WHERE t_s_firstyear.world=t_s_secyear.world AND t_s_firstyear.world=t_w_firstyear.world AND t_s_firstyear.world=t_w_secyear.world AND t_s_secyear.customer_id = t_s_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_secyear.customer_id + AND t_s_firstyear.customer_id = t_w_firstyear.customer_id + AND t_s_firstyear.sale_type = 's' + AND t_w_firstyear.sale_type = 'w' + AND t_s_secyear.sale_type = 's' + AND t_w_secyear.sale_type = 'w' + AND t_s_firstyear.dyear = 2001 + AND t_s_secyear.dyear = 2001+1 + AND t_w_firstyear.dyear = 2001 + AND t_w_secyear.dyear = 2001+1 + AND t_s_firstyear.year_total > 0 + AND t_w_firstyear.year_total > 0 + AND CASE + WHEN t_w_firstyear.year_total > 0 THEN (t_w_secyear.year_total*1.0000) / t_w_firstyear.year_total + ELSE 0.0 + END > CASE + WHEN t_s_firstyear.year_total > 0 THEN (t_s_secyear.year_total*1.0000) / t_s_firstyear.year_total + ELSE 0.0 + END +QUALIFY row_number() OVER (PARTITION BY t_s_firstyear.world ORDER BY t_s_secyear.customer_id NULLS FIRST, + t_s_secyear.customer_first_name NULLS FIRST, + t_s_secyear.customer_last_name NULLS FIRST, + t_s_secyear.customer_preferred_cust_flag NULLS FIRST) <= 100 +ORDER BY t_s_firstyear.world, t_s_secyear.customer_id NULLS FIRST, + t_s_secyear.customer_first_name NULLS FIRST, + t_s_secyear.customer_last_name NULLS FIRST, + t_s_secyear.customer_preferred_cust_flag NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q13.sql b/benchmark/tpcds/reference_queries/q13.sql new file mode 100644 index 00000000..35325b68 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q13.sql @@ -0,0 +1,55 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_quantity) AS avg1, + tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS avg2, + tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_ext_wholesale_cost) AS avg3, + tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_wholesale_cost) AS total, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.store_sales , + main.store , + main.customer_demographics , + main.household_demographics , + main.customer_address , + main.date_dim, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND s_store_sk = ss_store_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2001 and((ss_hdemo_sk=hd_demo_sk + AND cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'M' + AND cd_education_status = 'Advanced Degree' + AND ss_sales_price BETWEEN 100.00 AND 150.00 + AND hd_dep_count = 3) + OR (ss_hdemo_sk=hd_demo_sk + AND cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND ss_sales_price BETWEEN 50.00 AND 100.00 + AND hd_dep_count = 1 ) + OR (ss_hdemo_sk=hd_demo_sk + AND cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'W' + AND cd_education_status = '2 yr Degree' + AND ss_sales_price BETWEEN 150.00 AND 200.00 + AND hd_dep_count = 1)) and((ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('TX', 'OH', 'TX') + AND ss_net_profit BETWEEN 100 AND 200) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('OR', 'NM', 'KY') + AND ss_net_profit BETWEEN 150 AND 300) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('VA', 'TX', 'MS') + AND ss_net_profit BETWEEN 50 AND 250)) +), answers AS ( + SELECT world, avg1[world+1] AS avg1, + avg2[world+1] AS avg2, + avg3[world+1] AS avg3, + total[world+1] AS total + FROM stats CROSS JOIN range(64) worlds(world) +) +SELECT * FROM answers +ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q15.sql b/benchmark/tpcds/reference_queries/q15.sql new file mode 100644 index 00000000..07aa9526 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q15.sql @@ -0,0 +1,39 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT ca_zip, + tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), cs_sales_price) AS total, + tpcds_count((priv_hash(hash(cs_bill_customer_sk))), 1) AS presence + FROM main.catalog_sales, + main.customer, + main.customer_address, + main.date_dim, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE cs_bill_customer_sk = asa_owner0.asa_key0 + AND cs_bill_customer_sk = c_customer_sk + AND c_current_addr_sk = ca_address_sk + AND (SUBSTRING(ca_zip, 1, 5) IN ('85669', + '86197', + '88274', + '83405', + '86475', + '85392', + '85460', + '80348', + '81792') + OR ca_state IN ('CA', + 'WA', + 'GA') + OR cs_sales_price > 500) + AND cs_sold_date_sk = d_date_sk + AND d_qoy = 2 + AND d_year = 2001 +GROUP BY ca_zip +), answers AS ( + SELECT world, ca_zip, + total[world+1] AS total + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY ca_zip NULLS FIRST) <= 100 +ORDER BY world, ca_zip NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q16.sql b/benchmark/tpcds/reference_queries/q16.sql new file mode 100644 index 00000000..9d291fd6 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q16.sql @@ -0,0 +1,10 @@ +-- Order-level existence masks are combined before AS; DISTINCT unions memberships by order. +WITH warehouses AS (SELECT cs_order_number AS order_number,cs_warehouse_sk AS warehouse,bit_or(priv_hash(hash(cs_bill_customer_sk))) AS membership FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk GROUP BY cs_order_number,cs_warehouse_sk), +other_warehouses AS (SELECT a.order_number,a.warehouse,bit_or(b.membership) AS membership FROM warehouses a JOIN warehouses b ON a.order_number=b.order_number AND a.warehouse<>b.warehouse GROUP BY a.order_number,a.warehouse), +return_orders AS (SELECT cr_order_number AS order_number,bit_or(priv_hash(hash(cr_returning_customer_sk))) AS membership FROM main.catalog_returns JOIN main.customer ON cr_returning_customer_sk=c_customer_sk GROUP BY cr_order_number), +input AS (SELECT cs_order_number AS order_number,cs_ext_ship_cost AS shipping,cs_net_profit AS profit,(priv_hash(hash(cs_bill_customer_sk)) & other.membership & ~coalesce(ret.membership,0::UBIGINT)) AS membership +FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk JOIN main.date_dim ON cs_ship_date_sk=d_date_sk JOIN main.customer_address ON cs_ship_addr_sk=ca_address_sk JOIN main.call_center ON cs_call_center_sk=cc_call_center_sk JOIN other_warehouses other ON cs_order_number=other.order_number AND cs_warehouse_sk=other.warehouse LEFT JOIN return_orders ret ON cs_order_number=ret.order_number WHERE d_date BETWEEN DATE '2002-02-01' AND DATE '2002-04-02' AND ca_state='GA' AND cc_county='Williamson County'), +totals AS (SELECT tpcds_sum(membership,shipping) AS shipping,tpcds_sum(membership,profit) AS profit FROM input), +unique_orders AS (SELECT order_number,bit_or(membership) AS membership FROM input GROUP BY order_number), +counts AS (SELECT tpcds_count(membership,1) AS n FROM unique_orders) +SELECT world,n[world+1] AS order_count,shipping[world+1] AS total_shipping_cost,profit[world+1] AS total_net_profit FROM totals CROSS JOIN counts CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q17.sql b/benchmark/tpcds/reference_queries/q17.sql new file mode 100644 index 00000000..6910c700 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q17.sql @@ -0,0 +1,67 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT i_item_id, + i_item_desc, + s_state, + tpcds_count((priv_hash(hash(ss_customer_sk))), ss_quantity) AS v0_count, + tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_quantity) AS v0_avg, + tpcds_stddev_samp((priv_hash(hash(ss_customer_sk))), ss_quantity) AS v0_stddev_samp, + tpcds_count((priv_hash(hash(ss_customer_sk))), sr_return_quantity) AS v1_count, + tpcds_avg((priv_hash(hash(ss_customer_sk))), sr_return_quantity) AS v1_avg, + tpcds_stddev_samp((priv_hash(hash(ss_customer_sk))), sr_return_quantity) AS v1_stddev_samp, + tpcds_count((priv_hash(hash(ss_customer_sk))), cs_quantity) AS v2_count, + tpcds_avg((priv_hash(hash(ss_customer_sk))), cs_quantity) AS v2_avg, + tpcds_stddev_samp((priv_hash(hash(ss_customer_sk))), cs_quantity) AS v2_stddev_samp, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.store_sales, + main.store_returns, + main.catalog_sales, + main.date_dim d1, + main.date_dim d2, + main.date_dim d3, + main.store, + main.item, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND d1.d_quarter_name = '2001Q1' + AND d1.d_date_sk = ss_sold_date_sk + AND i_item_sk = ss_item_sk + AND s_store_sk = ss_store_sk + AND ss_customer_sk = sr_customer_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND sr_returned_date_sk = d2.d_date_sk + AND d2.d_quarter_name IN ('2001Q1', + '2001Q2', + '2001Q3') + AND sr_customer_sk = cs_bill_customer_sk + AND sr_item_sk = cs_item_sk + AND cs_sold_date_sk = d3.d_date_sk + AND d3.d_quarter_name IN ('2001Q1', + '2001Q2', + '2001Q3') +GROUP BY i_item_id, + i_item_desc, + s_state +), answers AS ( + SELECT world, i_item_id, + i_item_desc, + s_state, + v0_count[world+1] AS store_sales_quantitycount, + v0_avg[world+1] AS store_sales_quantityave, + v0_stddev_samp[world+1] AS store_sales_quantitystdev, + v0_stddev_samp[world+1]/v0_avg[world+1] AS store_sales_quantitycov, + v1_count[world+1] AS store_returns_quantitycount, + v1_avg[world+1] AS store_returns_quantityave, + v1_stddev_samp[world+1] AS store_returns_quantitystdev, + v1_stddev_samp[world+1]/v1_avg[world+1] AS store_returns_quantitycov, + v2_count[world+1] AS catalog_sales_quantitycount, + v2_avg[world+1] AS catalog_sales_quantityave, + v2_stddev_samp[world+1] AS catalog_sales_quantitystdev, + v2_stddev_samp[world+1]/v2_avg[world+1] AS catalog_sales_quantitycov + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id NULLS FIRST, i_item_desc NULLS FIRST, s_state NULLS FIRST) <= 100 +ORDER BY world, i_item_id NULLS FIRST, i_item_desc NULLS FIRST, s_state NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q18.sql b/benchmark/tpcds/reference_queries/q18.sql new file mode 100644 index 00000000..d00e2c97 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q18.sql @@ -0,0 +1,68 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT i_item_id, + ca_country, + ca_state, + ca_county, + tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cs_quantity AS DECIMAL(12,2))) AS agg1, + tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cs_list_price AS DECIMAL(12,2))) AS agg2, + tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cs_coupon_amt AS DECIMAL(12,2))) AS agg3, + tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cs_sales_price AS DECIMAL(12,2))) AS agg4, + tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cs_net_profit AS DECIMAL(12,2))) AS agg5, + tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(c_birth_year AS DECIMAL(12,2))) AS agg6, + tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cd1.cd_dep_count AS DECIMAL(12,2))) AS agg7, + tpcds_count((priv_hash(hash(cs_bill_customer_sk))), 1) AS presence, + GROUPING(i_item_id, ca_country, ca_state, ca_county) AS grouping_level + FROM main.catalog_sales, + main.customer_demographics cd1, + main.customer_demographics cd2, + main.customer, + main.customer_address, + main.date_dim, + main.item, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE cs_bill_customer_sk = asa_owner0.asa_key0 + AND cs_sold_date_sk = d_date_sk + AND cs_item_sk = i_item_sk + AND cs_bill_cdemo_sk = cd1.cd_demo_sk + AND cs_bill_customer_sk = c_customer_sk + AND cd1.cd_gender = 'F' + AND cd1.cd_education_status = 'Unknown' + AND c_current_cdemo_sk = cd2.cd_demo_sk + AND c_current_addr_sk = ca_address_sk + AND c_birth_month IN (1, + 6, + 8, + 9, + 12, + 2) + AND d_year = 1998 + AND ca_state IN ('MS', + 'IN', + 'ND', + 'OK', + 'NM', + 'VA', + 'MS') +GROUP BY ROLLUP (i_item_id, + ca_country, + ca_state, + ca_county) +), answers AS ( + SELECT world, i_item_id, + ca_country, + ca_state, + ca_county, + agg1[world+1] AS agg1, + agg2[world+1] AS agg2, + agg3[world+1] AS agg3, + agg4[world+1] AS agg4, + agg5[world+1] AS agg5, + agg6[world+1] AS agg6, + agg7[world+1] AS agg7 + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 OR grouping_level = 15 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY ca_country NULLS FIRST, ca_state NULLS FIRST, ca_county NULLS FIRST, i_item_id NULLS FIRST) <= 100 +ORDER BY world, ca_country NULLS FIRST, ca_state NULLS FIRST, ca_county NULLS FIRST, i_item_id NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q19.sql b/benchmark/tpcds/reference_queries/q19.sql new file mode 100644 index 00000000..beecad17 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q19.sql @@ -0,0 +1,41 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT i_brand_id AS brand_id, + i_brand AS brand, + i_manufact_id, + i_manufact, + tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS ext_price, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.date_dim, + main.store_sales, + main.item, + main.customer, + main.customer_address, + main.store, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND d_date_sk = ss_sold_date_sk + AND ss_item_sk = i_item_sk + AND i_manager_id=8 + AND d_moy=11 + AND d_year=1998 + AND ss_customer_sk = c_customer_sk + AND c_current_addr_sk = ca_address_sk + AND SUBSTRING(ca_zip, 1, 5) <> SUBSTRING(s_zip, 1, 5) + AND ss_store_sk = s_store_sk +GROUP BY i_brand, + i_brand_id, + i_manufact_id, + i_manufact +), answers AS ( + SELECT world, brand_id, + brand, + i_manufact_id, + i_manufact, + ext_price[world+1] AS ext_price + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY ext_price DESC, brand, brand_id, i_manufact_id, i_manufact) <= 100 +ORDER BY world, ext_price DESC, brand, brand_id, i_manufact_id, i_manufact; diff --git a/benchmark/tpcds/reference_queries/q23.sql b/benchmark/tpcds/reference_queries/q23.sql new file mode 100644 index 00000000..c863cb8f --- /dev/null +++ b/benchmark/tpcds/reference_queries/q23.sql @@ -0,0 +1,96 @@ +-- Item-date multiplicities are preserved; global maxima and customer cutoffs remain world-specific. +WITH frequent_stats AS ( +SELECT itemdesc, + i_item_sk item_sk, + d_date solddate, + tpcds_count(priv_hash(hash(ss_customer_sk)),1) cnt, tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence +FROM main.store_sales, + main.date_dim, + (SELECT SUBSTRING(i_item_desc, 1, 30) itemdesc, + * + FROM main.item) sq1, (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk=asa_owner0.asa_key0 AND ss_sold_date_sk = d_date_sk + AND ss_item_sk = i_item_sk + AND d_year IN (2000, + 2000+1, + 2000+2, + 2000+3) + GROUP BY itemdesc, + i_item_sk, + d_date + +), +frequent_ss_items AS (SELECT itemdesc,item_sk,solddate,priv_select(18446744073709551615::UBIGINT,list_transform(cnt,lambda x: x>4)) AS membership FROM frequent_stats), +customer_period_stats AS ( +SELECT c_customer_sk, + tpcds_sum(priv_hash(hash(c_customer_sk)), ss_quantity*ss_sales_price) csales, + tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence +FROM main.store_sales, + main.customer, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE c_customer_sk=asa_owner.asa_key AND ss_customer_sk = c_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year IN (2000, + 2000+1, + 2000+2, + 2000+3) + GROUP BY c_customer_sk +), +customer_period AS ( +SELECT world, c_customer_sk, csales[world+1] AS csales +FROM customer_period_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +max_sales AS (SELECT world,max(csales) AS maximum FROM customer_period GROUP BY world), +max_array AS (SELECT list(maximum ORDER BY w.world) AS thresholds FROM range(64) w(world) LEFT JOIN max_sales USING(world)), +best_stats AS (SELECT c_customer_sk,tpcds_sum(priv_hash(hash(c_customer_sk)),ss_quantity*ss_sales_price) AS ssales FROM main.store_sales JOIN main.customer ON ss_customer_sk=c_customer_sk GROUP BY c_customer_sk), +best_ss_customer AS (SELECT c_customer_sk,priv_select(priv_hash(hash(c_customer_sk)),list_transform(list_zip(ssales,thresholds),lambda x: x[1]>0.5*x[2])) AS membership FROM best_stats CROSS JOIN max_array), +channel0_stats AS ( +SELECT c_last_name, + c_first_name, + tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & frequent_ss_items.membership & best_ss_customer.membership), cs_quantity*cs_list_price) sales, tpcds_count((priv_hash(hash(cs_bill_customer_sk)) & frequent_ss_items.membership & best_ss_customer.membership),1) AS presence +FROM main.catalog_sales, + main.customer, + main.date_dim, + frequent_ss_items, + best_ss_customer +WHERE d_year = 2000 + AND d_moy = 2 + AND cs_sold_date_sk = d_date_sk + AND cs_item_sk = item_sk + AND cs_bill_customer_sk = best_ss_customer.c_customer_sk + AND cs_bill_customer_sk = customer.c_customer_sk + GROUP BY c_last_name, + c_first_name + +), +channel0 AS ( +SELECT world, c_last_name, c_first_name, sales[world+1] AS sales +FROM channel0_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +channel1_stats AS ( + SELECT c_last_name, + c_first_name, + tpcds_sum((priv_hash(hash(ws_bill_customer_sk)) & frequent_ss_items.membership & best_ss_customer.membership), ws_quantity*ws_list_price) sales, tpcds_count((priv_hash(hash(ws_bill_customer_sk)) & frequent_ss_items.membership & best_ss_customer.membership),1) AS presence +FROM main.web_sales, + main.customer, + main.date_dim, + frequent_ss_items, + best_ss_customer +WHERE d_year = 2000 + AND d_moy = 2 + AND ws_sold_date_sk = d_date_sk + AND ws_item_sk = item_sk + AND ws_bill_customer_sk = best_ss_customer.c_customer_sk + AND ws_bill_customer_sk = customer.c_customer_sk + GROUP BY c_last_name, + c_first_name +), +channel1 AS ( +SELECT world, c_last_name, c_first_name, sales[world+1] AS sales +FROM channel1_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +answers AS (SELECT * FROM channel0 UNION ALL SELECT * FROM channel1) +SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY c_last_name NULLS FIRST,c_first_name NULLS FIRST,sales NULLS FIRST)<=100 ORDER BY world,c_last_name NULLS FIRST,c_first_name NULLS FIRST,sales NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q24.sql b/benchmark/tpcds/reference_queries/q24.sql new file mode 100644 index 00000000..d58fe980 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q24.sql @@ -0,0 +1,58 @@ +-- Sale and return ownership masks are intersected before the shared inner aggregation. +WITH ssales_stats AS ( +SELECT c_last_name, + c_first_name, + s_store_name, + ca_state, + s_state, + i_color, + i_current_price, + i_manager_id, + i_units, + i_size, + tpcds_sum((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk))), ss_net_paid) netpaid, tpcds_count((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk))),1) AS presence +FROM main.store_sales, + main.store_returns, + main.store, + main.item, + main.customer, + main.customer_address, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_return_owner +WHERE sr_customer_sk=asa_return_owner.asa_key AND ss_ticket_number = sr_ticket_number + AND ss_item_sk = sr_item_sk + AND ss_customer_sk = c_customer_sk + AND ss_item_sk = i_item_sk + AND ss_store_sk = s_store_sk + AND c_current_addr_sk = ca_address_sk + AND c_birth_country <> upper(ca_country) + AND s_zip = ca_zip + AND s_market_id=8 + GROUP BY c_last_name, + c_first_name, + s_store_name, + ca_state, + s_state, + i_color, + i_current_price, + i_manager_id, + i_units, + i_size +), ssales AS ( +SELECT world, c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size, netpaid[world+1] AS netpaid +FROM ssales_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT outer_ss.world AS world, c_last_name, + c_first_name, + s_store_name, + sum(netpaid) paid +FROM ssales outer_ss +WHERE i_color = 'peach' +GROUP BY outer_ss.world, c_last_name, + c_first_name, + s_store_name +HAVING sum(netpaid) > + (SELECT 0.05*avg(netpaid) + FROM ssales inner_ss WHERE inner_ss.world=outer_ss.world) +ORDER BY outer_ss.world, c_last_name, + c_first_name, + s_store_name; diff --git a/benchmark/tpcds/reference_queries/q25.sql b/benchmark/tpcds/reference_queries/q25.sql new file mode 100644 index 00000000..963bf844 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q25.sql @@ -0,0 +1,54 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT i_item_id, + i_item_desc, + s_store_id, + s_store_name, + tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_net_profit) AS store_sales_profit, + tpcds_sum((priv_hash(hash(ss_customer_sk))), sr_net_loss) AS store_returns_loss, + tpcds_sum((priv_hash(hash(ss_customer_sk))), cs_net_profit) AS catalog_sales_profit, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.store_sales , + main.store_returns , + main.catalog_sales , + main.date_dim d1 , + main.date_dim d2 , + main.date_dim d3 , + main.store , + main.item, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND d1.d_moy = 4 + AND d1.d_year = 2001 + AND d1.d_date_sk = ss_sold_date_sk + AND i_item_sk = ss_item_sk + AND s_store_sk = ss_store_sk + AND ss_customer_sk = sr_customer_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND sr_returned_date_sk = d2.d_date_sk + AND d2.d_moy BETWEEN 4 AND 10 + AND d2.d_year = 2001 + AND sr_customer_sk = cs_bill_customer_sk + AND sr_item_sk = cs_item_sk + AND cs_sold_date_sk = d3.d_date_sk + AND d3.d_moy BETWEEN 4 AND 10 + AND d3.d_year = 2001 +GROUP BY i_item_id , + i_item_desc , + s_store_id , + s_store_name +), answers AS ( + SELECT world, i_item_id, + i_item_desc, + s_store_id, + s_store_name, + store_sales_profit[world+1] AS store_sales_profit, + store_returns_loss[world+1] AS store_returns_loss, + catalog_sales_profit[world+1] AS catalog_sales_profit + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id, i_item_desc, s_store_id, s_store_name) <= 100 +ORDER BY world, i_item_id, i_item_desc, s_store_id, s_store_name; diff --git a/benchmark/tpcds/reference_queries/q26.sql b/benchmark/tpcds/reference_queries/q26.sql new file mode 100644 index 00000000..5c8055f1 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q26.sql @@ -0,0 +1,38 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT i_item_id, + tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), cs_quantity) AS agg1, + tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), cs_list_price) AS agg2, + tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), cs_coupon_amt) AS agg3, + tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), cs_sales_price) AS agg4, + tpcds_count((priv_hash(hash(cs_bill_customer_sk))), 1) AS presence + FROM main.catalog_sales, + main.customer_demographics, + main.date_dim, + main.item, + main.promotion, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE cs_bill_customer_sk = asa_owner0.asa_key0 + AND cs_sold_date_sk = d_date_sk + AND cs_item_sk = i_item_sk + AND cs_bill_cdemo_sk = cd_demo_sk + AND cs_promo_sk = p_promo_sk + AND cd_gender = 'M' + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND (p_channel_email = 'N' + OR p_channel_event = 'N') + AND d_year = 2000 +GROUP BY i_item_id +), answers AS ( + SELECT world, i_item_id, + agg1[world+1] AS agg1, + agg2[world+1] AS agg2, + agg3[world+1] AS agg3, + agg4[world+1] AS agg4 + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id) <= 100 +ORDER BY world, i_item_id; diff --git a/benchmark/tpcds/reference_queries/q27.sql b/benchmark/tpcds/reference_queries/q27.sql new file mode 100644 index 00000000..45c51b9a --- /dev/null +++ b/benchmark/tpcds/reference_queries/q27.sql @@ -0,0 +1,25 @@ +-- Explicit grouping sets reproduce all three original UNION ALL branches, including empty grand totals. +WITH input AS ( +SELECT priv_hash(hash(ss_customer_sk)) AS membership, i_item_id, + s_state, + 0 AS g_state, + ss_quantity agg1, + ss_list_price agg2, + ss_coupon_amt agg3, + ss_sales_price agg4 + FROM main.store_sales, + main.customer_demographics, + main.date_dim, + main.store, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk = d_date_sk + AND ss_item_sk = i_item_sk + AND ss_store_sk = s_store_sk + AND ss_cdemo_sk = cd_demo_sk + AND cd_gender = 'M' + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND d_year = 2002 + AND s_state = 'TN' +), stats AS (SELECT i_item_id, s_state, grouping(s_state) AS g_state, grouping(i_item_id) AS grand_total, tpcds_avg(membership,agg1) AS agg1, tpcds_avg(membership,agg2) AS agg2, tpcds_avg(membership,agg3) AS agg3, tpcds_avg(membership,agg4) AS agg4, tpcds_count(membership,1) AS presence FROM input GROUP BY GROUPING SETS ((i_item_id,s_state),(i_item_id),())), answers AS (SELECT world, i_item_id, s_state, g_state, agg1[world+1] AS agg1, agg2[world+1] AS agg2, agg3[world+1] AS agg3, agg4[world+1] AS agg4 FROM stats CROSS JOIN range(64) worlds(world) WHERE presence[world+1]>0 OR grand_total=1) +SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id NULLS FIRST,s_state NULLS FIRST)<=100 ORDER BY world,i_item_id NULLS FIRST,s_state NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q28.sql b/benchmark/tpcds/reference_queries/q28.sql new file mode 100644 index 00000000..6c7dd994 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q28.sql @@ -0,0 +1,9 @@ +-- Per-value membership union implements COUNT(DISTINCT) without duplicating base rows by world. +WITH input AS (SELECT CASE WHEN (ss_quantity BETWEEN 0 AND 5 AND (ss_list_price BETWEEN 8 AND 18 OR ss_coupon_amt BETWEEN 459 AND 1459 OR ss_wholesale_cost BETWEEN 57 AND 77)) THEN 1 WHEN (ss_quantity BETWEEN 6 AND 10 AND (ss_list_price BETWEEN 90 AND 100 OR ss_coupon_amt BETWEEN 2323 AND 3323 OR ss_wholesale_cost BETWEEN 31 AND 51)) THEN 2 WHEN (ss_quantity BETWEEN 11 AND 15 AND (ss_list_price BETWEEN 142 AND 152 OR ss_coupon_amt BETWEEN 12214 AND 13214 OR ss_wholesale_cost BETWEEN 79 AND 99)) THEN 3 WHEN (ss_quantity BETWEEN 16 AND 20 AND (ss_list_price BETWEEN 135 AND 145 OR ss_coupon_amt BETWEEN 6071 AND 7071 OR ss_wholesale_cost BETWEEN 38 AND 58)) THEN 4 WHEN (ss_quantity BETWEEN 21 AND 25 AND (ss_list_price BETWEEN 122 AND 132 OR ss_coupon_amt BETWEEN 836 AND 1836 OR ss_wholesale_cost BETWEEN 17 AND 37)) THEN 5 WHEN (ss_quantity BETWEEN 26 AND 30 AND (ss_list_price BETWEEN 154 AND 164 OR ss_coupon_amt BETWEEN 7326 AND 8326 OR ss_wholesale_cost BETWEEN 7 AND 27)) THEN 6 END AS bucket, ss_list_price AS price, priv_hash(hash(ss_customer_sk)) AS membership +FROM main.store_sales JOIN main.customer ON ss_customer_sk=c_customer_sk WHERE (ss_quantity BETWEEN 0 AND 5 AND (ss_list_price BETWEEN 8 AND 18 OR ss_coupon_amt BETWEEN 459 AND 1459 OR ss_wholesale_cost BETWEEN 57 AND 77)) OR (ss_quantity BETWEEN 6 AND 10 AND (ss_list_price BETWEEN 90 AND 100 OR ss_coupon_amt BETWEEN 2323 AND 3323 OR ss_wholesale_cost BETWEEN 31 AND 51)) OR (ss_quantity BETWEEN 11 AND 15 AND (ss_list_price BETWEEN 142 AND 152 OR ss_coupon_amt BETWEEN 12214 AND 13214 OR ss_wholesale_cost BETWEEN 79 AND 99)) OR (ss_quantity BETWEEN 16 AND 20 AND (ss_list_price BETWEEN 135 AND 145 OR ss_coupon_amt BETWEEN 6071 AND 7071 OR ss_wholesale_cost BETWEEN 38 AND 58)) OR (ss_quantity BETWEEN 21 AND 25 AND (ss_list_price BETWEEN 122 AND 132 OR ss_coupon_amt BETWEEN 836 AND 1836 OR ss_wholesale_cost BETWEEN 17 AND 37)) OR (ss_quantity BETWEEN 26 AND 30 AND (ss_list_price BETWEEN 154 AND 164 OR ss_coupon_amt BETWEEN 7326 AND 8326 OR ss_wholesale_cost BETWEEN 7 AND 27))), +stats AS (SELECT bucket,tpcds_avg(membership,price) AS average,tpcds_count(membership,price) AS n FROM input GROUP BY bucket), +unique_prices AS (SELECT bucket,price,bit_or(membership) AS membership FROM input WHERE price IS NOT NULL GROUP BY bucket,price), +unique_counts AS (SELECT bucket,tpcds_count(membership,1) AS n FROM unique_prices GROUP BY bucket), +expanded AS (SELECT world,bucket,average[world+1] AS average,coalesce(stats.n[world+1],0) AS n,coalesce(unique_counts.n[world+1],0) AS nd +FROM (VALUES(1),(2),(3),(4),(5),(6)) b(bucket) CROSS JOIN range(64) worlds(world) LEFT JOIN stats USING(bucket) LEFT JOIN unique_counts USING(bucket)) +SELECT world,max(average) FILTER (WHERE bucket=1) AS b1_lp, max(n) FILTER (WHERE bucket=1) AS b1_cnt, max(nd) FILTER (WHERE bucket=1) AS b1_cntd, max(average) FILTER (WHERE bucket=2) AS b2_lp, max(n) FILTER (WHERE bucket=2) AS b2_cnt, max(nd) FILTER (WHERE bucket=2) AS b2_cntd, max(average) FILTER (WHERE bucket=3) AS b3_lp, max(n) FILTER (WHERE bucket=3) AS b3_cnt, max(nd) FILTER (WHERE bucket=3) AS b3_cntd, max(average) FILTER (WHERE bucket=4) AS b4_lp, max(n) FILTER (WHERE bucket=4) AS b4_cnt, max(nd) FILTER (WHERE bucket=4) AS b4_cntd, max(average) FILTER (WHERE bucket=5) AS b5_lp, max(n) FILTER (WHERE bucket=5) AS b5_cnt, max(nd) FILTER (WHERE bucket=5) AS b5_cntd, max(average) FILTER (WHERE bucket=6) AS b6_lp, max(n) FILTER (WHERE bucket=6) AS b6_cnt, max(nd) FILTER (WHERE bucket=6) AS b6_cntd FROM expanded GROUP BY world ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q29.sql b/benchmark/tpcds/reference_queries/q29.sql new file mode 100644 index 00000000..17f70177 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q29.sql @@ -0,0 +1,55 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT i_item_id, + i_item_desc, + s_store_id, + s_store_name, + tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_quantity) AS store_sales_quantity, + tpcds_sum((priv_hash(hash(ss_customer_sk))), sr_return_quantity) AS store_returns_quantity, + tpcds_sum((priv_hash(hash(ss_customer_sk))), cs_quantity) AS catalog_sales_quantity, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.store_sales, + main.store_returns, + main.catalog_sales, + main.date_dim d1, + main.date_dim d2, + main.date_dim d3, + main.store, + main.item, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND d1.d_moy = 9 + AND d1.d_year = 1999 + AND d1.d_date_sk = ss_sold_date_sk + AND i_item_sk = ss_item_sk + AND s_store_sk = ss_store_sk + AND ss_customer_sk = sr_customer_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND sr_returned_date_sk = d2.d_date_sk + AND d2.d_moy BETWEEN 9 AND 9 + 3 + AND d2.d_year = 1999 + AND sr_customer_sk = cs_bill_customer_sk + AND sr_item_sk = cs_item_sk + AND cs_sold_date_sk = d3.d_date_sk + AND d3.d_year IN (1999, + 1999+1, + 1999+2) +GROUP BY i_item_id, + i_item_desc, + s_store_id, + s_store_name +), answers AS ( + SELECT world, i_item_id, + i_item_desc, + s_store_id, + s_store_name, + store_sales_quantity[world+1] AS store_sales_quantity, + store_returns_quantity[world+1] AS store_returns_quantity, + catalog_sales_quantity[world+1] AS catalog_sales_quantity + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id, i_item_desc, s_store_id, s_store_name) <= 100 +ORDER BY world, i_item_id, i_item_desc, s_store_id, s_store_name; diff --git a/benchmark/tpcds/reference_queries/q30.sql b/benchmark/tpcds/reference_queries/q30.sql new file mode 100644 index 00000000..625a01f6 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q30.sql @@ -0,0 +1,68 @@ +-- Correlated thresholds use only the matching world of the reduced return aggregates. +WITH return_stats AS ( +SELECT wr_returning_customer_sk AS ctr_customer_sk, + ca_state AS ctr_state, + tpcds_sum(priv_hash(hash(wr_returning_customer_sk)), wr_return_amt) AS ctr_total_return, + tpcds_count(priv_hash(hash(wr_returning_customer_sk)),1) AS presence +FROM main.web_returns, + main.date_dim, + main.customer_address, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE wr_returning_customer_sk=asa_owner.asa_key AND wr_returned_date_sk = d_date_sk + AND d_year = 2002 + AND wr_returning_addr_sk = ca_address_sk + GROUP BY wr_returning_customer_sk, + ca_state +), customer_total_return AS ( +SELECT world, ctr_customer_sk, ctr_state, ctr_total_return[world+1] AS ctr_total_return +FROM return_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT ctr1.world AS world, c_customer_id, + c_salutation, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_day, + c_birth_month, + c_birth_year, + c_birth_country, + c_login, + c_email_address, + c_last_review_date_sk, + ctr_total_return +FROM customer_total_return ctr1, + main.customer_address, + main.customer +WHERE ctr1.ctr_total_return > + (SELECT avg(ctr_total_return)*1.2 + FROM customer_total_return ctr2 + WHERE ctr1.world=ctr2.world AND ctr1.ctr_state = ctr2.ctr_state) + AND ca_address_sk = c_current_addr_sk + AND ca_state = 'GA' + AND ctr1.ctr_customer_sk = c_customer_sk +QUALIFY row_number() OVER (PARTITION BY ctr1.world ORDER BY c_customer_id NULLS FIRST, + c_salutation NULLS FIRST, + c_first_name NULLS FIRST, + c_last_name NULLS FIRST, + c_preferred_cust_flag NULLS FIRST, + c_birth_day NULLS FIRST, + c_birth_month NULLS FIRST, + c_birth_year NULLS FIRST, + c_birth_country NULLS FIRST, + c_login NULLS FIRST, + c_email_address NULLS FIRST, + c_last_review_date_sk NULLS FIRST, + ctr_total_return NULLS FIRST) <= 100 +ORDER BY ctr1.world, c_customer_id NULLS FIRST, + c_salutation NULLS FIRST, + c_first_name NULLS FIRST, + c_last_name NULLS FIRST, + c_preferred_cust_flag NULLS FIRST, + c_birth_day NULLS FIRST, + c_birth_month NULLS FIRST, + c_birth_year NULLS FIRST, + c_birth_country NULLS FIRST, + c_login NULLS FIRST, + c_email_address NULLS FIRST, + c_last_review_date_sk NULLS FIRST, + ctr_total_return NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q31.sql b/benchmark/tpcds/reference_queries/q31.sql new file mode 100644 index 00000000..cdaf8c82 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q31.sql @@ -0,0 +1,85 @@ +-- Fact scans feed AS once; subsequent processing is over reduced aggregate results. +WITH ss_stats AS ( +SELECT ca_county, + d_qoy, + d_year, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS store_sales, + tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence +FROM main.store_sales, + main.date_dim, + main.customer_address, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk = d_date_sk + AND ss_addr_sk=ca_address_sk + GROUP BY ca_county, + d_qoy, + d_year +), +ss AS ( +SELECT world, ca_county, d_qoy, d_year, store_sales[world+1] AS store_sales +FROM ss_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +ws_stats AS ( +SELECT ca_county, + d_qoy, + d_year, + tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) AS web_sales, + tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence +FROM main.web_sales, + main.date_dim, + main.customer_address, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_sold_date_sk = d_date_sk + AND ws_bill_addr_sk=ca_address_sk + GROUP BY ca_county, + d_qoy, + d_year +), +ws AS ( +SELECT world, ca_county, d_qoy, d_year, web_sales[world+1] AS web_sales +FROM ws_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT ss1.world AS world, ss1.ca_county , + ss1.d_year , + (ws2.web_sales*1.0000)/ws1.web_sales web_q1_q2_increase , + (ss2.store_sales*1.0000)/ss1.store_sales store_q1_q2_increase , + (ws3.web_sales*1.0000)/ws2.web_sales web_q2_q3_increase , + (ss3.store_sales*1.0000)/ss2.store_sales store_q2_q3_increase +FROM ss ss1 , + ss ss2 , + ss ss3 , + ws ws1 , + ws ws2 , + ws ws3 +WHERE ss1.world=ss2.world AND ss1.world=ss3.world AND ss1.world=ws1.world AND ss1.world=ws2.world AND ss1.world=ws3.world AND ss1.d_qoy = 1 + AND ss1.d_year = 2000 + AND ss1.ca_county = ss2.ca_county + AND ss2.d_qoy = 2 + AND ss2.d_year = 2000 + AND ss2.ca_county = ss3.ca_county + AND ss3.d_qoy = 3 + AND ss3.d_year = 2000 + AND ss1.ca_county = ws1.ca_county + AND ws1.d_qoy = 1 + AND ws1.d_year = 2000 + AND ws1.ca_county = ws2.ca_county + AND ws2.d_qoy = 2 + AND ws2.d_year = 2000 + AND ws1.ca_county = ws3.ca_county + AND ws3.d_qoy = 3 + AND ws3.d_year = 2000 + AND CASE + WHEN ws1.web_sales > 0 THEN (ws2.web_sales*1.0000)/ws1.web_sales + ELSE NULL + END > CASE + WHEN ss1.store_sales > 0 THEN (ss2.store_sales*1.0000)/ss1.store_sales + ELSE NULL + END + AND CASE + WHEN ws2.web_sales > 0 THEN (ws3.web_sales*1.0000)/ws2.web_sales + ELSE NULL + END > CASE + WHEN ss2.store_sales > 0 THEN (ss3.store_sales*1.0000)/ss2.store_sales + ELSE NULL + END +ORDER BY ss1.world, ss1.ca_county; diff --git a/benchmark/tpcds/reference_queries/q32.sql b/benchmark/tpcds/reference_queries/q32.sql new file mode 100644 index 00000000..46f64028 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q32.sql @@ -0,0 +1,14 @@ +-- Correlated aggregate predicates update the mask before the outer AS SUM. +WITH thresholds AS ( + SELECT cs_item_sk, tpcds_avg(priv_hash(hash(cs_bill_customer_sk)),cs_ext_discount_amt) AS averages + FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk JOIN main.date_dim ON cs_sold_date_sk=d_date_sk + WHERE d_date BETWEEN DATE '2000-01-27' AND DATE '2000-04-26' GROUP BY cs_item_sk +), filtered AS ( + SELECT s.cs_ext_discount_amt AS discount, + priv_select(priv_hash(hash(cs_bill_customer_sk)),list_transform(averages, lambda average: s.cs_ext_discount_amt>1.3*average)) AS membership + FROM main.catalog_sales s JOIN main.customer ON cs_bill_customer_sk=c_customer_sk + JOIN main.item ON s.cs_item_sk=i_item_sk JOIN main.date_dim ON s.cs_sold_date_sk=d_date_sk + JOIN thresholds t ON s.cs_item_sk=t.cs_item_sk + WHERE i_manufact_id=977 AND d_date BETWEEN DATE '2000-01-27' AND DATE '2000-04-26' +), stats AS (SELECT tpcds_sum(membership,discount) AS total FROM filtered) +SELECT world,total[world+1] AS excess_discount_amount FROM stats CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q33.sql b/benchmark/tpcds/reference_queries/q33.sql new file mode 100644 index 00000000..b260140b --- /dev/null +++ b/benchmark/tpcds/reference_queries/q33.sql @@ -0,0 +1,90 @@ +-- Fact scans feed AS once; subsequent processing is over reduced aggregate results. +WITH ss_stats AS ( +SELECT i_manufact_id, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales, + tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence +FROM main.store_sales, + main.date_dim, + main.customer_address, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND i_manufact_id IN + (SELECT i_manufact_id + FROM main.item + WHERE i_category IN ('Electronics')) + AND ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 5 + AND ss_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_manufact_id +), +ss AS ( +SELECT world, i_manufact_id, total_sales[world+1] AS total_sales +FROM ss_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +cs_stats AS ( +SELECT i_manufact_id, + tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales, + tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence +FROM main.catalog_sales, + main.date_dim, + main.customer_address, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cs_bill_customer_sk=asa_owner.asa_key AND i_manufact_id IN + (SELECT i_manufact_id + FROM main.item + WHERE i_category IN ('Electronics')) + AND cs_item_sk = i_item_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 5 + AND cs_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_manufact_id +), +cs AS ( +SELECT world, i_manufact_id, total_sales[world+1] AS total_sales +FROM cs_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +ws_stats AS ( +SELECT i_manufact_id, + tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales, + tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence +FROM main.web_sales, + main.date_dim, + main.customer_address, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ws_bill_customer_sk=asa_owner.asa_key AND i_manufact_id IN + (SELECT i_manufact_id + FROM main.item + WHERE i_category IN ('Electronics')) + AND ws_item_sk = i_item_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 5 + AND ws_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_manufact_id +), +ws AS ( +SELECT world, i_manufact_id, total_sales[world+1] AS total_sales +FROM ws_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +combined AS ( +SELECT world, i_manufact_id, + sum(total_sales) total_sales +FROM + (SELECT * + FROM ss + UNION ALL SELECT * + FROM cs + UNION ALL SELECT * + FROM ws) tmp1 +GROUP BY world, i_manufact_id +) +SELECT * FROM combined QUALIFY row_number() OVER (PARTITION BY world ORDER BY total_sales) <= 100 +ORDER BY world, total_sales; diff --git a/benchmark/tpcds/reference_queries/q34.sql b/benchmark/tpcds/reference_queries/q34.sql new file mode 100644 index 00000000..97109eb0 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q34.sql @@ -0,0 +1,51 @@ +-- AS aggregation is shared; only the reduced per-ticket results are expanded by world. +WITH stats AS ( + SELECT ss_ticket_number, + ss_customer_sk, + tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS cnt, + tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS presence + FROM main.store_sales, + main.date_dim, + main.store, + main.household_demographics, + (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key + AND store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND (date_dim.d_dom BETWEEN 1 AND 3 + OR date_dim.d_dom BETWEEN 25 AND 28) + AND (household_demographics.hd_buy_potential = '>10000' + OR household_demographics.hd_buy_potential = 'Unknown') + AND household_demographics.hd_vehicle_count > 0 + AND (CASE + WHEN household_demographics.hd_vehicle_count > 0 THEN (household_demographics.hd_dep_count*1.000)/ household_demographics.hd_vehicle_count + ELSE NULL + END) > 1.2 + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_county = 'Williamson County' + GROUP BY ss_ticket_number, + ss_customer_sk +), expanded AS ( + SELECT world, ss_ticket_number, ss_customer_sk, cnt[world+1] AS cnt + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1]>0 +) +SELECT world, c_last_name , + c_first_name , + c_salutation , + c_preferred_cust_flag , + ss_ticket_number , + cnt +FROM + expanded dn, + main.customer +WHERE ss_customer_sk = c_customer_sk + AND cnt BETWEEN 15 AND 20 +ORDER BY world, c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + c_salutation NULLS FIRST, + c_preferred_cust_flag DESC NULLS FIRST, + ss_ticket_number NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q35.sql b/benchmark/tpcds/reference_queries/q35.sql new file mode 100644 index 00000000..4a421997 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q35.sql @@ -0,0 +1,111 @@ +-- Same-owner EXISTS is static after entity inclusion; shipping-customer EXISTS needs the bill-owner mask. +WITH catalog_presence AS ( +SELECT cs_ship_customer_sk,bit_or(priv_hash(hash(cs_bill_customer_sk))) AS membership + FROM main.catalog_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4 +GROUP BY cs_ship_customer_sk +), stats AS (SELECT ca_state, +cd_gender, +cd_marital_status, +cd_dep_count, +cd_dep_employed_count, +cd_dep_college_count, +tpcds_count((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),1) AS n, +tpcds_min((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_count) AS min1, +tpcds_max((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_count) AS max1, +tpcds_avg((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_count) AS avg1, +tpcds_min((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_employed_count) AS min2, +tpcds_max((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_employed_count) AS max2, +tpcds_avg((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_employed_count) AS avg2, +tpcds_min((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_college_count) AS min3, +tpcds_max((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_college_count) AS max3, +tpcds_avg((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_college_count) AS avg3 +FROM main.customer c CROSS JOIN + main.customer_address ca CROSS JOIN + main.customer_demographics LEFT JOIN catalog_presence ON c.c_customer_sk=cs_ship_customer_sk +WHERE c.c_current_addr_sk = ca.ca_address_sk + AND cd_demo_sk = c.c_current_cdemo_sk + +AND EXISTS(SELECT * + FROM main.store_sales, + main.date_dim + WHERE c.c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) +GROUP BY ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count +), answers AS (SELECT world, ca_state, cd_gender, cd_marital_status, cd_dep_count, n[world+1] AS cnt1, min1[world+1] AS min1, max1[world+1] AS max1, avg1[world+1] AS avg1, cd_dep_employed_count, n[world+1] AS cnt2, min2[world+1] AS min2, max2[world+1] AS max2, avg2[world+1] AS avg2, cd_dep_college_count, n[world+1] AS cnt3, min3[world+1] AS min3, max3[world+1] AS max3, avg3[world+1] AS avg3 FROM stats CROSS JOIN range(64) worlds(world) WHERE n[world+1]>0) +SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY ca_state NULLS FIRST, + cd_gender NULLS FIRST, + cd_marital_status NULLS FIRST, + cd_dep_count NULLS FIRST, + cd_dep_employed_count NULLS FIRST, + cd_dep_college_count NULLS FIRST)<=100 ORDER BY world,ca_state NULLS FIRST, + cd_gender NULLS FIRST, + cd_marital_status NULLS FIRST, + cd_dep_count NULLS FIRST, + cd_dep_employed_count NULLS FIRST, + cd_dep_college_count NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q40.sql b/benchmark/tpcds/reference_queries/q40.sql new file mode 100644 index 00000000..1533f6af --- /dev/null +++ b/benchmark/tpcds/reference_queries/q40.sql @@ -0,0 +1,16 @@ +-- Matched and NULL-extended memberships are disjoint; NULL extension is evaluated separately in each world. +WITH left_input AS ( +SELECT w_state,i_item_id,d_date,cs_sales_price,cs_order_number,cs_item_sk,priv_hash(hash(cs_bill_customer_sk)) AS membership +FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk +JOIN main.warehouse ON cs_warehouse_sk=w_warehouse_sk JOIN main.item ON i_item_sk=cs_item_sk +JOIN main.date_dim ON cs_sold_date_sk=d_date_sk +WHERE i_current_price BETWEEN 0.99 AND 1.49 AND d_date BETWEEN DATE '2000-02-10' AND DATE '2000-04-10' +), valid_returns AS (SELECT r.*,priv_hash(hash(cr_returning_customer_sk)) AS membership FROM main.catalog_returns r JOIN main.customer ON cr_returning_customer_sk=c_customer_sk), +return_presence AS (SELECT cr_order_number,cr_item_sk,bit_or(membership) AS membership FROM valid_returns GROUP BY cr_order_number,cr_item_sk), +joined AS ( + SELECT l.w_state,l.i_item_id,l.d_date,l.cs_sales_price,coalesce(r.cr_refunded_cash,0) AS refund,(l.membership & r.membership) AS membership FROM left_input l JOIN valid_returns r ON l.cs_order_number=r.cr_order_number AND l.cs_item_sk=r.cr_item_sk + UNION ALL + SELECT l.w_state,l.i_item_id,l.d_date,l.cs_sales_price,0 AS refund,(l.membership & ~coalesce(r.membership,0::UBIGINT)) AS membership FROM left_input l LEFT JOIN return_presence r ON l.cs_order_number=r.cr_order_number AND l.cs_item_sk=r.cr_item_sk +), stats AS (SELECT w_state,i_item_id,tpcds_sum(membership,CASE WHEN d_date=DATE '2000-03-11' THEN cs_sales_price-refund ELSE 0 END) AS sales_after,tpcds_count(membership,1) AS presence FROM joined GROUP BY w_state,i_item_id), +answers AS (SELECT world,w_state,i_item_id,sales_before[world+1] AS sales_before,sales_after[world+1] AS sales_after FROM stats CROSS JOIN range(64) worlds(world) WHERE presence[world+1]>0) +SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY w_state,i_item_id)<=100 ORDER BY world,w_state,i_item_id; diff --git a/benchmark/tpcds/reference_queries/q42.sql b/benchmark/tpcds/reference_queries/q42.sql new file mode 100644 index 00000000..3b28dcd2 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q42.sql @@ -0,0 +1,31 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT dt.d_year AS d_year, + item.i_category_id AS i_category_id, + item.i_category AS i_category, + tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS total, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.date_dim dt, + main.store_sales, + main.item, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND dt.d_date_sk = store_sales.ss_sold_date_sk + AND store_sales.ss_item_sk = item.i_item_sk + AND item.i_manager_id = 1 + AND dt.d_moy=11 + AND dt.d_year=2000 +GROUP BY dt.d_year, + main.item.i_category_id, + main.item.i_category +), answers AS ( + SELECT world, d_year, + i_category_id, + i_category, + total[world+1] AS total + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY total DESC, d_year, i_category_id, i_category) <= 100 +ORDER BY world, total DESC, d_year, i_category_id, i_category; diff --git a/benchmark/tpcds/reference_queries/q43.sql b/benchmark/tpcds/reference_queries/q43.sql new file mode 100644 index 00000000..63cc41c9 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q43.sql @@ -0,0 +1,39 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT s_store_name, + s_store_id, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Sunday' THEN ss_sales_price ELSE NULL END) AS sun_sales, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Monday' THEN ss_sales_price ELSE NULL END) AS mon_sales, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Tuesday' THEN ss_sales_price ELSE NULL END) AS tue_sales, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Wednesday' THEN ss_sales_price ELSE NULL END) AS wed_sales, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Thursday' THEN ss_sales_price ELSE NULL END) AS thu_sales, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Friday' THEN ss_sales_price ELSE NULL END) AS fri_sales, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Saturday' THEN ss_sales_price ELSE NULL END) AS sat_sales, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.date_dim, + main.store_sales, + main.store, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND d_date_sk = ss_sold_date_sk + AND s_store_sk = ss_store_sk + AND s_gmt_offset = -5 + AND d_year = 2000 +GROUP BY s_store_name, + s_store_id +), answers AS ( + SELECT world, s_store_name, + s_store_id, + sun_sales[world+1] AS sun_sales, + mon_sales[world+1] AS mon_sales, + tue_sales[world+1] AS tue_sales, + wed_sales[world+1] AS wed_sales, + thu_sales[world+1] AS thu_sales, + fri_sales[world+1] AS fri_sales, + sat_sales[world+1] AS sat_sales + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY s_store_name, s_store_id, sun_sales, mon_sales, tue_sales, wed_sales, thu_sales, fri_sales, sat_sales) <= 100 +ORDER BY world, s_store_name, s_store_id, sun_sales, mon_sales, tue_sales, wed_sales, thu_sales, fri_sales, sat_sales; diff --git a/benchmark/tpcds/reference_queries/q45.sql b/benchmark/tpcds/reference_queries/q45.sql new file mode 100644 index 00000000..f6f4d573 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q45.sql @@ -0,0 +1,53 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT ca_zip, + ca_city, + tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), ws_sales_price) AS total, + tpcds_count((priv_hash(hash(ws_bill_customer_sk))), 1) AS presence + FROM main.web_sales, + main.customer, + main.customer_address, + main.date_dim, + main.item, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ws_bill_customer_sk = asa_owner0.asa_key0 + AND ws_bill_customer_sk = c_customer_sk + AND c_current_addr_sk = ca_address_sk + AND ws_item_sk = i_item_sk + AND (SUBSTRING(ca_zip,1,5) IN ('85669', + '86197', + '88274', + '83405', + '86475', + '85392', + '85460', + '80348', + '81792') + OR i_item_id IN + (SELECT i_item_id + FROM main.item + WHERE i_item_sk IN (2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29) )) + AND ws_sold_date_sk = d_date_sk + AND d_qoy = 2 + AND d_year = 2001 +GROUP BY ca_zip, + ca_city +), answers AS ( + SELECT world, ca_zip, + ca_city, + total[world+1] AS total + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY ca_zip, ca_city) <= 100 +ORDER BY world, ca_zip, ca_city; diff --git a/benchmark/tpcds/reference_queries/q46.sql b/benchmark/tpcds/reference_queries/q46.sql new file mode 100644 index 00000000..3d95ef46 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q46.sql @@ -0,0 +1,61 @@ +-- AS aggregation is shared; only the reduced per-ticket results are expanded by world. +WITH stats AS ( + SELECT ss_ticket_number, + ss_customer_sk, + ca_city AS bought_city, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) AS amt, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) AS profit, + tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS presence + FROM main.store_sales, + main.date_dim, + main.store, + main.household_demographics, + main.customer_address, + (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key + AND store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND store_sales.ss_addr_sk = customer_address.ca_address_sk + AND (household_demographics.hd_dep_count = 4 + OR household_demographics.hd_vehicle_count= 3) + AND date_dim.d_dow IN (6, + 0) + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_city IN ('Fairview', + 'Midway') + GROUP BY ss_ticket_number, + ss_customer_sk, + ss_addr_sk, + ca_city +), expanded AS ( + SELECT world, ss_ticket_number, ss_customer_sk, bought_city, amt[world+1] AS amt, profit[world+1] AS profit + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1]>0 +) +SELECT world, c_last_name, + c_first_name, + ca_city, + bought_city, + ss_ticket_number, + amt, + profit +FROM + expanded dn, + main.customer, + main.customer_address current_addr +WHERE ss_customer_sk = c_customer_sk + AND customer.c_current_addr_sk = current_addr.ca_address_sk + AND current_addr.ca_city <> bought_city +QUALIFY row_number() OVER (PARTITION BY world ORDER BY c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + ca_city NULLS FIRST, + bought_city NULLS FIRST, + ss_ticket_number NULLS FIRST) <= 100 +ORDER BY world, c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + ca_city NULLS FIRST, + bought_city NULLS FIRST, + ss_ticket_number NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q48.sql b/benchmark/tpcds/reference_queries/q48.sql new file mode 100644 index 00000000..ebf74011 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q48.sql @@ -0,0 +1,50 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_quantity) AS total, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.store_sales, + main.store, + main.customer_demographics, + main.customer_address, + main.date_dim, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND s_store_sk = ss_store_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2000 + AND ((cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'M' + AND cd_education_status = '4 yr Degree' + AND ss_sales_price BETWEEN 100.00 AND 150.00) + OR (cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'D' + AND cd_education_status = '2 yr Degree' + AND ss_sales_price BETWEEN 50.00 AND 100.00) + OR (cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND ss_sales_price BETWEEN 150.00 AND 200.00)) + AND ((ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('CO', + 'OH', + 'TX') + AND ss_net_profit BETWEEN 0 AND 2000) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('OR', + 'MN', + 'KY') + AND ss_net_profit BETWEEN 150 AND 3000) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('VA', + 'CA', + 'MS') + AND ss_net_profit BETWEEN 50 AND 25000)) +), answers AS ( + SELECT world, total[world+1] AS total + FROM stats CROSS JOIN range(64) worlds(world) +) +SELECT * FROM answers +ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q50.sql b/benchmark/tpcds/reference_queries/q50.sql new file mode 100644 index 00000000..4a699354 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q50.sql @@ -0,0 +1,65 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT s_store_name, + s_company_id, + s_street_number, + s_street_name, + s_street_type, + s_suite_number, + s_city, + s_county, + s_state, + s_zip, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN sr_returned_date_sk - ss_sold_date_sk <= 30 THEN 1 ELSE 0 END) AS bucket0, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN sr_returned_date_sk - ss_sold_date_sk > 30 AND sr_returned_date_sk - ss_sold_date_sk <= 60 THEN 1 ELSE 0 END) AS bucket1, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN sr_returned_date_sk - ss_sold_date_sk > 60 AND sr_returned_date_sk - ss_sold_date_sk <= 90 THEN 1 ELSE 0 END) AS bucket2, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN sr_returned_date_sk - ss_sold_date_sk > 90 AND sr_returned_date_sk - ss_sold_date_sk <= 120 THEN 1 ELSE 0 END) AS bucket3, + tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN sr_returned_date_sk - ss_sold_date_sk > 120 THEN 1 ELSE 0 END) AS bucket4, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.store_sales, + main.store_returns, + main.store, + main.date_dim d1, + main.date_dim d2, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND d2.d_year = 2001 + AND d2.d_moy = 8 + AND ss_ticket_number = sr_ticket_number + AND ss_item_sk = sr_item_sk + AND ss_sold_date_sk = d1.d_date_sk + AND sr_returned_date_sk = d2.d_date_sk + AND ss_customer_sk = sr_customer_sk + AND ss_store_sk = s_store_sk +GROUP BY s_store_name, + s_company_id, + s_street_number, + s_street_name, + s_street_type, + s_suite_number, + s_city, + s_county, + s_state, + s_zip +), answers AS ( + SELECT world, s_store_name, + s_company_id, + s_street_number, + s_street_name, + s_street_type, + s_suite_number, + s_city, + s_county, + s_state, + s_zip, + bucket0[world+1] AS bucket0, + bucket1[world+1] AS bucket1, + bucket2[world+1] AS bucket2, + bucket3[world+1] AS bucket3, + bucket4[world+1] AS bucket4 + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip) <= 100 +ORDER BY world, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip; diff --git a/benchmark/tpcds/reference_queries/q52.sql b/benchmark/tpcds/reference_queries/q52.sql new file mode 100644 index 00000000..dbbc5c6c --- /dev/null +++ b/benchmark/tpcds/reference_queries/q52.sql @@ -0,0 +1,31 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT dt.d_year AS d_year, + item.i_brand_id AS brand_id, + item.i_brand AS brand, + tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS ext_price, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.date_dim dt, + main.store_sales, + main.item, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND dt.d_date_sk = store_sales.ss_sold_date_sk + AND store_sales.ss_item_sk = item.i_item_sk + AND item.i_manager_id = 1 + AND dt.d_moy=11 + AND dt.d_year=2000 +GROUP BY dt.d_year, + main.item.i_brand, + main.item.i_brand_id +), answers AS ( + SELECT world, d_year, + brand_id, + brand, + ext_price[world+1] AS ext_price + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY d_year, ext_price DESC, brand_id) <= 100 +ORDER BY world, d_year, ext_price DESC, brand_id; diff --git a/benchmark/tpcds/reference_queries/q55.sql b/benchmark/tpcds/reference_queries/q55.sql new file mode 100644 index 00000000..14f28825 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q55.sql @@ -0,0 +1,28 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT i_brand_id AS brand_id, + i_brand AS brand, + tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS ext_price, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.date_dim, + main.store_sales, + main.item, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND d_date_sk = ss_sold_date_sk + AND ss_item_sk = i_item_sk + AND i_manager_id=28 + AND d_moy=11 + AND d_year=1999 +GROUP BY i_brand, + i_brand_id +), answers AS ( + SELECT world, brand_id, + brand, + ext_price[world+1] AS ext_price + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY ext_price DESC, brand_id) <= 100 +ORDER BY world, ext_price DESC, brand_id; diff --git a/benchmark/tpcds/reference_queries/q56.sql b/benchmark/tpcds/reference_queries/q56.sql new file mode 100644 index 00000000..267dc3fd --- /dev/null +++ b/benchmark/tpcds/reference_queries/q56.sql @@ -0,0 +1,98 @@ +-- Fact scans feed AS once; subsequent processing is over reduced aggregate results. +WITH ss_stats AS ( +SELECT i_item_id, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales, + tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence +FROM main.store_sales, + main.date_dim, + main.customer_address, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND i_item_id IN + (SELECT i_item_id + FROM main.item + WHERE i_color IN ('slate', + 'blanched', + 'burnished')) + AND ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 2 + AND ss_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id +), +ss AS ( +SELECT world, i_item_id, total_sales[world+1] AS total_sales +FROM ss_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +cs_stats AS ( +SELECT i_item_id, + tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales, + tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence +FROM main.catalog_sales, + main.date_dim, + main.customer_address, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cs_bill_customer_sk=asa_owner.asa_key AND i_item_id IN + (SELECT i_item_id + FROM main.item + WHERE i_color IN ('slate', + 'blanched', + 'burnished')) + AND cs_item_sk = i_item_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 2 + AND cs_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id +), +cs AS ( +SELECT world, i_item_id, total_sales[world+1] AS total_sales +FROM cs_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +ws_stats AS ( +SELECT i_item_id, + tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales, + tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence +FROM main.web_sales, + main.date_dim, + main.customer_address, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ws_bill_customer_sk=asa_owner.asa_key AND i_item_id IN + (SELECT i_item_id + FROM main.item + WHERE i_color IN ('slate', + 'blanched', + 'burnished')) + AND ws_item_sk = i_item_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 2 + AND ws_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id +), +ws AS ( +SELECT world, i_item_id, total_sales[world+1] AS total_sales +FROM ws_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +combined AS ( +SELECT world, i_item_id, + sum(total_sales) total_sales +FROM + (SELECT * + FROM ss + UNION ALL SELECT * + FROM cs + UNION ALL SELECT * + FROM ws) tmp1 +GROUP BY world, i_item_id +) +SELECT * FROM combined QUALIFY row_number() OVER (PARTITION BY world ORDER BY total_sales NULLS FIRST, + i_item_id NULLS FIRST) <= 100 +ORDER BY world, total_sales NULLS FIRST, + i_item_id NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q58.sql b/benchmark/tpcds/reference_queries/q58.sql new file mode 100644 index 00000000..3396ad99 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q58.sql @@ -0,0 +1,93 @@ +-- Fact scans feed AS once; subsequent processing is over reduced aggregate results. +WITH ss_items_stats AS ( +SELECT i_item_id item_id, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) ss_item_rev, + tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence +FROM main.store_sales, + main.item, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND ss_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM main.date_dim + WHERE d_week_seq = + (SELECT d_week_seq + FROM main.date_dim + WHERE d_date = '2000-01-03')) + AND ss_sold_date_sk = d_date_sk + GROUP BY i_item_id +), +ss_items AS ( +SELECT world, item_id, ss_item_rev[world+1] AS ss_item_rev +FROM ss_items_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +cs_items_stats AS ( +SELECT i_item_id item_id, + tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) cs_item_rev, + tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence +FROM main.catalog_sales, + main.item, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM main.date_dim + WHERE d_week_seq = + (SELECT d_week_seq + FROM main.date_dim + WHERE d_date = '2000-01-03')) + AND cs_sold_date_sk = d_date_sk + GROUP BY i_item_id +), +cs_items AS ( +SELECT world, item_id, cs_item_rev[world+1] AS cs_item_rev +FROM cs_items_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +ws_items_stats AS ( +SELECT i_item_id item_id, + tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) ws_item_rev, + tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence +FROM main.web_sales, + main.item, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM main.date_dim + WHERE d_week_seq = + (SELECT d_week_seq + FROM main.date_dim + WHERE d_date = '2000-01-03')) + AND ws_sold_date_sk = d_date_sk + GROUP BY i_item_id +), +ws_items AS ( +SELECT world, item_id, ws_item_rev[world+1] AS ws_item_rev +FROM ws_items_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT ss_items.world AS world, ss_items.item_id, + ss_item_rev, + ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev, + cs_item_rev, + cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev, + ws_item_rev, + ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev, + (ss_item_rev+cs_item_rev+ws_item_rev)/3 average +FROM ss_items, + cs_items, + ws_items +WHERE ss_items.world=cs_items.world AND ss_items.world=ws_items.world AND ss_items.item_id=cs_items.item_id + AND ss_items.item_id=ws_items.item_id + AND ss_item_rev BETWEEN 0.9 * cs_item_rev AND 1.1 * cs_item_rev + AND ss_item_rev BETWEEN 0.9 * ws_item_rev AND 1.1 * ws_item_rev + AND cs_item_rev BETWEEN 0.9 * ss_item_rev AND 1.1 * ss_item_rev + AND cs_item_rev BETWEEN 0.9 * ws_item_rev AND 1.1 * ws_item_rev + AND ws_item_rev BETWEEN 0.9 * ss_item_rev AND 1.1 * ss_item_rev + AND ws_item_rev BETWEEN 0.9 * cs_item_rev AND 1.1 * cs_item_rev +QUALIFY row_number() OVER (PARTITION BY ss_items.world ORDER BY ss_items.item_id NULLS FIRST, + ss_item_rev NULLS FIRST) <= 100 +ORDER BY ss_items.world, ss_items.item_id NULLS FIRST, + ss_item_rev NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q59.sql b/benchmark/tpcds/reference_queries/q59.sql new file mode 100644 index 00000000..e2695b7c --- /dev/null +++ b/benchmark/tpcds/reference_queries/q59.sql @@ -0,0 +1,94 @@ +-- A shared per-store/week AS result is reused across both years. +WITH wss_stats AS ( +SELECT d_week_seq, + ss_store_sk, + tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Sunday') THEN ss_sales_price + ELSE NULL + END) sun_sales, + tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Monday') THEN ss_sales_price + ELSE NULL + END) mon_sales, + tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Tuesday') THEN ss_sales_price + ELSE NULL + END) tue_sales, + tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Wednesday') THEN ss_sales_price + ELSE NULL + END) wed_sales, + tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Thursday') THEN ss_sales_price + ELSE NULL + END) thu_sales, + tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Friday') THEN ss_sales_price + ELSE NULL + END) fri_sales, + tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE + WHEN (d_day_name='Saturday') THEN ss_sales_price + ELSE NULL + END) sat_sales, + tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence +FROM main.store_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND d_date_sk = ss_sold_date_sk + GROUP BY d_week_seq, + ss_store_sk +), wss AS ( +SELECT world, d_week_seq, ss_store_sk, sun_sales[world+1] AS sun_sales, mon_sales[world+1] AS mon_sales, tue_sales[world+1] AS tue_sales, wed_sales[world+1] AS wed_sales, thu_sales[world+1] AS thu_sales, fri_sales[world+1] AS fri_sales, sat_sales[world+1] AS sat_sales +FROM wss_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT y.world AS world, s_store_name1, + s_store_id1, + d_week_seq1, + sun_sales1/sun_sales2 AS sun_sales_ratio, + mon_sales1/mon_sales2 AS mon_sales_ratio, + tue_sales1/tue_sales2 AS tue_sales_ratio, + wed_sales1/wed_sales2 AS wed_sales_ratio, + thu_sales1/thu_sales2 AS thu_sales_ratio, + fri_sales1/fri_sales2 AS fri_sales_ratio, + sat_sales1/sat_sales2 AS sat_sales_ratio +FROM + (SELECT wss.world, s_store_name s_store_name1, + wss.d_week_seq d_week_seq1, + s_store_id s_store_id1, + sun_sales sun_sales1, + mon_sales mon_sales1, + tue_sales tue_sales1, + wed_sales wed_sales1, + thu_sales thu_sales1, + fri_sales fri_sales1, + sat_sales sat_sales1 + FROM wss, + main.store, + main.date_dim d + WHERE d.d_week_seq = wss.d_week_seq + AND ss_store_sk = s_store_sk + AND d_month_seq BETWEEN 1212 AND 1212 + 11) y, + (SELECT wss.world, s_store_name s_store_name2, + wss.d_week_seq d_week_seq2, + s_store_id s_store_id2, + sun_sales sun_sales2, + mon_sales mon_sales2, + tue_sales tue_sales2, + wed_sales wed_sales2, + thu_sales thu_sales2, + fri_sales fri_sales2, + sat_sales sat_sales2 + FROM wss, + main.store, + main.date_dim d + WHERE d.d_week_seq = wss.d_week_seq + AND ss_store_sk = s_store_sk + AND d_month_seq BETWEEN 1212 + 12 AND 1212 + 23) x +WHERE y.world=x.world AND s_store_id1=s_store_id2 + AND d_week_seq1=d_week_seq2-52 +QUALIFY row_number() OVER (PARTITION BY y.world ORDER BY s_store_name1 NULLS FIRST, + s_store_id1 NULLS FIRST, + d_week_seq1 NULLS FIRST) <= 100 +ORDER BY y.world, s_store_name1 NULLS FIRST, + s_store_id1 NULLS FIRST, + d_week_seq1 NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q60.sql b/benchmark/tpcds/reference_queries/q60.sql new file mode 100644 index 00000000..86f4a209 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q60.sql @@ -0,0 +1,92 @@ +-- Fact scans feed AS once; subsequent processing is over reduced aggregate results. +WITH ss_stats AS ( +SELECT i_item_id, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales, + tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence +FROM main.store_sales, + main.date_dim, + main.customer_address, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND i_item_id IN + (SELECT i_item_id + FROM main.item + WHERE i_category ='Music') + AND ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 9 + AND ss_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id +), +ss AS ( +SELECT world, i_item_id, total_sales[world+1] AS total_sales +FROM ss_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +cs_stats AS ( +SELECT i_item_id, + tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales, + tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence +FROM main.catalog_sales, + main.date_dim, + main.customer_address, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cs_bill_customer_sk=asa_owner.asa_key AND i_item_id IN + (SELECT i_item_id + FROM main.item + WHERE i_category ='Music') + AND cs_item_sk = i_item_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 9 + AND cs_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id +), +cs AS ( +SELECT world, i_item_id, total_sales[world+1] AS total_sales +FROM cs_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +ws_stats AS ( +SELECT i_item_id, + tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales, + tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence +FROM main.web_sales, + main.date_dim, + main.customer_address, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ws_bill_customer_sk=asa_owner.asa_key AND i_item_id IN + (SELECT i_item_id + FROM main.item + WHERE i_category = 'Music') + AND ws_item_sk = i_item_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 9 + AND ws_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id +), +ws AS ( +SELECT world, i_item_id, total_sales[world+1] AS total_sales +FROM ws_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +combined AS ( +SELECT world, i_item_id, + sum(total_sales) total_sales +FROM + (SELECT * + FROM ss + UNION ALL SELECT * + FROM cs + UNION ALL SELECT * + FROM ws) tmp1 +GROUP BY world, i_item_id +) +SELECT * FROM combined QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id, + total_sales) <= 100 +ORDER BY world, i_item_id, + total_sales; diff --git a/benchmark/tpcds/reference_queries/q61.sql b/benchmark/tpcds/reference_queries/q61.sql new file mode 100644 index 00000000..737d3390 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q61.sql @@ -0,0 +1,49 @@ +-- Scalar AS arrays are combined only after their independent shared scans. +WITH raw0 AS ( +SELECT tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) promotions + FROM main.store_sales, + main.store, + main.promotion, + main.date_dim, + main.customer, + main.customer_address, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk = d_date_sk + AND ss_store_sk = s_store_sk + AND ss_promo_sk = p_promo_sk + AND ss_customer_sk= c_customer_sk + AND ca_address_sk = c_current_addr_sk + AND ss_item_sk = i_item_sk + AND ca_gmt_offset = -5 + AND i_category = 'Jewelry' + AND (p_channel_dmail = 'Y' + OR p_channel_email = 'Y' + OR p_channel_tv = 'Y') + AND s_gmt_offset = -5 + AND d_year = 1998 + AND d_moy = 11 +), +raw1 AS ( +SELECT tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total + FROM main.store_sales, + main.store, + main.date_dim, + main.customer, + main.customer_address, + main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk = d_date_sk + AND ss_store_sk = s_store_sk + AND ss_customer_sk= c_customer_sk + AND ca_address_sk = c_current_addr_sk + AND ss_item_sk = i_item_sk + AND ca_gmt_offset = -5 + AND i_category = 'Jewelry' + AND s_gmt_offset = -5 + AND d_year = 1998 + AND d_moy = 11 +), +p AS (SELECT world, raw0.promotions[world+1] AS promotions, raw1.total[world+1] AS total FROM raw0 CROSS JOIN raw1 CROSS JOIN range(64) worlds(world)) +SELECT world, p.promotions, + p.total, + cast(p.promotions AS decimal(15,4))/cast(p.total AS decimal(15,4))*100 + FROM p ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q62.sql b/benchmark/tpcds/reference_queries/q62.sql new file mode 100644 index 00000000..b247bb7f --- /dev/null +++ b/benchmark/tpcds/reference_queries/q62.sql @@ -0,0 +1,43 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT w_substr, + sm_type, + web_name, + tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), CASE WHEN ws_ship_date_sk - ws_sold_date_sk <= 30 THEN 1 ELSE 0 END) AS bucket0, + tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), CASE WHEN ws_ship_date_sk - ws_sold_date_sk > 30 AND ws_ship_date_sk - ws_sold_date_sk <= 60 THEN 1 ELSE 0 END) AS bucket1, + tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), CASE WHEN ws_ship_date_sk - ws_sold_date_sk > 60 AND ws_ship_date_sk - ws_sold_date_sk <= 90 THEN 1 ELSE 0 END) AS bucket2, + tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), CASE WHEN ws_ship_date_sk - ws_sold_date_sk > 90 AND ws_ship_date_sk - ws_sold_date_sk <= 120 THEN 1 ELSE 0 END) AS bucket3, + tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), CASE WHEN ws_ship_date_sk - ws_sold_date_sk > 120 THEN 1 ELSE 0 END) AS bucket4, + tpcds_count((priv_hash(hash(ws_bill_customer_sk))), 1) AS presence + FROM main.web_sales, + (SELECT SUBSTRING(w_warehouse_name,1,20) w_substr, + * + FROM main.warehouse) sq1, + main.ship_mode, + main.web_site, + main.date_dim, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ws_bill_customer_sk = asa_owner0.asa_key0 + AND d_month_seq BETWEEN 1200 AND 1200 + 11 + AND ws_ship_date_sk = d_date_sk + AND ws_warehouse_sk = w_warehouse_sk + AND ws_ship_mode_sk = sm_ship_mode_sk + AND ws_web_site_sk = web_site_sk +GROUP BY w_substr, + sm_type, + web_name +), answers AS ( + SELECT world, w_substr, + sm_type, + web_name, + bucket0[world+1] AS bucket0, + bucket1[world+1] AS bucket1, + bucket2[world+1] AS bucket2, + bucket3[world+1] AS bucket3, + bucket4[world+1] AS bucket4 + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY w_substr NULLS FIRST, sm_type NULLS FIRST, web_name NULLS FIRST) <= 100 +ORDER BY world, w_substr NULLS FIRST, sm_type NULLS FIRST, web_name NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q64.sql b/benchmark/tpcds/reference_queries/q64.sql new file mode 100644 index 00000000..bb5a2b4d --- /dev/null +++ b/benchmark/tpcds/reference_queries/q64.sql @@ -0,0 +1,133 @@ +-- The first stochastic HAVING is folded into membership before the cross-sales aggregate. +WITH cs_ui_stats AS ( +SELECT cs_item_sk, + tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & priv_hash(hash(cr_returning_customer_sk))), cs_ext_list_price) AS sale, + tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & priv_hash(hash(cr_returning_customer_sk))), cr_refunded_cash+cr_reversed_charge+cr_store_credit) AS refund, tpcds_count((priv_hash(hash(cs_bill_customer_sk)) & priv_hash(hash(cr_returning_customer_sk))),1) AS presence +FROM main.catalog_sales, + main.catalog_returns, (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0, (SELECT c_customer_sk AS asa_key1 FROM main.customer) asa_owner1 +WHERE cs_bill_customer_sk=asa_owner0.asa_key0 AND cr_returning_customer_sk=asa_owner1.asa_key1 AND cs_item_sk = cr_item_sk + AND cs_order_number = cr_order_number + GROUP BY cs_item_sk + +), +cs_ui AS (SELECT cs_item_sk,priv_select(18446744073709551615::UBIGINT,list_transform(list_zip(sale,refund),lambda x: x[1]>2*x[2])) AS membership FROM cs_ui_stats), +cross_sales_stats AS ( +SELECT i_product_name product_name, + i_item_sk item_sk, + s_store_name store_name, + s_zip store_zip, + ad1.ca_street_number b_street_number, + ad1.ca_street_name b_street_name, + ad1.ca_city b_city, + ad1.ca_zip b_zip, + ad2.ca_street_number c_street_number, + ad2.ca_street_name c_street_name, + ad2.ca_city c_city, + ad2.ca_zip c_zip, + d1.d_year AS syear, + d2.d_year AS fsyear, + d3.d_year s2year, + tpcds_count((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk)) & cs_ui.membership),1) cnt, + tpcds_sum((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk)) & cs_ui.membership), ss_wholesale_cost) s1, + tpcds_sum((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk)) & cs_ui.membership), ss_list_price) s2, + tpcds_sum((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk)) & cs_ui.membership), ss_coupon_amt) s3, tpcds_count((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk)) & cs_ui.membership),1) AS presence +FROM main.store_sales, + main.store_returns, + cs_ui, + main.date_dim d1, + main.date_dim d2, + main.date_dim d3, + main.store, + main.customer, + main.customer_demographics cd1, + main.customer_demographics cd2, + main.promotion, + main.household_demographics hd1, + main.household_demographics hd2, + main.customer_address ad1, + main.customer_address ad2, + main.income_band ib1, + main.income_band ib2, + main.item, (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE sr_customer_sk=asa_owner0.asa_key0 AND ss_store_sk = s_store_sk + AND ss_sold_date_sk = d1.d_date_sk + AND ss_customer_sk = c_customer_sk + AND ss_cdemo_sk= cd1.cd_demo_sk + AND ss_hdemo_sk = hd1.hd_demo_sk + AND ss_addr_sk = ad1.ca_address_sk + AND ss_item_sk = i_item_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND ss_item_sk = cs_ui.cs_item_sk + AND c_current_cdemo_sk = cd2.cd_demo_sk + AND c_current_hdemo_sk = hd2.hd_demo_sk + AND c_current_addr_sk = ad2.ca_address_sk + AND c_first_sales_date_sk = d2.d_date_sk + AND c_first_shipto_date_sk = d3.d_date_sk + AND ss_promo_sk = p_promo_sk + AND hd1.hd_income_band_sk = ib1.ib_income_band_sk + AND hd2.hd_income_band_sk = ib2.ib_income_band_sk + AND cd1.cd_marital_status <> cd2.cd_marital_status + AND i_color IN ('purple', + 'burlywood', + 'indian', + 'spring', + 'floral', + 'medium') + AND i_current_price BETWEEN 64 AND 64 + 10 + AND i_current_price BETWEEN 64 + 1 AND 64 + 15 + GROUP BY i_product_name, + i_item_sk, + s_store_name, + s_zip, + ad1.ca_street_number, + ad1.ca_street_name, + ad1.ca_city, + ad1.ca_zip, + ad2.ca_street_number, + ad2.ca_street_name, + ad2.ca_city, + ad2.ca_zip, + d1.d_year, + d2.d_year, + d3.d_year +), +cross_sales AS ( +SELECT world, product_name, item_sk, store_name, store_zip, b_street_number, b_street_name, b_city, b_zip, c_street_number, c_street_name, c_city, c_zip, syear, fsyear, s2year, cnt[world+1] AS cnt, s1[world+1] AS s1, s2[world+1] AS s2, s3[world+1] AS s3 +FROM cross_sales_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT cs1.world AS world, cs1.product_name, + cs1.store_name, + cs1.store_zip, + cs1.b_street_number, + cs1.b_street_name, + cs1.b_city, + cs1.b_zip, + cs1.c_street_number, + cs1.c_street_name, + cs1.c_city, + cs1.c_zip, + cs1.syear cs1syear, + cs1.cnt cs1cnt, + cs1.s1 AS s11, + cs1.s2 AS s21, + cs1.s3 AS s31, + cs2.s1 AS s12, + cs2.s2 AS s22, + cs2.s3 AS s32, + cs2.syear, + cs2.cnt +FROM cross_sales cs1, + cross_sales cs2 +WHERE cs1.world=cs2.world AND cs1.item_sk=cs2.item_sk + AND cs1.syear = 1999 + AND cs2.syear = 1999 + 1 + AND cs2.cnt <= cs1.cnt + AND cs1.store_name = cs2.store_name + AND cs1.store_zip = cs2.store_zip +ORDER BY cs1.world, cs1.product_name, + cs1.store_name, + cs2.cnt, + cs1.s1, + cs2.s1; diff --git a/benchmark/tpcds/reference_queries/q65.sql b/benchmark/tpcds/reference_queries/q65.sql new file mode 100644 index 00000000..f3d32ffe --- /dev/null +++ b/benchmark/tpcds/reference_queries/q65.sql @@ -0,0 +1,17 @@ +-- Inner item revenues are shared across both outer uses; AVG remains world-specific. +WITH revenue_stats AS ( +SELECT ss_store_sk, ss_item_sk, tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_sales_price) AS revenue, + tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence +FROM main.store_sales, main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk=d_date_sk AND d_month_seq BETWEEN 1176 AND 1176+11 +GROUP BY ss_store_sk, ss_item_sk +), sc AS ( +SELECT world, ss_store_sk, ss_item_sk, revenue[world+1] AS revenue +FROM revenue_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), sb AS (SELECT world, ss_store_sk, avg(revenue) AS ave FROM sc GROUP BY world, ss_store_sk) +SELECT sc.world AS world, s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand +FROM main.store, main.item, sb, sc +WHERE sc.world=sb.world AND sb.ss_store_sk=sc.ss_store_sk AND sc.revenue<=0.1*sb.ave AND s_store_sk=sc.ss_store_sk AND i_item_sk=sc.ss_item_sk +QUALIFY row_number() OVER (PARTITION BY sc.world ORDER BY s_store_name NULLS FIRST, i_item_desc NULLS FIRST) <= 100 +ORDER BY sc.world, s_store_name NULLS FIRST, i_item_desc NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q66.sql b/benchmark/tpcds/reference_queries/q66.sql new file mode 100644 index 00000000..b33e2bdf --- /dev/null +++ b/benchmark/tpcds/reference_queries/q66.sql @@ -0,0 +1,227 @@ +-- Shared monthly AS aggregates retain each channel's original price/profit expression. +WITH stats AS ( + + select + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,'DHL,BARIAN' as ship_carriers + ,d_year as year_ + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 1 + then ws_ext_sales_price* ws_quantity else 0 end) as jan_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 2 + then ws_ext_sales_price* ws_quantity else 0 end) as feb_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 3 + then ws_ext_sales_price* ws_quantity else 0 end) as mar_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 4 + then ws_ext_sales_price* ws_quantity else 0 end) as apr_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 5 + then ws_ext_sales_price* ws_quantity else 0 end) as may_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 6 + then ws_ext_sales_price* ws_quantity else 0 end) as jun_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 7 + then ws_ext_sales_price* ws_quantity else 0 end) as jul_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 8 + then ws_ext_sales_price* ws_quantity else 0 end) as aug_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 9 + then ws_ext_sales_price* ws_quantity else 0 end) as sep_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 10 + then ws_ext_sales_price* ws_quantity else 0 end) as oct_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 11 + then ws_ext_sales_price* ws_quantity else 0 end) as nov_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 12 + then ws_ext_sales_price* ws_quantity else 0 end) as dec_sales + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 1 + then ws_net_paid * ws_quantity else 0 end) as jan_net + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 2 + then ws_net_paid * ws_quantity else 0 end) as feb_net + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 3 + then ws_net_paid * ws_quantity else 0 end) as mar_net + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 4 + then ws_net_paid * ws_quantity else 0 end) as apr_net + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 5 + then ws_net_paid * ws_quantity else 0 end) as may_net + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 6 + then ws_net_paid * ws_quantity else 0 end) as jun_net + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 7 + then ws_net_paid * ws_quantity else 0 end) as jul_net + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 8 + then ws_net_paid * ws_quantity else 0 end) as aug_net + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 9 + then ws_net_paid * ws_quantity else 0 end) as sep_net + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 10 + then ws_net_paid * ws_quantity else 0 end) as oct_net + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 11 + then ws_net_paid * ws_quantity else 0 end) as nov_net + ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 12 + then ws_net_paid * ws_quantity else 0 end) as dec_net, + tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence +from + main.web_sales + ,main.warehouse + ,main.date_dim + ,main.time_dim + ,main.ship_mode, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_warehouse_sk = w_warehouse_sk + and ws_sold_date_sk = d_date_sk + and ws_sold_time_sk = t_time_sk + and ws_ship_mode_sk = sm_ship_mode_sk + and d_year = 2001 + and t_time between 30838 and 30838+28800 + and sm_carrier in ('DHL','BARIAN') + group by + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,d_year + +UNION ALL + + select + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,'DHL,BARIAN' as ship_carriers + ,d_year as year_ + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 1 + then cs_sales_price* cs_quantity else 0 end) as jan_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 2 + then cs_sales_price* cs_quantity else 0 end) as feb_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 3 + then cs_sales_price* cs_quantity else 0 end) as mar_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 4 + then cs_sales_price* cs_quantity else 0 end) as apr_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 5 + then cs_sales_price* cs_quantity else 0 end) as may_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 6 + then cs_sales_price* cs_quantity else 0 end) as jun_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 7 + then cs_sales_price* cs_quantity else 0 end) as jul_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 8 + then cs_sales_price* cs_quantity else 0 end) as aug_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 9 + then cs_sales_price* cs_quantity else 0 end) as sep_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 10 + then cs_sales_price* cs_quantity else 0 end) as oct_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 11 + then cs_sales_price* cs_quantity else 0 end) as nov_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 12 + then cs_sales_price* cs_quantity else 0 end) as dec_sales + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 1 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as jan_net + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 2 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as feb_net + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 3 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as mar_net + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 4 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as apr_net + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 5 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as may_net + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 6 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as jun_net + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 7 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as jul_net + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 8 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as aug_net + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 9 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as sep_net + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 10 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as oct_net + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 11 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as nov_net + ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 12 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as dec_net, + tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence +from + main.catalog_sales + ,main.warehouse + ,main.date_dim + ,main.time_dim + ,main.ship_mode, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_warehouse_sk = w_warehouse_sk + and cs_sold_date_sk = d_date_sk + and cs_sold_time_sk = t_time_sk + and cs_ship_mode_sk = sm_ship_mode_sk + and d_year = 2001 + and t_time between 30838 AND 30838+28800 + and sm_carrier in ('DHL','BARIAN') + group by + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,d_year + +), expanded AS ( +SELECT world, w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, ship_carriers, year_, jan_sales[world+1] AS jan_sales, feb_sales[world+1] AS feb_sales, mar_sales[world+1] AS mar_sales, apr_sales[world+1] AS apr_sales, may_sales[world+1] AS may_sales, jun_sales[world+1] AS jun_sales, jul_sales[world+1] AS jul_sales, aug_sales[world+1] AS aug_sales, sep_sales[world+1] AS sep_sales, oct_sales[world+1] AS oct_sales, nov_sales[world+1] AS nov_sales, dec_sales[world+1] AS dec_sales, jan_net[world+1] AS jan_net, feb_net[world+1] AS feb_net, mar_net[world+1] AS mar_net, apr_net[world+1] AS apr_net, may_net[world+1] AS may_net, jun_net[world+1] AS jun_net, jul_net[world+1] AS jul_net, aug_net[world+1] AS aug_net, sep_net[world+1] AS sep_net, oct_net[world+1] AS oct_net, nov_net[world+1] AS nov_net, dec_net[world+1] AS dec_net +FROM stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), answers AS ( +select world, + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,ship_carriers + ,year_ + ,sum(jan_sales) as jan_sales + ,sum(feb_sales) as feb_sales + ,sum(mar_sales) as mar_sales + ,sum(apr_sales) as apr_sales + ,sum(may_sales) as may_sales + ,sum(jun_sales) as jun_sales + ,sum(jul_sales) as jul_sales + ,sum(aug_sales) as aug_sales + ,sum(sep_sales) as sep_sales + ,sum(oct_sales) as oct_sales + ,sum(nov_sales) as nov_sales + ,sum(dec_sales) as dec_sales + ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot + ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot + ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot + ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot + ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot + ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot + ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot + ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot + ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot + ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot + ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot + ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot + ,sum(jan_net) as jan_net + ,sum(feb_net) as feb_net + ,sum(mar_net) as mar_net + ,sum(apr_net) as apr_net + ,sum(may_net) as may_net + ,sum(jun_net) as jun_net + ,sum(jul_net) as jul_net + ,sum(aug_net) as aug_net + ,sum(sep_net) as sep_net + ,sum(oct_net) as oct_net + ,sum(nov_net) as nov_net + ,sum(dec_net) as dec_net + from expanded x + GROUP BY world, w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,ship_carriers + ,year_ + +) +SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY w_warehouse_name NULLS FIRST)<=100 ORDER BY world,w_warehouse_name NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q68.sql b/benchmark/tpcds/reference_queries/q68.sql new file mode 100644 index 00000000..cd722de8 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q68.sql @@ -0,0 +1,56 @@ +-- AS aggregation is shared; only the reduced per-ticket results are expanded by world. +WITH stats AS ( + SELECT ss_ticket_number, + ss_customer_sk, + ca_city AS bought_city, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS extended_price, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_list_price) AS list_price, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_tax) AS extended_tax, + tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS presence + FROM main.store_sales, + main.date_dim, + main.store, + main.household_demographics, + main.customer_address, + (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key + AND store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND store_sales.ss_addr_sk = customer_address.ca_address_sk + AND date_dim.d_dom BETWEEN 1 AND 2 + AND (household_demographics.hd_dep_count = 4 + OR household_demographics.hd_vehicle_count= 3) + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_city IN ('Fairview', + 'Midway') + GROUP BY ss_ticket_number, + ss_customer_sk, + ss_addr_sk, + ca_city +), expanded AS ( + SELECT world, ss_ticket_number, ss_customer_sk, bought_city, extended_price[world+1] AS extended_price, list_price[world+1] AS list_price, extended_tax[world+1] AS extended_tax + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1]>0 +) +SELECT world, c_last_name, + c_first_name, + ca_city, + bought_city, + ss_ticket_number, + extended_price, + extended_tax, + list_price +FROM + expanded dn, + main.customer, + main.customer_address current_addr +WHERE ss_customer_sk = c_customer_sk + AND customer.c_current_addr_sk = current_addr.ca_address_sk + AND current_addr.ca_city <> bought_city +QUALIFY row_number() OVER (PARTITION BY world ORDER BY c_last_name NULLS FIRST, + ss_ticket_number NULLS FIRST) <= 100 +ORDER BY world, c_last_name NULLS FIRST, + ss_ticket_number NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q69.sql b/benchmark/tpcds/reference_queries/q69.sql new file mode 100644 index 00000000..3d68f40d --- /dev/null +++ b/benchmark/tpcds/reference_queries/q69.sql @@ -0,0 +1,49 @@ +-- Same-owner EXISTS is static after entity inclusion; shipping-customer EXISTS needs the bill-owner mask. +WITH catalog_presence AS ( +SELECT cs_ship_customer_sk,bit_or(priv_hash(hash(cs_bill_customer_sk))) AS membership + FROM main.catalog_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy BETWEEN 4 AND 4+2 +GROUP BY cs_ship_customer_sk +), stats AS (SELECT cd_gender, +cd_marital_status, +cd_education_status, +cd_purchase_estimate, +cd_credit_rating, +tpcds_count((priv_hash(hash(c.c_customer_sk)) & ~coalesce(catalog_presence.membership,0::UBIGINT)),1) AS n +FROM main.customer c CROSS JOIN + main.customer_address ca CROSS JOIN + main.customer_demographics LEFT JOIN catalog_presence ON c.c_customer_sk=cs_ship_customer_sk +WHERE c.c_current_addr_sk = ca.ca_address_sk + AND ca_state IN ('KY', + 'GA', + 'NM') + AND cd_demo_sk = c.c_current_cdemo_sk + +AND EXISTS(SELECT * + FROM main.store_sales, + main.date_dim + WHERE c.c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy BETWEEN 4 AND 4+2) +AND NOT EXISTS(SELECT * + FROM main.web_sales, + main.date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy BETWEEN 4 AND 4+2) +GROUP BY cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating +), answers AS (SELECT world, cd_gender, cd_marital_status, cd_education_status, n[world+1] AS cnt1, cd_purchase_estimate, n[world+1] AS cnt2, cd_credit_rating, n[world+1] AS cnt3 FROM stats CROSS JOIN range(64) worlds(world) WHERE n[world+1]>0) +SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating)<=100 ORDER BY world,cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating; diff --git a/benchmark/tpcds/reference_queries/q72.sql b/benchmark/tpcds/reference_queries/q72.sql new file mode 100644 index 00000000..60a704ad --- /dev/null +++ b/benchmark/tpcds/reference_queries/q72.sql @@ -0,0 +1,20 @@ +-- Matched and NULL-extended memberships are disjoint; NULL extension is evaluated separately in each world. +WITH left_input AS ( +SELECT i_item_desc,w_warehouse_name,d1.d_week_seq,p_promo_sk,cs_order_number,cs_item_sk,priv_hash(hash(cs_bill_customer_sk)) AS membership +FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk +JOIN main.inventory ON cs_item_sk=inv_item_sk JOIN main.warehouse ON w_warehouse_sk=inv_warehouse_sk +JOIN main.item ON i_item_sk=cs_item_sk JOIN main.customer_demographics ON cs_bill_cdemo_sk=cd_demo_sk +JOIN main.household_demographics ON cs_bill_hdemo_sk=hd_demo_sk +JOIN main.date_dim d1 ON cs_sold_date_sk=d1.d_date_sk JOIN main.date_dim d2 ON inv_date_sk=d2.d_date_sk JOIN main.date_dim d3 ON cs_ship_date_sk=d3.d_date_sk +LEFT JOIN main.promotion ON cs_promo_sk=p_promo_sk +WHERE d1.d_week_seq=d2.d_week_seq AND inv_quantity_on_handd1.d_date+5 +AND hd_buy_potential='>10000' AND d1.d_year=1999 AND cd_marital_status='D' +), valid_returns AS (SELECT r.*,priv_hash(hash(cr_returning_customer_sk)) AS membership FROM main.catalog_returns r JOIN main.customer ON cr_returning_customer_sk=c_customer_sk), +return_presence AS (SELECT cr_order_number,cr_item_sk,bit_or(membership) AS membership FROM valid_returns GROUP BY cr_order_number,cr_item_sk), +joined AS ( + SELECT l.i_item_desc,l.w_warehouse_name,l.d_week_seq,l.p_promo_sk,(l.membership & r.membership) AS membership FROM left_input l JOIN valid_returns r ON l.cs_order_number=r.cr_order_number AND l.cs_item_sk=r.cr_item_sk + UNION ALL + SELECT l.i_item_desc,l.w_warehouse_name,l.d_week_seq,l.p_promo_sk,(l.membership & ~coalesce(r.membership,0::UBIGINT)) AS membership FROM left_input l LEFT JOIN return_presence r ON l.cs_order_number=r.cr_order_number AND l.cs_item_sk=r.cr_item_sk +), stats AS (SELECT i_item_desc,w_warehouse_name,d_week_seq,tpcds_sum(membership,CASE WHEN p_promo_sk IS NULL THEN 1 ELSE 0 END) AS no_promo,tpcds_sum(membership,CASE WHEN p_promo_sk IS NOT NULL THEN 1 ELSE 0 END) AS promo,tpcds_count(membership,1) AS total_cnt,tpcds_count(membership,1) AS presence FROM joined GROUP BY i_item_desc,w_warehouse_name,d_week_seq), +answers AS (SELECT world,i_item_desc,w_warehouse_name,d_week_seq,no_promo[world+1] AS no_promo,promo[world+1] AS promo,total_cnt[world+1] AS total_cnt FROM stats CROSS JOIN range(64) worlds(world) WHERE presence[world+1]>0) +SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY total_cnt DESC NULLS FIRST,i_item_desc NULLS FIRST,w_warehouse_name NULLS FIRST,d_week_seq NULLS FIRST)<=100 ORDER BY world,total_cnt DESC NULLS FIRST,i_item_desc NULLS FIRST,w_warehouse_name NULLS FIRST,d_week_seq NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q73.sql b/benchmark/tpcds/reference_queries/q73.sql new file mode 100644 index 00000000..b4e01e9b --- /dev/null +++ b/benchmark/tpcds/reference_queries/q73.sql @@ -0,0 +1,50 @@ +-- AS aggregation is shared; only the reduced per-ticket results are expanded by world. +WITH stats AS ( + SELECT ss_ticket_number, + ss_customer_sk, + tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS cnt, + tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS presence + FROM main.store_sales, + main.date_dim, + main.store, + main.household_demographics, + (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key + AND store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND date_dim.d_dom BETWEEN 1 AND 2 + AND (household_demographics.hd_buy_potential = 'Unknown' + OR household_demographics.hd_buy_potential = '>10000') + AND household_demographics.hd_vehicle_count > 0 + AND CASE + WHEN household_demographics.hd_vehicle_count > 0 THEN (household_demographics.hd_dep_count*1.000)/ household_demographics.hd_vehicle_count + ELSE NULL + END > 1 + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_county IN ('Orange County', + 'Bronx County', + 'Franklin Parish', + 'Williamson County') + GROUP BY ss_ticket_number, + ss_customer_sk +), expanded AS ( + SELECT world, ss_ticket_number, ss_customer_sk, cnt[world+1] AS cnt + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1]>0 +) +SELECT world, c_last_name, + c_first_name, + c_salutation, + c_preferred_cust_flag, + ss_ticket_number, + cnt +FROM + expanded dj, + main.customer +WHERE ss_customer_sk = c_customer_sk + AND cnt BETWEEN 1 AND 5 +ORDER BY world, cnt DESC, + c_last_name ASC; diff --git a/benchmark/tpcds/reference_queries/q74.sql b/benchmark/tpcds/reference_queries/q74.sql new file mode 100644 index 00000000..57335d54 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q74.sql @@ -0,0 +1,74 @@ +-- Shared AS states per channel/customer/year; joins align the world index. +WITH year_total_stats AS ( +SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + d_year AS year_, + tpcds_sum(priv_hash(hash(c_customer_sk)), ss_net_paid) year_total, + 's' sale_type, + tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence +FROM main.customer, + main.store_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year IN (2001, + 2001+1) + GROUP BY c_customer_id, + c_first_name, + c_last_name, + d_year + +UNION ALL + SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + d_year AS year_, + tpcds_sum(priv_hash(hash(c_customer_sk)), ws_net_paid) year_total, + 'w' sale_type, + tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence +FROM main.customer, + main.web_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year IN (2001, + 2001+1) + GROUP BY c_customer_id, + c_first_name, + c_last_name, + d_year +), year_total AS ( +SELECT world, customer_id, customer_first_name, customer_last_name, year_, year_total[world+1] AS year_total, sale_type +FROM year_total_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT t_s_firstyear.world AS world, t_s_secyear.customer_id, + t_s_secyear.customer_first_name, + t_s_secyear.customer_last_name +FROM year_total t_s_firstyear, + year_total t_s_secyear, + year_total t_w_firstyear, + year_total t_w_secyear +WHERE t_s_firstyear.world=t_s_secyear.world AND t_s_firstyear.world=t_w_firstyear.world AND t_s_firstyear.world=t_w_secyear.world AND t_s_secyear.customer_id = t_s_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_secyear.customer_id + AND t_s_firstyear.customer_id = t_w_firstyear.customer_id + AND t_s_firstyear.sale_type = 's' + AND t_w_firstyear.sale_type = 'w' + AND t_s_secyear.sale_type = 's' + AND t_w_secyear.sale_type = 'w' + AND t_s_firstyear.year_ = 2001 + AND t_s_secyear.year_ = 2001+1 + AND t_w_firstyear.year_ = 2001 + AND t_w_secyear.year_ = 2001+1 + AND t_s_firstyear.year_total > 0 + AND t_w_firstyear.year_total > 0 + AND CASE + WHEN t_w_firstyear.year_total > 0 THEN t_w_secyear.year_total / t_w_firstyear.year_total + ELSE NULL + END > CASE + WHEN t_s_firstyear.year_total > 0 THEN t_s_secyear.year_total / t_s_firstyear.year_total + ELSE NULL + END +QUALIFY row_number() OVER (PARTITION BY t_s_firstyear.world ORDER BY t_s_secyear.customer_id NULLS FIRST) <= 100 +ORDER BY t_s_firstyear.world, t_s_secyear.customer_id NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q77.sql b/benchmark/tpcds/reference_queries/q77.sql new file mode 100644 index 00000000..23ef4544 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q77.sql @@ -0,0 +1,114 @@ +-- Grouped channel results retain world-local LEFT JOIN semantics and empty-world ROLLUP totals. +WITH ss_stats AS ( +SELECT s_store_sk, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS sales, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) AS profit, + tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence +FROM main.store_sales, + main.date_dim, + main.store, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND ss_store_sk = s_store_sk + GROUP BY s_store_sk +), +ss AS ( +SELECT world, s_store_sk, sales[world+1] AS sales, profit[world+1] AS profit +FROM ss_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +sr_stats AS ( +SELECT s_store_sk, + tpcds_sum(priv_hash(hash(sr_customer_sk)), sr_return_amt) AS returns_, + tpcds_sum(priv_hash(hash(sr_customer_sk)), sr_net_loss) AS profit_loss, + tpcds_count(priv_hash(hash(sr_customer_sk)),1) AS presence +FROM main.store_returns, + main.date_dim, + main.store, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE sr_customer_sk=asa_owner.asa_key AND sr_returned_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND sr_store_sk = s_store_sk + GROUP BY s_store_sk +), +sr AS ( +SELECT world, s_store_sk, returns_[world+1] AS returns_, profit_loss[world+1] AS profit_loss +FROM sr_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +cs_stats AS ( +SELECT cs_call_center_sk, + tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) AS sales, + tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_net_profit) AS profit, + tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence +FROM main.catalog_sales, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + GROUP BY cs_call_center_sk +), +cs AS ( +SELECT world, cs_call_center_sk, sales[world+1] AS sales, profit[world+1] AS profit +FROM cs_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +cr_stats AS ( +SELECT cr_call_center_sk, + tpcds_sum(priv_hash(hash(cr_returning_customer_sk)), cr_return_amount) AS returns_, + tpcds_sum(priv_hash(hash(cr_returning_customer_sk)), cr_net_loss) AS profit_loss, + tpcds_count(priv_hash(hash(cr_returning_customer_sk)),1) AS presence +FROM main.catalog_returns, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cr_returning_customer_sk=asa_owner.asa_key AND cr_returned_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + GROUP BY cr_call_center_sk +), +cr AS ( +SELECT world, cr_call_center_sk, returns_[world+1] AS returns_, profit_loss[world+1] AS profit_loss +FROM cr_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +ws_stats AS ( +SELECT wp_web_page_sk, + tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) AS sales, + tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_net_profit) AS profit, + tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence +FROM main.web_sales, + main.date_dim, + main.web_page, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND ws_web_page_sk = wp_web_page_sk + GROUP BY wp_web_page_sk +), +ws AS ( +SELECT world, wp_web_page_sk, sales[world+1] AS sales, profit[world+1] AS profit +FROM ws_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +wr_stats AS ( +SELECT wp_web_page_sk, + tpcds_sum(priv_hash(hash(wr_returning_customer_sk)), wr_return_amt) AS returns_, + tpcds_sum(priv_hash(hash(wr_returning_customer_sk)), wr_net_loss) AS profit_loss, + tpcds_count(priv_hash(hash(wr_returning_customer_sk)),1) AS presence +FROM main.web_returns, + main.date_dim, + main.web_page, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE wr_returning_customer_sk=asa_owner.asa_key AND wr_returned_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND wr_web_page_sk = wp_web_page_sk + GROUP BY wp_web_page_sk +), +wr AS ( +SELECT world, wp_web_page_sk, returns_[world+1] AS returns_, profit_loss[world+1] AS profit_loss +FROM wr_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +channels AS ( +SELECT ss.world,'store channel' AS channel,ss.s_store_sk AS id,sales,coalesce(returns_,0) AS returns_,profit-coalesce(profit_loss,0) AS profit FROM ss LEFT JOIN sr ON ss.world=sr.world AND ss.s_store_sk=sr.s_store_sk +UNION ALL SELECT cs.world,'catalog channel',cs_call_center_sk,sales,returns_,profit-profit_loss FROM cs JOIN cr ON cs.world=cr.world +UNION ALL SELECT ws.world,'web channel',ws.wp_web_page_sk,sales,coalesce(returns_,0),profit-coalesce(profit_loss,0) FROM ws LEFT JOIN wr ON ws.world=wr.world AND ws.wp_web_page_sk=wr.wp_web_page_sk +), +answers AS ( +SELECT w.world,g.* FROM range(64) w(world) CROSS JOIN LATERAL (SELECT channel,id,sum(sales) AS sales,sum(returns_) AS returns_,sum(profit) AS profit FROM channels WHERE channels.world=w.world GROUP BY ROLLUP(channel,id)) g +) +SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY channel NULLS FIRST,id NULLS FIRST,returns_ DESC)<=100 ORDER BY world,channel NULLS FIRST,id NULLS FIRST,returns_ DESC; diff --git a/benchmark/tpcds/reference_queries/q78.sql b/benchmark/tpcds/reference_queries/q78.sql new file mode 100644 index 00000000..0efa3403 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q78.sql @@ -0,0 +1,80 @@ +-- Return anti-memberships feed the per-customer/item/year AS sums before channel joins. +WITH ws_return_presence AS (SELECT wr_order_number AS order_number,wr_item_sk AS item_sk,bit_or(priv_hash(hash(wr_returning_customer_sk))) AS membership FROM main.web_returns JOIN main.customer ON wr_returning_customer_sk=c_customer_sk GROUP BY wr_order_number,wr_item_sk), +ws_stats AS ( +SELECT d_year AS ws_sold_year,ws_item_sk,ws_bill_customer_sk AS ws_customer_sk, + tpcds_sum((priv_hash(hash(ws_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ws_quantity) AS ws_qty,tpcds_sum((priv_hash(hash(ws_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ws_wholesale_cost) AS ws_wc,tpcds_sum((priv_hash(hash(ws_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ws_sales_price) AS ws_sp,tpcds_count((priv_hash(hash(ws_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),1) AS presence +FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk JOIN main.date_dim ON ws_sold_date_sk=d_date_sk +LEFT JOIN ws_return_presence ret ON ws_order_number=ret.order_number AND ws_item_sk=ret.item_sk +GROUP BY d_year,ws_item_sk,ws_bill_customer_sk +), +ws AS ( +SELECT world, ws_sold_year, ws_item_sk, ws_customer_sk, ws_qty[world+1] AS ws_qty, ws_wc[world+1] AS ws_wc, ws_sp[world+1] AS ws_sp +FROM ws_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +cs_return_presence AS (SELECT cr_order_number AS order_number,cr_item_sk AS item_sk,bit_or(priv_hash(hash(cr_returning_customer_sk))) AS membership FROM main.catalog_returns JOIN main.customer ON cr_returning_customer_sk=c_customer_sk GROUP BY cr_order_number,cr_item_sk), +cs_stats AS ( +SELECT d_year AS cs_sold_year,cs_item_sk,cs_bill_customer_sk AS cs_customer_sk, + tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),cs_quantity) AS cs_qty,tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),cs_wholesale_cost) AS cs_wc,tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),cs_sales_price) AS cs_sp,tpcds_count((priv_hash(hash(cs_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),1) AS presence +FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk JOIN main.date_dim ON cs_sold_date_sk=d_date_sk +LEFT JOIN cs_return_presence ret ON cs_order_number=ret.order_number AND cs_item_sk=ret.item_sk +GROUP BY d_year,cs_item_sk,cs_bill_customer_sk +), +cs AS ( +SELECT world, cs_sold_year, cs_item_sk, cs_customer_sk, cs_qty[world+1] AS cs_qty, cs_wc[world+1] AS cs_wc, cs_sp[world+1] AS cs_sp +FROM cs_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +ss_return_presence AS (SELECT sr_ticket_number AS order_number,sr_item_sk AS item_sk,bit_or(priv_hash(hash(sr_customer_sk))) AS membership FROM main.store_returns JOIN main.customer ON sr_customer_sk=c_customer_sk GROUP BY sr_ticket_number,sr_item_sk), +ss_stats AS ( +SELECT d_year AS ss_sold_year,ss_item_sk,ss_customer_sk AS ss_customer_sk, + tpcds_sum((priv_hash(hash(ss_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ss_quantity) AS ss_qty,tpcds_sum((priv_hash(hash(ss_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ss_wholesale_cost) AS ss_wc,tpcds_sum((priv_hash(hash(ss_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ss_sales_price) AS ss_sp,tpcds_count((priv_hash(hash(ss_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),1) AS presence +FROM main.store_sales JOIN main.customer ON ss_customer_sk=c_customer_sk JOIN main.date_dim ON ss_sold_date_sk=d_date_sk +LEFT JOIN ss_return_presence ret ON ss_ticket_number=ret.order_number AND ss_item_sk=ret.item_sk +GROUP BY d_year,ss_item_sk,ss_customer_sk +), +ss AS ( +SELECT world, ss_sold_year, ss_item_sk, ss_customer_sk, ss_qty[world+1] AS ss_qty, ss_wc[world+1] AS ss_wc, ss_sp[world+1] AS ss_sp +FROM ss_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT ss.world AS world, ss_sold_year, + ss_item_sk, + ss_customer_sk, + round((ss_qty*1.00)/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, + ss_qty store_qty, + ss_wc store_wholesale_cost, + ss_sp store_sales_price, + coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, + coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, + coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price +FROM ss +LEFT JOIN ws ON (ss.world=ws.world AND ws_sold_year=ss_sold_year + AND ws_item_sk=ss_item_sk + AND ws_customer_sk=ss_customer_sk) +LEFT JOIN cs ON (ss.world=cs.world AND cs_sold_year=ss_sold_year + AND cs_item_sk=ss_item_sk + AND cs_customer_sk=ss_customer_sk) +WHERE (coalesce(ws_qty,0)>0 + OR coalesce(cs_qty, 0)>0) + AND ss_sold_year=2000 +QUALIFY row_number() OVER (PARTITION BY ss.world ORDER BY ss_sold_year, + ss_item_sk, + ss_customer_sk, + ss_qty DESC, + ss_wc DESC, + ss_sp DESC, + other_chan_qty, + other_chan_wholesale_cost, + other_chan_sales_price, + ratio)<=100 +ORDER BY ss.world,ss_sold_year, + ss_item_sk, + ss_customer_sk, + ss_qty DESC, + ss_wc DESC, + ss_sp DESC, + other_chan_qty, + other_chan_wholesale_cost, + other_chan_sales_price, + ratio; diff --git a/benchmark/tpcds/reference_queries/q79.sql b/benchmark/tpcds/reference_queries/q79.sql new file mode 100644 index 00000000..a22b04be --- /dev/null +++ b/benchmark/tpcds/reference_queries/q79.sql @@ -0,0 +1,53 @@ +-- AS aggregation is shared; only the reduced per-ticket results are expanded by world. +WITH stats AS ( + SELECT ss_ticket_number, + ss_customer_sk, + store.s_city AS s_city, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) AS amt, + tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) AS profit, + tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS presence + FROM main.store_sales, + main.date_dim, + main.store, + main.household_demographics, + (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key + AND store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND (household_demographics.hd_dep_count = 6 + OR household_demographics.hd_vehicle_count > 2) + AND date_dim.d_dow = 1 + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_number_employees BETWEEN 200 AND 295 + GROUP BY ss_ticket_number, + ss_customer_sk, + ss_addr_sk, + main.store.s_city +), expanded AS ( + SELECT world, ss_ticket_number, ss_customer_sk, s_city, amt[world+1] AS amt, profit[world+1] AS profit + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1]>0 +) +SELECT world, c_last_name, + c_first_name, + SUBSTRING(s_city,1,30), + ss_ticket_number, + amt, + profit +FROM + expanded ms, + main.customer +WHERE ss_customer_sk = c_customer_sk +QUALIFY row_number() OVER (PARTITION BY world ORDER BY c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + SUBSTRING(s_city,1,30) NULLS FIRST, + profit NULLS FIRST, + ss_ticket_number) <= 100 +ORDER BY world, c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + SUBSTRING(s_city,1,30) NULLS FIRST, + profit NULLS FIRST, + ss_ticket_number; diff --git a/benchmark/tpcds/reference_queries/q80.sql b/benchmark/tpcds/reference_queries/q80.sql new file mode 100644 index 00000000..5fdeacf0 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q80.sql @@ -0,0 +1,58 @@ +-- Matched/fallback row masks preserve sampled outer joins before each channel's AS aggregates. +WITH ssr_left AS ( +SELECT s_store_id AS store_id,ss_ticket_number AS order_number,ss_item_sk AS item_sk,ss_ext_sales_price AS sales_price,ss_net_profit AS net_profit,priv_hash(hash(ss_customer_sk)) AS membership +FROM main.store_sales JOIN main.customer ON ss_customer_sk=c_customer_sk JOIN main.date_dim ON ss_sold_date_sk=d_date_sk +JOIN main.store ON ss_store_sk=s_store_sk JOIN main.item ON ss_item_sk=i_item_sk JOIN main.promotion ON ss_promo_sk=p_promo_sk +WHERE d_date BETWEEN DATE '2000-08-23' AND DATE '2000-09-22' AND i_current_price>50 AND p_channel_tv='N' +), +ssr_returns AS (SELECT sr_ticket_number AS order_number,sr_item_sk AS item_sk,sr_return_amt AS amount,sr_net_loss AS loss,priv_hash(hash(sr_customer_sk)) AS membership FROM main.store_returns JOIN main.customer ON sr_customer_sk=c_customer_sk), +ssr_return_presence AS (SELECT order_number,item_sk,bit_or(membership) AS membership FROM ssr_returns GROUP BY order_number,item_sk), +ssr_joined AS ( +SELECT l.store_id,l.sales_price,l.net_profit,r.amount,r.loss,(l.membership & r.membership) AS membership FROM ssr_left l JOIN ssr_returns r ON l.order_number=r.order_number AND l.item_sk=r.item_sk +UNION ALL SELECT l.store_id,l.sales_price,l.net_profit,0 AS amount,0 AS loss,(l.membership & ~coalesce(r.membership,0::UBIGINT)) AS membership FROM ssr_left l LEFT JOIN ssr_return_presence r ON l.order_number=r.order_number AND l.item_sk=r.item_sk +), +ssr_stats AS (SELECT store_id,tpcds_sum(membership,sales_price) AS sales,tpcds_sum(membership,coalesce(amount,0)) AS returns_,tpcds_sum(membership,net_profit-coalesce(loss,0)) AS profit,tpcds_count(membership,1) AS presence FROM ssr_joined GROUP BY store_id), +ssr AS ( +SELECT world, store_id, sales[world+1] AS sales, returns_[world+1] AS returns_, profit[world+1] AS profit +FROM ssr_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +csr_left AS ( +SELECT cp_catalog_page_id AS catalog_page_id,cs_order_number AS order_number,cs_item_sk AS item_sk,cs_ext_sales_price AS sales_price,cs_net_profit AS net_profit,priv_hash(hash(cs_bill_customer_sk)) AS membership +FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk JOIN main.date_dim ON cs_sold_date_sk=d_date_sk +JOIN main.catalog_page ON cs_catalog_page_sk=cp_catalog_page_sk JOIN main.item ON cs_item_sk=i_item_sk JOIN main.promotion ON cs_promo_sk=p_promo_sk +WHERE d_date BETWEEN DATE '2000-08-23' AND DATE '2000-09-22' AND i_current_price>50 AND p_channel_tv='N' +), +csr_returns AS (SELECT cr_order_number AS order_number,cr_item_sk AS item_sk,cr_return_amount AS amount,cr_net_loss AS loss,priv_hash(hash(cr_returning_customer_sk)) AS membership FROM main.catalog_returns JOIN main.customer ON cr_returning_customer_sk=c_customer_sk), +csr_return_presence AS (SELECT order_number,item_sk,bit_or(membership) AS membership FROM csr_returns GROUP BY order_number,item_sk), +csr_joined AS ( +SELECT l.catalog_page_id,l.sales_price,l.net_profit,r.amount,r.loss,(l.membership & r.membership) AS membership FROM csr_left l JOIN csr_returns r ON l.order_number=r.order_number AND l.item_sk=r.item_sk +UNION ALL SELECT l.catalog_page_id,l.sales_price,l.net_profit,0 AS amount,0 AS loss,(l.membership & ~coalesce(r.membership,0::UBIGINT)) AS membership FROM csr_left l LEFT JOIN csr_return_presence r ON l.order_number=r.order_number AND l.item_sk=r.item_sk +), +csr_stats AS (SELECT catalog_page_id,tpcds_sum(membership,sales_price) AS sales,tpcds_sum(membership,coalesce(amount,0)) AS returns_,tpcds_sum(membership,net_profit-coalesce(loss,0)) AS profit,tpcds_count(membership,1) AS presence FROM csr_joined GROUP BY catalog_page_id), +csr AS ( +SELECT world, catalog_page_id, sales[world+1] AS sales, returns_[world+1] AS returns_, profit[world+1] AS profit +FROM csr_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +wsr_left AS ( +SELECT web_site_id AS web_site_id,ws_order_number AS order_number,ws_item_sk AS item_sk,ws_ext_sales_price AS sales_price,ws_net_profit AS net_profit,priv_hash(hash(ws_bill_customer_sk)) AS membership +FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk JOIN main.date_dim ON ws_sold_date_sk=d_date_sk +JOIN main.web_site ON ws_web_site_sk=web_site_sk JOIN main.item ON ws_item_sk=i_item_sk JOIN main.promotion ON ws_promo_sk=p_promo_sk +WHERE d_date BETWEEN DATE '2000-08-23' AND DATE '2000-09-22' AND i_current_price>50 AND p_channel_tv='N' +), +wsr_returns AS (SELECT wr_order_number AS order_number,wr_item_sk AS item_sk,wr_return_amt AS amount,wr_net_loss AS loss,priv_hash(hash(wr_returning_customer_sk)) AS membership FROM main.web_returns JOIN main.customer ON wr_returning_customer_sk=c_customer_sk), +wsr_return_presence AS (SELECT order_number,item_sk,bit_or(membership) AS membership FROM wsr_returns GROUP BY order_number,item_sk), +wsr_joined AS ( +SELECT l.web_site_id,l.sales_price,l.net_profit,r.amount,r.loss,(l.membership & r.membership) AS membership FROM wsr_left l JOIN wsr_returns r ON l.order_number=r.order_number AND l.item_sk=r.item_sk +UNION ALL SELECT l.web_site_id,l.sales_price,l.net_profit,0 AS amount,0 AS loss,(l.membership & ~coalesce(r.membership,0::UBIGINT)) AS membership FROM wsr_left l LEFT JOIN wsr_return_presence r ON l.order_number=r.order_number AND l.item_sk=r.item_sk +), +wsr_stats AS (SELECT web_site_id,tpcds_sum(membership,sales_price) AS sales,tpcds_sum(membership,coalesce(amount,0)) AS returns_,tpcds_sum(membership,net_profit-coalesce(loss,0)) AS profit,tpcds_count(membership,1) AS presence FROM wsr_joined GROUP BY web_site_id), +wsr AS ( +SELECT world, web_site_id, sales[world+1] AS sales, returns_[world+1] AS returns_, profit[world+1] AS profit +FROM wsr_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +channels AS (SELECT world,'store channel' AS channel,concat('store',store_id) AS id,sales,returns_,profit FROM ssr UNION ALL SELECT world,'catalog channel',concat('catalog_page',catalog_page_id),sales,returns_,profit FROM csr UNION ALL SELECT world,'web channel',concat('web_site',web_site_id),sales,returns_,profit FROM wsr), +answers AS (SELECT w.world,g.* FROM range(64) w(world) CROSS JOIN LATERAL (SELECT channel,id,sum(sales) AS sales,sum(returns_) AS returns_,sum(profit) AS profit FROM channels WHERE channels.world=w.world GROUP BY ROLLUP(channel,id)) g) +SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY channel NULLS FIRST,id NULLS FIRST)<=100 ORDER BY world,channel NULLS FIRST,id NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q81.sql b/benchmark/tpcds/reference_queries/q81.sql new file mode 100644 index 00000000..ffe67f71 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q81.sql @@ -0,0 +1,77 @@ +-- Correlated thresholds use only the matching world of the reduced return aggregates. +WITH return_stats AS ( +SELECT cr_returning_customer_sk AS ctr_customer_sk , + ca_state AS ctr_state, + tpcds_sum(priv_hash(hash(cr_returning_customer_sk)), cr_return_amt_inc_tax) AS ctr_total_return, + tpcds_count(priv_hash(hash(cr_returning_customer_sk)),1) AS presence +FROM main.catalog_returns , + main.date_dim , + main.customer_address, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cr_returning_customer_sk=asa_owner.asa_key AND cr_returned_date_sk = d_date_sk + AND d_year = 2000 + AND cr_returning_addr_sk = ca_address_sk + GROUP BY cr_returning_customer_sk , + ca_state +), customer_total_return AS ( +SELECT world, ctr_customer_sk, ctr_state, ctr_total_return[world+1] AS ctr_total_return +FROM return_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT ctr1.world AS world, c_customer_id, + c_salutation, + c_first_name, + c_last_name, + ca_street_number, + ca_street_name , + ca_street_type, + ca_suite_number, + ca_city, + ca_county, + ca_state, + ca_zip, + ca_country, + ca_gmt_offset , + ca_location_type, + ctr_total_return +FROM customer_total_return ctr1 , + main.customer_address , + main.customer +WHERE ctr1.ctr_total_return > + (SELECT avg(ctr_total_return)*1.2 + FROM customer_total_return ctr2 + WHERE ctr1.world=ctr2.world AND ctr1.ctr_state = ctr2.ctr_state) + AND ca_address_sk = c_current_addr_sk + AND ca_state = 'GA' + AND ctr1.ctr_customer_sk = c_customer_sk +QUALIFY row_number() OVER (PARTITION BY ctr1.world ORDER BY c_customer_id, + c_salutation, + c_first_name, + c_last_name, + ca_street_number, + ca_street_name , + ca_street_type, + ca_suite_number, + ca_city, + ca_county, + ca_state, + ca_zip, + ca_country, + ca_gmt_offset , + ca_location_type, + ctr_total_return) <= 100 +ORDER BY ctr1.world, c_customer_id, + c_salutation, + c_first_name, + c_last_name, + ca_street_number, + ca_street_name , + ca_street_type, + ca_suite_number, + ca_city, + ca_county, + ca_state, + ca_zip, + ca_country, + ca_gmt_offset , + ca_location_type, + ctr_total_return; diff --git a/benchmark/tpcds/reference_queries/q83.sql b/benchmark/tpcds/reference_queries/q83.sql new file mode 100644 index 00000000..ba388e54 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q83.sql @@ -0,0 +1,93 @@ +-- Each return channel is aggregated once, then joined on item and world. +WITH sr_items_stats AS ( +SELECT i_item_id item_id, + tpcds_sum(priv_hash(hash(sr_customer_sk)), sr_return_quantity) sr_item_qty, + tpcds_count(priv_hash(hash(sr_customer_sk)),1) AS presence +FROM main.store_returns, + main.item, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE sr_customer_sk=asa_owner.asa_key AND sr_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM main.date_dim + WHERE d_week_seq IN + (SELECT d_week_seq + FROM main.date_dim + WHERE d_date IN ('2000-06-30', + '2000-09-27', + '2000-11-17'))) + AND sr_returned_date_sk = d_date_sk + GROUP BY i_item_id +), +sr_items AS ( +SELECT world, item_id, sr_item_qty[world+1] AS sr_item_qty +FROM sr_items_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +cr_items_stats AS ( +SELECT i_item_id item_id, + tpcds_sum(priv_hash(hash(cr_returning_customer_sk)), cr_return_quantity) cr_item_qty, + tpcds_count(priv_hash(hash(cr_returning_customer_sk)),1) AS presence +FROM main.catalog_returns, + main.item, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE cr_returning_customer_sk=asa_owner.asa_key AND cr_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM main.date_dim + WHERE d_week_seq IN + (SELECT d_week_seq + FROM main.date_dim + WHERE d_date IN ('2000-06-30', + '2000-09-27', + '2000-11-17'))) + AND cr_returned_date_sk = d_date_sk + GROUP BY i_item_id +), +cr_items AS ( +SELECT world, item_id, cr_item_qty[world+1] AS cr_item_qty +FROM cr_items_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +), +wr_items_stats AS ( +SELECT i_item_id item_id, + tpcds_sum(priv_hash(hash(wr_returning_customer_sk)), wr_return_quantity) wr_item_qty, + tpcds_count(priv_hash(hash(wr_returning_customer_sk)),1) AS presence +FROM main.web_returns, + main.item, + main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE wr_returning_customer_sk=asa_owner.asa_key AND wr_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM main.date_dim + WHERE d_week_seq IN + (SELECT d_week_seq + FROM main.date_dim + WHERE d_date IN ('2000-06-30', + '2000-09-27', + '2000-11-17'))) + AND wr_returned_date_sk = d_date_sk + GROUP BY i_item_id +), +wr_items AS ( +SELECT world, item_id, wr_item_qty[world+1] AS wr_item_qty +FROM wr_items_stats CROSS JOIN range(64) worlds(world) +WHERE presence[world+1]>0 +) +SELECT sr_items.world AS world, sr_items.item_id , + sr_item_qty , + (sr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 sr_dev , + cr_item_qty , + (cr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 cr_dev , + wr_item_qty , + (wr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 wr_dev , + (sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average +FROM sr_items , + cr_items , + wr_items +WHERE sr_items.world=cr_items.world AND sr_items.world=wr_items.world AND sr_items.item_id=cr_items.item_id + AND sr_items.item_id=wr_items.item_id +QUALIFY row_number() OVER (PARTITION BY sr_items.world ORDER BY sr_items.item_id NULLS FIRST, + sr_item_qty NULLS FIRST) <= 100 +ORDER BY sr_items.world, sr_items.item_id NULLS FIRST, + sr_item_qty NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q85.sql b/benchmark/tpcds/reference_queries/q85.sql new file mode 100644 index 00000000..746882d7 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q85.sql @@ -0,0 +1,68 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT SUBSTRING(r_reason_desc,1,20) AS reason, + tpcds_avg((priv_hash(hash(ws_bill_customer_sk)) & priv_hash(hash(wr_returning_customer_sk))), ws_quantity) AS avg1, + tpcds_avg((priv_hash(hash(ws_bill_customer_sk)) & priv_hash(hash(wr_returning_customer_sk))), wr_refunded_cash) AS avg2, + tpcds_avg((priv_hash(hash(ws_bill_customer_sk)) & priv_hash(hash(wr_returning_customer_sk))), wr_fee) AS avg3, + tpcds_count((priv_hash(hash(ws_bill_customer_sk)) & priv_hash(hash(wr_returning_customer_sk))), 1) AS presence + FROM main.web_sales, + main.web_returns, + main.web_page, + main.customer_demographics cd1, + main.customer_demographics cd2, + main.customer_address, + main.date_dim, + main.reason, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0, (SELECT c_customer_sk AS asa_key1 FROM main.customer) asa_owner1 +WHERE ws_bill_customer_sk = asa_owner0.asa_key0 AND wr_returning_customer_sk = asa_owner1.asa_key1 + AND ws_web_page_sk = wp_web_page_sk + AND ws_item_sk = wr_item_sk + AND ws_order_number = wr_order_number + AND ws_sold_date_sk = d_date_sk + AND d_year = 2000 + AND cd1.cd_demo_sk = wr_refunded_cdemo_sk + AND cd2.cd_demo_sk = wr_returning_cdemo_sk + AND ca_address_sk = wr_refunded_addr_sk + AND r_reason_sk = wr_reason_sk + AND ( ( cd1.cd_marital_status = 'M' + AND cd1.cd_marital_status = cd2.cd_marital_status + AND cd1.cd_education_status = 'Advanced Degree' + AND cd1.cd_education_status = cd2.cd_education_status + AND ws_sales_price BETWEEN 100.00 AND 150.00 ) + OR ( cd1.cd_marital_status = 'S' + AND cd1.cd_marital_status = cd2.cd_marital_status + AND cd1.cd_education_status = 'College' + AND cd1.cd_education_status = cd2.cd_education_status + AND ws_sales_price BETWEEN 50.00 AND 100.00 ) + OR ( cd1.cd_marital_status = 'W' + AND cd1.cd_marital_status = cd2.cd_marital_status + AND cd1.cd_education_status = '2 yr Degree' + AND cd1.cd_education_status = cd2.cd_education_status + AND ws_sales_price BETWEEN 150.00 AND 200.00 ) ) + AND ( ( ca_country = 'United States' + AND ca_state IN ('IN', + 'OH', + 'NJ') + AND ws_net_profit BETWEEN 100 AND 200) + OR ( ca_country = 'United States' + AND ca_state IN ('WI', + 'CT', + 'KY') + AND ws_net_profit BETWEEN 150 AND 300) + OR ( ca_country = 'United States' + AND ca_state IN ('LA', + 'IA', + 'AR') + AND ws_net_profit BETWEEN 50 AND 250) ) +GROUP BY r_reason_desc +), answers AS ( + SELECT world, reason, + avg1[world+1] AS avg1, + avg2[world+1] AS avg2, + avg3[world+1] AS avg3 + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY reason, avg1, avg2, avg3) <= 100 +ORDER BY world, reason, avg1, avg2, avg3; diff --git a/benchmark/tpcds/reference_queries/q88.sql b/benchmark/tpcds/reference_queries/q88.sql new file mode 100644 index 00000000..4cd5e384 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q88.sql @@ -0,0 +1,25 @@ +-- All time-bucket counters share the fact/dimension joins. +WITH stats AS (SELECT tpcds_count(CASE WHEN t_hour=8 AND t_minute >= 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h8_30_to_9, +tpcds_count(CASE WHEN t_hour=9 AND t_minute < 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h9_to_9_30, +tpcds_count(CASE WHEN t_hour=9 AND t_minute >= 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h9_30_to_10, +tpcds_count(CASE WHEN t_hour=10 AND t_minute < 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h10_to_10_30, +tpcds_count(CASE WHEN t_hour=10 AND t_minute >= 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h10_30_to_11, +tpcds_count(CASE WHEN t_hour=11 AND t_minute < 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h11_to_11_30, +tpcds_count(CASE WHEN t_hour=11 AND t_minute >= 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h11_30_to_12, +tpcds_count(CASE WHEN t_hour=12 AND t_minute < 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h12_to_12_30 +FROM main.store_sales, + main.household_demographics, + main.time_dim, + main.store, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese' +) +SELECT world,h8_30_to_9[world+1] AS h8_30_to_9, h9_to_9_30[world+1] AS h9_to_9_30, h9_30_to_10[world+1] AS h9_30_to_10, h10_to_10_30[world+1] AS h10_to_10_30, h10_30_to_11[world+1] AS h10_30_to_11, h11_to_11_30[world+1] AS h11_to_11_30, h11_30_to_12[world+1] AS h11_30_to_12, h12_to_12_30[world+1] AS h12_to_12_30 FROM stats CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q90.sql b/benchmark/tpcds/reference_queries/q90.sql new file mode 100644 index 00000000..6e279282 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q90.sql @@ -0,0 +1,30 @@ +-- Scalar AS arrays are combined only after their independent shared scans. +WITH raw0 AS ( +SELECT tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) amc + FROM main.web_sales, + main.household_demographics, + main.time_dim, + main.web_page, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_sold_time_sk = time_dim.t_time_sk + AND ws_ship_hdemo_sk = household_demographics.hd_demo_sk + AND ws_web_page_sk = web_page.wp_web_page_sk + AND time_dim.t_hour BETWEEN 8 AND 8+1 + AND household_demographics.hd_dep_count = 6 + AND web_page.wp_char_count BETWEEN 5000 AND 5200 +), +raw1 AS ( +SELECT tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) pmc + FROM main.web_sales, + main.household_demographics, + main.time_dim, + main.web_page, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner +WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_sold_time_sk = time_dim.t_time_sk + AND ws_ship_hdemo_sk = household_demographics.hd_demo_sk + AND ws_web_page_sk = web_page.wp_web_page_sk + AND time_dim.t_hour BETWEEN 19 AND 19+1 + AND household_demographics.hd_dep_count = 6 + AND web_page.wp_char_count BETWEEN 5000 AND 5200 +), +p AS (SELECT world, raw0.amc[world+1] AS amc, raw1.pmc[world+1] AS pmc FROM raw0 CROSS JOIN raw1 CROSS JOIN range(64) worlds(world)) +SELECT world, case when p.pmc=0 then null else cast(p.amc AS decimal(15,4))/cast(p.pmc AS decimal(15,4)) end am_pm_ratio + FROM p ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q91.sql b/benchmark/tpcds/reference_queries/q91.sql new file mode 100644 index 00000000..d3c9ad21 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q91.sql @@ -0,0 +1,44 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT cc_call_center_id AS call_center, + cc_name AS call_center_name, + cc_manager AS manager, + tpcds_sum((priv_hash(hash(cr_returning_customer_sk))), cr_net_loss) AS returns_loss, + tpcds_count((priv_hash(hash(cr_returning_customer_sk))), 1) AS presence + FROM main.call_center, + main.catalog_returns, + main.date_dim, + main.customer, + main.customer_address, + main.customer_demographics, + main.household_demographics, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE cr_returning_customer_sk = asa_owner0.asa_key0 + AND cr_call_center_sk = cc_call_center_sk + AND cr_returned_date_sk = d_date_sk + AND cr_returning_customer_sk= c_customer_sk + AND cd_demo_sk = c_current_cdemo_sk + AND hd_demo_sk = c_current_hdemo_sk + AND ca_address_sk = c_current_addr_sk + AND d_year = 1998 + AND d_moy = 11 + AND ((cd_marital_status = 'M' + AND cd_education_status = 'Unknown') or(cd_marital_status = 'W' + AND cd_education_status = 'Advanced Degree')) + AND hd_buy_potential LIKE 'Unknown%' + AND ca_gmt_offset = -7 +GROUP BY cc_call_center_id, + cc_name, + cc_manager, + cd_marital_status, + cd_education_status +), answers AS ( + SELECT world, call_center, + call_center_name, + manager, + returns_loss[world+1] AS returns_loss + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +ORDER BY world, returns_loss DESC; diff --git a/benchmark/tpcds/reference_queries/q92.sql b/benchmark/tpcds/reference_queries/q92.sql new file mode 100644 index 00000000..f8fc761b --- /dev/null +++ b/benchmark/tpcds/reference_queries/q92.sql @@ -0,0 +1,14 @@ +-- Correlated aggregate predicates update the mask before the outer AS SUM. +WITH thresholds AS ( + SELECT ws_item_sk, tpcds_avg(priv_hash(hash(ws_bill_customer_sk)),ws_ext_discount_amt) AS averages + FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk JOIN main.date_dim ON ws_sold_date_sk=d_date_sk + WHERE d_date BETWEEN DATE '2000-01-27' AND DATE '2000-04-26' GROUP BY ws_item_sk +), filtered AS ( + SELECT s.ws_ext_discount_amt AS discount, + priv_select(priv_hash(hash(ws_bill_customer_sk)),list_transform(averages, lambda average: s.ws_ext_discount_amt>1.3*average)) AS membership + FROM main.web_sales s JOIN main.customer ON ws_bill_customer_sk=c_customer_sk + JOIN main.item ON s.ws_item_sk=i_item_sk JOIN main.date_dim ON s.ws_sold_date_sk=d_date_sk + JOIN thresholds t ON s.ws_item_sk=t.ws_item_sk + WHERE i_manufact_id=350 AND d_date BETWEEN DATE '2000-01-27' AND DATE '2000-04-26' +), stats AS (SELECT tpcds_sum(membership,discount) AS total FROM filtered) +SELECT world,total[world+1] AS excess_discount_amount FROM stats CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q93.sql b/benchmark/tpcds/reference_queries/q93.sql new file mode 100644 index 00000000..3a14d36b --- /dev/null +++ b/benchmark/tpcds/reference_queries/q93.sql @@ -0,0 +1,12 @@ +-- The reason predicate rejects unmatched returns; both owners must be present. +WITH input AS ( + SELECT ss_customer_sk,CASE WHEN sr_return_quantity IS NOT NULL THEN (ss_quantity-sr_return_quantity)*ss_sales_price ELSE ss_quantity*ss_sales_price END AS act_sales, + (priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk))) AS membership + FROM main.store_sales JOIN main.store_returns ON sr_item_sk=ss_item_sk AND sr_ticket_number=ss_ticket_number + JOIN main.reason ON sr_reason_sk=r_reason_sk + JOIN main.customer sale_owner ON ss_customer_sk=sale_owner.c_customer_sk + JOIN main.customer return_owner ON sr_customer_sk=return_owner.c_customer_sk + WHERE r_reason_desc='reason 28' +), stats AS (SELECT ss_customer_sk,tpcds_sum(membership,act_sales) AS sumsales,tpcds_count(membership,1) AS presence FROM input GROUP BY ss_customer_sk), +answers AS (SELECT world,ss_customer_sk,sumsales[world+1] AS sumsales FROM stats CROSS JOIN range(64) worlds(world) WHERE presence[world+1]>0) +SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY sumsales NULLS FIRST,ss_customer_sk NULLS FIRST)<=100 ORDER BY world,sumsales NULLS FIRST,ss_customer_sk NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q94.sql b/benchmark/tpcds/reference_queries/q94.sql new file mode 100644 index 00000000..708051dc --- /dev/null +++ b/benchmark/tpcds/reference_queries/q94.sql @@ -0,0 +1,10 @@ +-- Order-level existence masks are combined before AS; DISTINCT unions memberships by order. +WITH warehouses AS (SELECT ws_order_number AS order_number,ws_warehouse_sk AS warehouse,bit_or(priv_hash(hash(ws_bill_customer_sk))) AS membership FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk GROUP BY ws_order_number,ws_warehouse_sk), +other_warehouses AS (SELECT a.order_number,a.warehouse,bit_or(b.membership) AS membership FROM warehouses a JOIN warehouses b ON a.order_number=b.order_number AND a.warehouse<>b.warehouse GROUP BY a.order_number,a.warehouse), +return_orders AS (SELECT wr_order_number AS order_number,bit_or(priv_hash(hash(wr_returning_customer_sk))) AS membership FROM main.web_returns JOIN main.customer ON wr_returning_customer_sk=c_customer_sk GROUP BY wr_order_number), +input AS (SELECT ws_order_number AS order_number,ws_ext_ship_cost AS shipping,ws_net_profit AS profit,(priv_hash(hash(ws_bill_customer_sk)) & other.membership & ~coalesce(ret.membership,0::UBIGINT)) AS membership +FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk JOIN main.date_dim ON ws_ship_date_sk=d_date_sk JOIN main.customer_address ON ws_ship_addr_sk=ca_address_sk JOIN main.web_site ON ws_web_site_sk=web_site_sk JOIN other_warehouses other ON ws_order_number=other.order_number AND ws_warehouse_sk=other.warehouse LEFT JOIN return_orders ret ON ws_order_number=ret.order_number WHERE d_date BETWEEN DATE '1999-02-01' AND DATE '1999-04-02' AND ca_state='IL' AND web_company_name='pri'), +totals AS (SELECT tpcds_sum(membership,shipping) AS shipping,tpcds_sum(membership,profit) AS profit FROM input), +unique_orders AS (SELECT order_number,bit_or(membership) AS membership FROM input GROUP BY order_number), +counts AS (SELECT tpcds_count(membership,1) AS n FROM unique_orders) +SELECT world,n[world+1] AS order_count,shipping[world+1] AS total_shipping_cost,profit[world+1] AS total_net_profit FROM totals CROSS JOIN counts CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q95.sql b/benchmark/tpcds/reference_queries/q95.sql new file mode 100644 index 00000000..7eb87985 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q95.sql @@ -0,0 +1,10 @@ +-- Order-level existence masks are combined before AS; DISTINCT unions memberships by order. +WITH warehouses AS (SELECT ws_order_number AS order_number,ws_warehouse_sk AS warehouse,bit_or(priv_hash(hash(ws_bill_customer_sk))) AS membership FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk GROUP BY ws_order_number,ws_warehouse_sk), +pair_orders AS (SELECT a.order_number,bit_or(a.membership & b.membership) AS membership FROM warehouses a JOIN warehouses b ON a.order_number=b.order_number AND a.warehouse<>b.warehouse GROUP BY a.order_number), +return_orders AS (SELECT wr_order_number AS order_number,bit_or(priv_hash(hash(wr_returning_customer_sk))) AS membership FROM main.web_returns JOIN main.customer ON wr_returning_customer_sk=c_customer_sk GROUP BY wr_order_number), +input AS (SELECT ws_order_number AS order_number,ws_ext_ship_cost AS shipping,ws_net_profit AS profit,(priv_hash(hash(ws_bill_customer_sk)) & other.membership & coalesce(ret.membership,0::UBIGINT)) AS membership +FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk JOIN main.date_dim ON ws_ship_date_sk=d_date_sk JOIN main.customer_address ON ws_ship_addr_sk=ca_address_sk JOIN main.web_site ON ws_web_site_sk=web_site_sk JOIN pair_orders other ON ws_order_number=other.order_number LEFT JOIN return_orders ret ON ws_order_number=ret.order_number WHERE d_date BETWEEN DATE '1999-02-01' AND DATE '1999-04-02' AND ca_state='IL' AND web_company_name='pri'), +totals AS (SELECT tpcds_sum(membership,shipping) AS shipping,tpcds_sum(membership,profit) AS profit FROM input), +unique_orders AS (SELECT order_number,bit_or(membership) AS membership FROM input GROUP BY order_number), +counts AS (SELECT tpcds_count(membership,1) AS n FROM unique_orders) +SELECT world,n[world+1] AS order_count,shipping[world+1] AS total_shipping_cost,profit[world+1] AS total_net_profit FROM totals CROSS JOIN counts CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q96.sql b/benchmark/tpcds/reference_queries/q96.sql new file mode 100644 index 00000000..f14f19b0 --- /dev/null +++ b/benchmark/tpcds/reference_queries/q96.sql @@ -0,0 +1,24 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS total, + tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence + FROM main.store_sales , + main.household_demographics, + main.time_dim, + main.store, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE ss_customer_sk = asa_owner0.asa_key0 + AND ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 20 + AND time_dim.t_minute >= 30 + AND household_demographics.hd_dep_count = 7 + AND store.s_store_name = 'ese' +), answers AS ( + SELECT world, total[world+1] AS total + FROM stats CROSS JOIN range(64) worlds(world) +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY total) <= 100 +ORDER BY world, total; diff --git a/benchmark/tpcds/reference_queries/q99.sql b/benchmark/tpcds/reference_queries/q99.sql new file mode 100644 index 00000000..2366503a --- /dev/null +++ b/benchmark/tpcds/reference_queries/q99.sql @@ -0,0 +1,42 @@ +-- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. +WITH stats AS ( + SELECT w_substr, + sm_type, + LOWER(cc_name) AS cc_name_lower, + tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), CASE WHEN cs_ship_date_sk - cs_sold_date_sk <= 30 THEN 1 ELSE 0 END) AS bucket0, + tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), CASE WHEN cs_ship_date_sk - cs_sold_date_sk > 30 AND cs_ship_date_sk - cs_sold_date_sk <= 60 THEN 1 ELSE 0 END) AS bucket1, + tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), CASE WHEN cs_ship_date_sk - cs_sold_date_sk > 60 AND cs_ship_date_sk - cs_sold_date_sk <= 90 THEN 1 ELSE 0 END) AS bucket2, + tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), CASE WHEN cs_ship_date_sk - cs_sold_date_sk > 90 AND cs_ship_date_sk - cs_sold_date_sk <= 120 THEN 1 ELSE 0 END) AS bucket3, + tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), CASE WHEN cs_ship_date_sk - cs_sold_date_sk > 120 THEN 1 ELSE 0 END) AS bucket4, + tpcds_count((priv_hash(hash(cs_bill_customer_sk))), 1) AS presence + FROM main.catalog_sales , + (SELECT SUBSTRING(w_warehouse_name,1,20) w_substr, * + FROM main.warehouse) AS sq1 , + main.ship_mode , + main.call_center , + main.date_dim, + (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 +WHERE cs_bill_customer_sk = asa_owner0.asa_key0 + AND d_month_seq BETWEEN 1200 AND 1200 + 11 + AND cs_ship_date_sk = d_date_sk + AND cs_warehouse_sk = w_warehouse_sk + AND cs_ship_mode_sk = sm_ship_mode_sk + AND cs_call_center_sk = cc_call_center_sk +GROUP BY w_substr , + sm_type , + cc_name +), answers AS ( + SELECT world, w_substr, + sm_type, + cc_name_lower, + bucket0[world+1] AS bucket0, + bucket1[world+1] AS bucket1, + bucket2[world+1] AS bucket2, + bucket3[world+1] AS bucket3, + bucket4[world+1] AS bucket4 + FROM stats CROSS JOIN range(64) worlds(world) + WHERE presence[world+1] > 0 +) +SELECT * FROM answers +QUALIFY row_number() OVER (PARTITION BY world ORDER BY w_substr NULLS FIRST, sm_type NULLS FIRST, cc_name_lower NULLS FIRST) <= 100 +ORDER BY world, w_substr NULLS FIRST, sm_type NULLS FIRST, cc_name_lower NULLS FIRST; diff --git a/benchmark/tpcds/run.py b/benchmark/tpcds/run.py index 6901c4d6..19bab0f8 100644 --- a/benchmark/tpcds/run.py +++ b/benchmark/tpcds/run.py @@ -2,58 +2,98 @@ """Isolate TPC-DS queries in processes; all reported timings come from C++.""" import argparse import csv +import hashlib import json import pathlib import statistics import subprocess +def digest_file(path): + digest = hashlib.sha256() + with pathlib.Path(path).open("rb") as file: + for block in iter(lambda: file.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", required=True) parser.add_argument("--runner", default="build/release/extension/privacy/as_tpcds_benchmark") parser.add_argument("--queries", help="Comma-separated query numbers; default: all 99") parser.add_argument("--resume", action="store_true") + validation = parser.add_mutually_exclusive_group() + validation.add_argument("--validate", action="store_true", help="Compare handwritten PAC SQL to automatic compilation") + validation.add_argument("--validate-compiled", action="store_true", help="Compare PAC output to world-zero SQL (semantics diagnostic, not compiler correctness)") + validation.add_argument("--validate-worlds", action="store_true", help="Run the separate 64-world AS reference checks") args = parser.parse_args() + if args.validate_compiled or args.validate_worlds: + args.validate = True config = json.loads(pathlib.Path(args.config).read_text()) - query_ids = [int(q) for q in args.queries.split(",")] if args.queries else list(range(1, 100)) + handwritten_dir = pathlib.Path(config.get("reference_queries_dir", "benchmark/tpcds/reference_queries") if args.validate_worlds else config.get("as_queries_dir", "benchmark/tpcds/as_queries")) + default_ids = sorted(int(p.stem[1:]) for p in handwritten_dir.glob("q[0-9][0-9].sql")) if args.validate else list(range(1, 100)) + query_ids = [int(q) for q in args.queries.split(",")] if args.queries else default_ids + suffix = "_compiled_validation" if args.validate_compiled else "_validation" if args.validate_worlds else "_pac_validation" if args.validate else "" if not query_ids or any(q < 1 or q > 99 for q in query_ids) or len(set(query_ids)) != len(query_ids): parser.error("Query numbers must be unique and between 1 and 99") output = pathlib.Path(config["out_dir"]) manifest_path = output / "run_config.json" if manifest_path.exists() and json.loads(manifest_path.read_text()) != config: parser.error("Configuration changed: use a fresh out_dir to avoid mixing experiments") - if not args.resume and any((output / f"q{q:02}.csv").exists() for q in query_ids): + if not args.resume and any((output / f"q{q:02}{suffix}.csv").exists() for q in query_ids): parser.error("Results already exist: use a fresh out_dir or --resume") subprocess.run([args.runner, args.config, "--prepare"], check=True) manifest_path.write_text(json.dumps(config, indent=2) + "\n") summary = [] + runner_digest = digest_file(args.runner) for query in query_ids: name = f"q{query:02}" - path = output / f"{name}.csv" + path = output / f"{name}{suffix}.csv" process_status = "completed" process_error = "" - process_path = output / f"{name}_process.json" - if args.resume and process_path.exists(): - state = json.loads(process_path.read_text()) + process_path = output / f"{name}{suffix}_process.json" + inputs = (runner_digest + digest_file(pathlib.Path(config["queries_dir"]) / f"{name}.sql") + + digest_file(config.get("schema", "benchmark/tpcds/schema.sql"))) + hand_sql = handwritten_dir / f"{name}.sql" + if not args.validate_compiled and hand_sql.exists(): + inputs += digest_file(hand_sql) + inputs_digest = hashlib.sha256(inputs.encode()).hexdigest() + state = json.loads(process_path.read_text()) if process_path.exists() else {} + if args.resume and state and state.get("inputs_sha256") == inputs_digest: process_status, process_error = state["status"], state["error"] else: print(f"Running {name}", flush=True) - with (output / f"{name}.log").open("w") as log: + with (output / f"{name}{suffix}.log").open("w") as log: try: result = subprocess.run( - [args.runner, args.config, str(query)], stdout=log, stderr=subprocess.STDOUT, + [args.runner, args.config, str(query)] + (["--validate-compiled"] if args.validate_compiled else ["--validate-worlds"] if args.validate_worlds else ["--validate"] if args.validate else []), stdout=log, stderr=subprocess.STDOUT, timeout=config.get("timeout_seconds", 120), ) if result.returncode: process_status = "process_error" - process_error = f"exit code {result.returncode}; see {name}.log" + process_error = f"exit code {result.returncode}; see {name}{suffix}.log" except subprocess.TimeoutExpired: process_status = "timeout" process_error = "Whole-query process timeout (both modes and all repetitions)" - process_path.write_text(json.dumps(dict(status=process_status, error=process_error))) + process_path.write_text(json.dumps(dict(status=process_status, error=process_error, inputs_sha256=inputs_digest))) rows = list(csv.DictReader(path.open())) if path.exists() else [] - for mode in ("duckdb", "simd_asa_auto"): + if args.validate: + if not args.validate_worlds: + for row in rows: + row["method"] = "world_zero_diagnostic" if args.validate_compiled else "pac_equivalence" + if rows: + summary.extend(rows) + else: + summary.append(dict(query=name, method="validation", status=process_status if process_status != "completed" else "incomplete", error=process_error)) + fields = list(dict.fromkeys(key for row in summary for key in row)) + with (output / ("compiled_validation_summary.csv" if args.validate_compiled else "validation_summary.csv" if args.validate_worlds else "pac_validation_summary.csv")).open("w", newline="") as file: + writer = csv.DictWriter(file, fieldnames=fields) + writer.writeheader() + writer.writerows(summary) + print(f"{name}: " + (", ".join(f"{r['method']}={r['status']}" for r in rows) if rows else process_status + ": " + process_error), flush=True) + continue + for mode in ("duckdb", "simd_asa_handwritten", "simd_asa_auto", "naive_as"): entries = [row for row in rows if row["mode"] == mode] measured = [row for row in entries if row["status"] == "ok"] complete = len(measured) == config.get("repetitions", 5) @@ -65,6 +105,7 @@ def main(): query=name, mode=mode, status=status, rewritten=entries[-1]["rewritten"] if entries else "", measured_runs=len(measured), + world_multiplier=entries[-1].get("world_multiplier", "") if entries else "", median_ms=statistics.median(float(row["runtime_ms"]) for row in measured) if complete else "", empty_result=all(row["rows"] == "0" for row in measured) if complete else "", correctness="not_validated", @@ -74,8 +115,8 @@ def main(): writer = csv.DictWriter(file, fieldnames=list(summary[0])) writer.writeheader() writer.writerows(summary) - print(f"{name}: " + ", ".join(f"{r['mode']}={r['status']}" for r in summary[-2:]), flush=True) - print(f"Results: {output / 'summary.csv'}") + print(f"{name}: " + ", ".join(f"{r['mode']}={r['status']}" for r in summary[-4:]), flush=True) + print(f"Results: {output / ('compiled_validation_summary.csv' if args.validate_compiled else 'validation_summary.csv' if args.validate_worlds else 'pac_validation_summary.csv' if args.validate else 'summary.csv')}") if __name__ == "__main__": From 0b57ceed112c7aeee117d4c3a6e743d84c5194ce Mon Sep 17 00:00:00 2001 From: ila Date: Tue, 10 Nov 2026 15:13:15 +0100 Subject: [PATCH 20/32] Add sequential Graviton revision rerun and preserve raw measurements --- benchmark/microbench/common.sh | 16 ++++-- benchmark/microbench/run_accuracy.py | 28 +++++++++ benchmark/microbench/run_graviton.sh | 33 +++++++++++ benchmark/prepare_graviton_revision_data.sh | 31 ++++++++++ benchmark/prepare_revision_configs.py | 49 ++++++++++++++++ benchmark/run_graviton_full_revision.sh | 63 +++++++++++++++++++++ docs/benchmark/graviton_duckdb2_rerun.md | 42 ++++++++++++++ 7 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 benchmark/microbench/run_accuracy.py create mode 100644 benchmark/microbench/run_graviton.sh create mode 100644 benchmark/prepare_graviton_revision_data.sh create mode 100644 benchmark/prepare_revision_configs.py create mode 100644 benchmark/run_graviton_full_revision.sh create mode 100644 docs/benchmark/graviton_duckdb2_rerun.md diff --git a/benchmark/microbench/common.sh b/benchmark/microbench/common.sh index 5e2a0e07..21e10ec1 100755 --- a/benchmark/microbench/common.sh +++ b/benchmark/microbench/common.sh @@ -3,7 +3,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BINARIES_DIR="$SCRIPT_DIR/binaries" -RESULTS_DIR="$SCRIPT_DIR/results" +RESULTS_DIR="${RESULTS_DIR:-$SCRIPT_DIR/results}" # Default binary (with all optimizations enabled) DEFAULT_BINARY="$BINARIES_DIR/duckdb_default" TEST_DB="$BINARIES_DIR/test_data.duckdb" @@ -14,10 +14,14 @@ BENCH_RUNS=${BENCH_RUNS:-3} THREADS=${THREADS:-1} # Default to single-threaded to avoid partitioning issues # Query setup commands -QUERY_SETUP="SET pac_mi=0.0; SET threads=4; SET max_memory='30G'; SET max_temp_directory_size='90GB';" +QUERY_SETUP="${QUERY_SETUP:-SET pac_mi=0.0; SET threads=4; SET max_memory='30G'; SET max_temp_directory_size='90GB';}" mkdir -p "$RESULTS_DIR" +median_seconds() { + sort -n | awk '{a[NR]=$1} END {if (NR % 2) print a[(NR+1)/2]; else if (NR) print (a[NR/2]+a[NR/2+1])/2}' +} + # Run a benchmark query and extract timing # Args: binary, query, [runs] # Returns: wall_median,agg_median,wall_times,agg_times @@ -98,8 +102,8 @@ except: fi # Calculate medians - local wall_median=$(printf '%s\n' "${wall_times[@]}" | sort -n | awk 'NR==int((NR+1)/2)') - local agg_median=$(printf '%s\n' "${agg_times[@]}" | sort -n | awk 'NR==int((NR+1)/2)') + local wall_median=$(printf '%s\n' "${wall_times[@]}" | median_seconds) + local agg_median=$(printf '%s\n' "${agg_times[@]}" | median_seconds) echo "$wall_median,$agg_median,${wall_times[*]},${agg_times[*]}" } @@ -274,8 +278,8 @@ except: fi # Calculate medians - local wall_median=$(printf '%s\n' "${wall_times[@]}" | sort -n | awk 'NR==int((NR+1)/2)') - local agg_median=$(printf '%s\n' "${agg_times[@]}" | sort -n | awk 'NR==int((NR+1)/2)') + local wall_median=$(printf '%s\n' "${wall_times[@]}" | median_seconds) + local agg_median=$(printf '%s\n' "${agg_times[@]}" | median_seconds) echo "$wall_median,$agg_median,${wall_times[*]},${agg_times[*]}" } diff --git a/benchmark/microbench/run_accuracy.py b/benchmark/microbench/run_accuracy.py new file mode 100644 index 00000000..1fa07200 --- /dev/null +++ b/benchmark/microbench/run_accuracy.py @@ -0,0 +1,28 @@ +"""Repeat the existing ten-distribution SUM accuracy experiment on a common scale.""" +import argparse +import subprocess +from pathlib import Path + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("binary", type=Path) +parser.add_argument("output", type=Path) +args = parser.parse_args() +source = (Path(__file__).parent / "approx_sum_stability.sql").read_text() +marker = source.index("-- test query:") +setup = source[:marker] +start = source.index("WITH bit_positions", marker) +query = source[start:source.index(";", start)] +# as_sum returns 2x the sampled totals. Rescale once, before computing both +# squared errors and the approximate variance, so every statistic uses the +# same units as independently computed sample sums. +query = query.replace("as_sum(priv_hash(hash(rowid)), val) as approx_counters", + "list_transform(as_sum(priv_hash(hash(rowid)), val), lambda x: x / 2.0) as approx_counters") +query = query.replace("a.approx_counters[x+1] / 2", "a.approx_counters[x+1]") +query = query.replace("x -> abs(x)", "lambda x: abs(x)") +path = str(args.output.resolve()).replace("'", "''") +sql = ("SET threads=4; SET memory_limit='30GB'; SET priv_rewrite=false; " + "SET pac_mi=1; SET privacy_seed=42; SELECT setseed(0.42);\n" + setup + + "\nCOPY (" + query + ") TO '" + path + "' (HEADER, DELIMITER ',');") +args.output.parent.mkdir(parents=True, exist_ok=True) +args.output.with_suffix(".sql").write_text(sql) +subprocess.run([str(args.binary.resolve()), "-batch"], input=sql, text=True, check=True) diff --git a/benchmark/microbench/run_graviton.sh b/benchmark/microbench/run_graviton.sh new file mode 100644 index 00000000..174a8c96 --- /dev/null +++ b/benchmark/microbench/run_graviton.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Build variants without deleting the analytical build or its measured binaries. +set -euo pipefail +results="${1:?Usage: run_graviton.sh NEW_RESULTS_DIR}" +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +test ! -e "$results" || { echo "Refusing existing results: $results" >&2; exit 1; } +mkdir -p "$results/builds" "$repo/benchmark/microbench/binaries" +variants=(default noboundopt nobuffering signedsum exactsum nocascading nosimd) +flags=('' '-DPAC_NOBOUNDOPT' '-DPAC_NOBUFFERING' '-DPAC_SIGNEDSUM' '-DPAC_EXACTSUM' '-DPAC_NOCASCADING' '-DPAC_NOCASCADING -DPAC_NOSIMD -fno-vectorize -fno-slp-vectorize') +for i in "${!variants[@]}"; do + name="${variants[$i]}" + cmake -S "$repo/duckdb" -B "$repo/build/graviton_micro" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang-22 -DCMAKE_CXX_COMPILER=clang++-22 \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DDUCKDB_EXTENSION_CONFIGS="$repo/extension_config.cmake" -DBUILD_UNITTESTS=OFF \ + -DPAC_NOAPPROX=OFF -DCMAKE_CXX_FLAGS="${flags[$i]}" \ + > "$results/builds/$name.configure.log" 2>&1 + cmake --build "$repo/build/graviton_micro" --target duckdb -j 4 > "$results/builds/$name.build.log" 2>&1 + cp "$repo/build/graviton_micro/duckdb" "$repo/benchmark/microbench/binaries/duckdb_$name" + cp "$repo/build/graviton_micro/CMakeCache.txt" "$results/builds/$name.CMakeCache.txt" +done +bash "$repo/benchmark/microbench/create_test_db.sh" > "$results/create_data.log" 2>&1 +export RESULTS_DIR="$results/csv" +export QUERY_SETUP="SET pac_mi=0.0; SET threads=4; SET max_memory='30G'; SET max_temp_directory_size='90GB'; SET temp_directory='$repo/benchmark/microbench/tmp';" +export WARMUP_RUNS=1 BENCH_RUNS=3 +for benchmark in count sum_avg min_max; do + bash "$repo/benchmark/microbench/bench_$benchmark.sh" > "$results/$benchmark.log" 2>&1 +done +for variant in default signedsum exactsum; do + python3 "$repo/benchmark/microbench/run_accuracy.py" \ + "$repo/benchmark/microbench/binaries/duckdb_$variant" "$results/csv/accuracy_$variant.csv" \ + > "$results/accuracy_$variant.log" 2>&1 +done diff --git a/benchmark/prepare_graviton_revision_data.sh b/benchmark/prepare_graviton_revision_data.sh new file mode 100644 index 00000000..0908269c --- /dev/null +++ b/benchmark/prepare_graviton_revision_data.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +root="${1:?Usage: prepare_graviton_revision_data.sh ROOT}" +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +mkdir -p "$root/data" "$root/repro/data_prepare" +exec > >(tee -a "$root/repro/data_prepare/run.log") 2>&1 +trap 'printf "%s\n" "$?" > "$root/repro/data_prepare/exit_code"' EXIT +cd "$repo" +if [ ! -d "$root/ssb-dbgen" ]; then + git clone https://github.com/electrum/ssb-dbgen.git "$root/ssb-dbgen" + git -C "$root/ssb-dbgen" checkout 219403ad7d1dd32ae1f97b5553abf92129fccd7f + git -C "$root/ssb-dbgen" apply "$repo/benchmark/ssb/dbgen-macos.patch" +fi +make -C "$root/ssb-dbgen" dbgen CC=clang-22 \ + CFLAGS='-O2 -std=gnu89 -include unistd.h -D_GNU_SOURCE -DDBNAME=\"dss\" -DLINUX -DDB2 -DSSBM' +if [ ! -f "$root/data/ssb_sf30/generation_complete" ]; then + bash benchmark/ssb/prepare.sh "$root/ssb-dbgen" "$root/data/ssb_sf30" 30 +fi +python3 -m venv --system-site-packages "$root/prepare-env" +"$root/prepare-env/bin/python" -m pip freeze > "$root/repro/data_prepare/python_requirements.txt" +for workload in damico2006 ahrq2016; do + if [ ! -f "$root/data/meps/$workload/manifest.json" ]; then + "$root/prepare-env/bin/python" benchmark/meps_brr/prepare.py "$root/data/meps/$workload" --workload "$workload" + fi +done +if [ ! -f "$repo/hits.parquet" ]; then + curl --fail --location --retry 3 https://datasets.clickhouse.com/hits_compatible/hits.parquet \ + --output "$repo/hits.parquet.part" + mv "$repo/hits.parquet.part" "$repo/hits.parquet" +fi +sha256sum "$repo/hits.parquet" > "$root/repro/data_prepare/clickbench.sha256" diff --git a/benchmark/prepare_revision_configs.py b/benchmark/prepare_revision_configs.py new file mode 100644 index 00000000..94eb194a --- /dev/null +++ b/benchmark/prepare_revision_configs.py @@ -0,0 +1,49 @@ +"""Prepare fresh Graviton run configurations without changing paper configurations.""" +import argparse +import json +from pathlib import Path + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("root", type=Path) +parser.add_argument("output", type=Path) +args = parser.parse_args() +repo = Path(__file__).resolve().parent.parent +args.output.mkdir(parents=True, exist_ok=True) +tpch_db = str(args.root / "data/tpch_sf30_duckdb2.db") +for source in sorted((repo / "benchmark/configs/utility").glob("*.json")): + if source.name.startswith("as_tpcds"): + continue + config = json.loads(source.read_text()) + config["out"] = str(args.output.parent / "csv" / (source.stem + ".csv")) + if source.name.startswith("as_tpch"): + config.update(db=tpch_db, queries_dir=str(repo / "benchmark")) + elif source.name.startswith("as_clickbench"): + config.update(db=str(args.root / "data/clickbench_duckdb2.db"), + queries_dir=str(repo / "benchmark/clickbench/clickbench_queries")) + elif source.name.startswith("dp_tpch"): + for dataset in config.get("datasets", []): + dataset["db"] = tpch_db + else: + continue + (args.output / source.name).write_text(json.dumps(config, indent=2) + "\n") + +config = json.loads((repo / "benchmark/configs/utility/as_tpcds_sf30.json").read_text()) +config.update(db=str(args.root / "data/tpcds_sf30_duckdb2/data.db"), + queries_dir=str(args.root / "data/tpcds_sf30_duckdb2/queries"), + out_dir=str(args.output.parent / "tpcds_sf30"), + schema=str(repo / "benchmark/tpcds/schema.sql"), + as_queries_dir=str(repo / "benchmark/tpcds/as_queries"), + reference_queries_dir=str(repo / "benchmark/tpcds/reference_queries"), + extension_directory=str(args.root / "data/tpcds_extensions_duckdb2")) +(args.output / "as_tpcds_sf30.json").write_text(json.dumps(config, indent=2) + "\n") +config.update(sf=1, threads=1, repetitions=1, + db=str(args.root / "data/tpcds_sf1_duckdb2/data.db"), + queries_dir=str(args.root / "data/tpcds_sf1_duckdb2/queries"), + out_dir=str(args.output.parent / "tpcds_sf1_validation")) +(args.output / "as_tpcds_validate_sf1.json").write_text(json.dumps(config, indent=2) + "\n") + +main = json.loads((repo / "benchmark/configs/utility/as_tpch_sf30.json").read_text()) +main.update(db=str(args.root / "data/tpch_sf30_main.db"), + queries_dir=str(args.root / "privacy/benchmark"), + out=str(args.output.parent / "csv/as_tpch_sf30_main.csv")) +(args.output / "as_tpch_sf30_main.json").write_text(json.dumps(main, indent=2) + "\n") diff --git a/benchmark/run_graviton_full_revision.sh b/benchmark/run_graviton_full_revision.sh new file mode 100644 index 00000000..862e3936 --- /dev/null +++ b/benchmark/run_graviton_full_revision.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Run analytical experiments first, then DP experiments, then microbenchmarks. +set -euo pipefail +root="${1:?Usage: run_graviton_full_revision.sh ROOT NEW_RESULTS_DIR}" +results="${2:?A fresh result directory is required}" +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +test ! -e "$results" || { echo "Refusing existing results: $results" >&2; exit 1; } +mkdir -p "$results/repro" "$results/csv" +exec > >(tee -a "$results/repro/run.log") 2>&1 +printf 'running\n' > "$results/repro/exit_code" +trap 'code=$?; printf "%s\n" "$code" > "$results/repro/exit_code"; date -u > "$results/repro/finished_utc.txt"' EXIT +date -u > "$results/repro/started_utc.txt" +cd "$repo" +git rev-parse HEAD > "$results/repro/source_commit.txt" +git status --short > "$results/repro/source_status.txt" +git submodule status > "$results/repro/submodules.txt" +{ uname -a; lscpu; free -h; lsblk -o NAME,SIZE,MODEL,MOUNTPOINTS; df -h "$root"; clang-22 --version; } > "$results/repro/environment.txt" +for unit in asa-main-prepare-20261110 asa-duckdb2-build-20261110 asa-data-prepare-20261110; do + while systemctl is-active --quiet "$unit"; do sleep 10; done +done +test "$(cat "$root/repro/main_prepare/exit_code")" = 0 +test "$(cat "$root/repro/duckdb2_build/exit_code")" = 0 +test "$(cat "$root/repro/data_prepare/exit_code")" = 0 +cp build/release/CMakeCache.txt "$results/repro/CMakeCache.txt" +python3 benchmark/prepare_revision_configs.py "$root" "$results/configs" +cp --reflink=auto "$root/data/tpch_sf30_base.db" "$root/data/tpch_sf30_main.db" +cp --reflink=auto "$root/data/tpch_sf30_base.db" "$root/data/tpch_sf30_duckdb2.db" +failures=0 +run_stage() { + local stage="$1" + shift + printf '%s\n' "$stage" > "$results/repro/stage" + date -u + if "$@" > "$results/repro/$stage.log" 2>&1; then + printf '%s,0\n' "$stage" >> "$results/repro/stage_status.csv" + else + local code=$? + printf '%s,%s\n' "$stage" "$code" >> "$results/repro/stage_status.csv" + failures=$((failures+1)) + tail -25 "$results/repro/$stage.log" + fi +} +run_stage tpch_main "$root/privacy/build/release/extension/privacy/as_tpch_benchmark" --config "$results/configs/as_tpch_sf30_main.json" +run_stage tpch_2 "$repo/build/release/extension/privacy/as_tpch_benchmark" --config "$results/configs/as_tpch_sf30.json" +run_stage tpch_m_sweep "$repo/build/release/extension/privacy/as_tpch_benchmark" --config "$results/configs/as_tpch_sf30_m_sweep.json" +run_stage clickbench "$repo/build/release/extension/privacy/as_clickbench_benchmark" --config "$results/configs/as_clickbench_graviton.json" +run_stage ssb env SSB_MEMORY_LIMIT=30GB bash benchmark/ssb/run.sh "$root/data/ssb_sf30" "$results/ssb_sf30" 16 5 +run_stage meps bash benchmark/meps_brr/run.sh "$root/data/meps" "$results/meps" 16 5 +run_stage tpcds_validation python3 benchmark/tpcds/run.py --config "$results/configs/as_tpcds_validate_sf1.json" --validate +run_stage tpcds python3 benchmark/tpcds/run.py --config "$results/configs/as_tpcds_sf30.json" +for config in "$results"/configs/dp_tpch*.json; do + name="$(basename "$config" .json)" + if [[ "$name" == *stability* ]]; then + run_stage "$name" python3 benchmark/dp/run_tpch_sass_stability.py --config "$config" \ + --duckdb "$repo/build/release/duckdb" --out "$results/csv/$name.csv" \ + --summary-out "$results/csv/${name}_summary.csv" --strict + else + run_stage "$name" "$repo/build/release/extension/privacy/dp_benchmark_runner" --config "$config" + fi +done +run_stage microbench bash benchmark/microbench/run_graviton.sh "$results/microbench" +printf 'complete\n' > "$results/repro/stage" +test "$failures" = 0 diff --git a/docs/benchmark/graviton_duckdb2_rerun.md b/docs/benchmark/graviton_duckdb2_rerun.md new file mode 100644 index 00000000..cc8d62dd --- /dev/null +++ b/docs/benchmark/graviton_duckdb2_rerun.md @@ -0,0 +1,42 @@ +# Graviton revision rerun, November 2026 + +The DuckDB 2.0 branch is rebased onto main `52d4a3b`. The Graviton host is +`ec2-3-136-25-192.us-east-2.compute.amazonaws.com`: 16 Neoverse-V2 cores, +32 GB RAM, Ubuntu 26.04, Clang 22.1.2, and a local NVMe SSD mounted at +`/mnt/asa`. The SSD was blank after instance restart; datasets are regenerated. + +`/mnt/asa/privacy` holds main; `/mnt/asa/privacy2` holds the upgrade branch. +Both use separate build directories and database copies. The upgrade's pinned +DuckDB source is `d54cbbe3df17e8e5ce68312f483b4155836112aa`. TPC-H and TPC-DS +generator extensions are built from that same source and installed locally. + +Preparation/build services record their commits, settings, logs, and exit codes +under `/mnt/asa/repro`. Run the suite after starting these services: + +```sh +sudo systemd-run --unit=asa-full-revision-20261110 --uid=ubuntu \ + --working-directory=/mnt/asa/privacy2 \ + /bin/bash /mnt/asa/privacy2/benchmark/run_graviton_full_revision.sh \ + /mnt/asa /mnt/asa/results/duckdb2_revision_20261110 +``` + +Order: matched main TPC-H control, 2.0 TPC-H and its sample-count sweep, +ClickBench, SSB, MEPS, TPC-DS validation/performance, DP utility/stability, +then aggregation microbenchmarks. Analytical timings retain five measured +executions after warmup; DP configurations retain their existing three runs. +SSB and MEPS run approximate and exact SUM. Microbenchmarks retain their +existing four-thread setup, one warmup, three runs, and slow-query cutoff. +All stages run sequentially. A failed stage is logged and subsequent stages +continue; individual TPC-DS query failures are recorded in its CSVs. + +The microbenchmark driver uses a separate build tree and never deletes the +analytical build. Its summary now computes a true median: the old awk expression +selected the minimum sorted observation. Raw timing samples remain in the CSVs. +SUM accuracy compares exact and approximate samples on the same unscaled units +for both errors and variances. These measurement corrections must be considered +when comparing old summaries. + +Copy CSVs, configs, manifests, plans, and logs locally before stopping the instance. +The local collection is `benchmark/results/graviton_duckdb2_20261110/`; prior +paper CSVs and the October 28 five-run Graviton controls are retained under +`previous/`. No paper text or figures are updated by this driver. From 69a1d6e9ef2c2507190ae1b9eeb47e50325089ff Mon Sep 17 00:00:00 2001 From: ila Date: Tue, 10 Nov 2026 15:17:21 +0100 Subject: [PATCH 21/32] Wait for preparation dependencies before converting MEPS inputs --- benchmark/prepare_graviton_revision_data.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/prepare_graviton_revision_data.sh b/benchmark/prepare_graviton_revision_data.sh index 0908269c..8da9f022 100644 --- a/benchmark/prepare_graviton_revision_data.sh +++ b/benchmark/prepare_graviton_revision_data.sh @@ -16,6 +16,7 @@ make -C "$root/ssb-dbgen" dbgen CC=clang-22 \ if [ ! -f "$root/data/ssb_sf30/generation_complete" ]; then bash benchmark/ssb/prepare.sh "$root/ssb-dbgen" "$root/data/ssb_sf30" 30 fi +while pgrep -x apt-get >/dev/null; do sleep 5; done python3 -m venv --system-site-packages "$root/prepare-env" "$root/prepare-env/bin/python" -m pip freeze > "$root/repro/data_prepare/python_requirements.txt" for workload in damico2006 ahrq2016; do From 43146f896b8d70ef2b7bde3f8605b50548efe61a Mon Sep 17 00:00:00 2001 From: ila Date: Tue, 10 Nov 2026 15:19:24 +0100 Subject: [PATCH 22/32] Audit revision CSVs against matched and archived analytical results --- benchmark/audit_revision_results.py | 76 +++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 benchmark/audit_revision_results.py diff --git a/benchmark/audit_revision_results.py b/benchmark/audit_revision_results.py new file mode 100644 index 00000000..726263da --- /dev/null +++ b/benchmark/audit_revision_results.py @@ -0,0 +1,76 @@ +"""Collect runtime comparisons and execution anomalies without generating figures.""" +import argparse +import csv +import json +import statistics +from pathlib import Path + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("collection", type=Path) +args = parser.parse_args() +fresh = args.collection / "results/duckdb2_revision_20261110" +previous = args.collection / "previous" +comparisons, anomalies = [], [] + +def read(path): + if not path.exists(): + return [] + return list(csv.DictReader(path.open())) + +def medians(path, value="median_ms", warmup=False): + grouped = {} + record_anomalies = path.is_relative_to(fresh) + for row in read(path): + if warmup and row.get("run") == "0": + continue + if row.get("status") not in (None, "ok", "executed_not_validated"): + if record_anomalies: + anomalies.append(dict(file=str(path.relative_to(args.collection)), query=row.get("query", ""), + mode=row.get("mode", ""), status=row["status"], error=row.get("error", ""))) + continue + if not row.get(value): + continue + number = float(row[value]) + if number < 0: + if record_anomalies: + anomalies.append(dict(file=str(path.relative_to(args.collection)), query=row.get("query", ""), + mode=row.get("mode", ""), status="negative_runtime", error="")) + continue + mode = "".join(c for c in row.get("mode", "").lower() if c.isalnum()) + m = row.get("m") or ("0" if mode == "baseline" else "64") + key = row.get("query", ""), mode, m + grouped.setdefault(key, []).append(number) + return {key: statistics.median(values) for key, values in grouped.items()} + +def compare(name, old, new, value="median_ms", warmup=False): + reference, measured = medians(old, value, warmup), medians(new, value, warmup) + for key in sorted(reference.keys() & measured.keys()): + before, after = reference[key], measured[key] + comparisons.append(dict(benchmark=name, query=key[0], mode=key[1], m=key[2], + reference_ms=before, new_ms=after, + runtime_ratio=after / before if before else "", + inspect=bool(before and after / before > 1.2))) + +compare("tpch_matched_main", fresh / "csv/as_tpch_sf30_main.csv", fresh / "csv/as_tpch_sf30.csv") +compare("tpch_previous_paper", previous / "as_tpch_m64.csv", fresh / "csv/as_tpch_sf30.csv") +compare("tpch_m_previous_paper", previous / "as_tpch_m_sweep.csv", fresh / "csv/as_tpch_sf30_m_sweep.csv") +compare("clickbench_previous_paper", previous / "as_clickbench_m64_m512.csv", fresh / "csv/as_clickbench_graviton.csv") +compare("tpcds_previous_paper", previous / "tpcds_sf30_plot_data.csv", fresh / "tpcds_sf30/summary.csv") +for kernel in ("approximate", "exact"): + subpath = Path("ssb_sf30") / kernel / "threads_16_timings.csv" + compare("ssb_" + kernel, previous / "graviton_5runs" / subpath, fresh / subpath, "query_ms") + for workload in ("damico2006", "ahrq2016"): + subpath = Path("meps") / kernel / workload / "threads_16_timings.csv" + compare("meps_" + kernel + "_" + workload, previous / "graviton_5runs" / subpath, + fresh / subpath, "query_ms", warmup=True) + +for name, rows, fields in ( + ("runtime_comparison.csv", comparisons, ["benchmark", "query", "mode", "m", "reference_ms", "new_ms", "runtime_ratio", "inspect"]), + ("execution_anomalies.csv", anomalies, ["file", "query", "mode", "status", "error"]), +): + with (args.collection / name).open("w", newline="") as file: + writer = csv.DictWriter(file, fieldnames=fields) + writer.writeheader() + writer.writerows(rows) +print(json.dumps(dict(comparisons=len(comparisons), inspect=sum(r["inspect"] for r in comparisons), + execution_anomalies=len(anomalies)))) From b53a1c0533e28acfa1819f841c56586edec8be21 Mon Sep 17 00:00:00 2001 From: ila Date: Tue, 10 Nov 2026 15:35:44 +0100 Subject: [PATCH 23/32] Wait for retried dataset preparation before timing benchmarks --- benchmark/prepare_graviton_revision_data.sh | 1 + benchmark/run_graviton_full_revision.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmark/prepare_graviton_revision_data.sh b/benchmark/prepare_graviton_revision_data.sh index 8da9f022..df283724 100644 --- a/benchmark/prepare_graviton_revision_data.sh +++ b/benchmark/prepare_graviton_revision_data.sh @@ -4,6 +4,7 @@ root="${1:?Usage: prepare_graviton_revision_data.sh ROOT}" repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" mkdir -p "$root/data" "$root/repro/data_prepare" exec > >(tee -a "$root/repro/data_prepare/run.log") 2>&1 +printf 'running\n' > "$root/repro/data_prepare/exit_code" trap 'printf "%s\n" "$?" > "$root/repro/data_prepare/exit_code"' EXIT cd "$repo" if [ ! -d "$root/ssb-dbgen" ]; then diff --git a/benchmark/run_graviton_full_revision.sh b/benchmark/run_graviton_full_revision.sh index 862e3936..11e767bf 100644 --- a/benchmark/run_graviton_full_revision.sh +++ b/benchmark/run_graviton_full_revision.sh @@ -15,7 +15,7 @@ git rev-parse HEAD > "$results/repro/source_commit.txt" git status --short > "$results/repro/source_status.txt" git submodule status > "$results/repro/submodules.txt" { uname -a; lscpu; free -h; lsblk -o NAME,SIZE,MODEL,MOUNTPOINTS; df -h "$root"; clang-22 --version; } > "$results/repro/environment.txt" -for unit in asa-main-prepare-20261110 asa-duckdb2-build-20261110 asa-data-prepare-20261110; do +for unit in asa-main-prepare-20261110 asa-duckdb2-build-20261110 asa-data-prepare-20261110 asa-data-prepare-retry-20261110 asa-duckdb2-validated-20261110; do while systemctl is-active --quiet "$unit"; do sleep 10; done done test "$(cat "$root/repro/main_prepare/exit_code")" = 0 From e18010fc94db2f3e141ff4b2f71aa5adbeb598cb Mon Sep 17 00:00:00 2001 From: ila Date: Tue, 10 Nov 2026 17:29:50 +0100 Subject: [PATCH 24/32] Update SSB and MEPS setup for DuckDB 2.0 and queue analytical retries --- .../ahrq2016/as_queries/q01_office.sql | 20 ++++----- .../as_queries/q02_office_physician.sql | 20 ++++----- .../ahrq2016/as_queries/q03_outpatient.sql | 20 ++++----- .../as_queries/q04_outpatient_physician.sql | 20 ++++----- .../as_queries/q05_visits_per_person.sql | 8 ++-- .../damico2006/as_queries/q01_adults.sql | 2 +- .../damico2006/as_queries/q02_nonelderly.sql | 2 +- benchmark/microbench/run_graviton.sh | 2 + benchmark/retry_graviton_analytical.sh | 41 +++++++++++++++++++ benchmark/ssb/as_ssb_benchmark.cpp | 4 +- 10 files changed, 91 insertions(+), 48 deletions(-) create mode 100644 benchmark/retry_graviton_analytical.sh diff --git a/benchmark/meps_brr/ahrq2016/as_queries/q01_office.sql b/benchmark/meps_brr/ahrq2016/as_queries/q01_office.sql index 99eaf846..90fb892d 100644 --- a/benchmark/meps_brr/ahrq2016/as_queries/q01_office.sql +++ b/benchmark/meps_brr/ahrq2016/as_queries/q01_office.sql @@ -2,24 +2,24 @@ SELECT as_sum(p.mask_lo, p.weight) AS event_count_lo, as_sum(p.mask_hi, p.weight) AS event_count_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.self), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_self_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_self_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.self), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_self_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_self_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.private), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_private_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_private_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.private), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_private_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_private_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.medicare), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicare_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicare_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.medicare), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicare_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicare_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.medicaid), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicaid_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicaid_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.medicaid), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicaid_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicaid_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.other), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_other_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_other_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.other), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_other_hi + lambda x: x[1] / nullif(x[2], 0)) AS mean_other_hi FROM persons_as p JOIN events e USING (person_id) WHERE e.kind = 'OB'; diff --git a/benchmark/meps_brr/ahrq2016/as_queries/q02_office_physician.sql b/benchmark/meps_brr/ahrq2016/as_queries/q02_office_physician.sql index d46c93bb..3ed1b356 100644 --- a/benchmark/meps_brr/ahrq2016/as_queries/q02_office_physician.sql +++ b/benchmark/meps_brr/ahrq2016/as_queries/q02_office_physician.sql @@ -2,24 +2,24 @@ SELECT as_sum(p.mask_lo, p.weight) AS event_count_lo, as_sum(p.mask_hi, p.weight) AS event_count_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.self), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_self_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_self_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.self), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_self_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_self_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.private), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_private_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_private_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.private), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_private_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_private_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.medicare), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicare_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicare_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.medicare), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicare_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicare_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.medicaid), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicaid_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicaid_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.medicaid), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicaid_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicaid_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.other), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_other_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_other_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.other), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_other_hi + lambda x: x[1] / nullif(x[2], 0)) AS mean_other_hi FROM persons_as p JOIN events e USING (person_id) WHERE e.kind = 'OB' AND e.physician = 1; diff --git a/benchmark/meps_brr/ahrq2016/as_queries/q03_outpatient.sql b/benchmark/meps_brr/ahrq2016/as_queries/q03_outpatient.sql index 0ac89328..d6af9da7 100644 --- a/benchmark/meps_brr/ahrq2016/as_queries/q03_outpatient.sql +++ b/benchmark/meps_brr/ahrq2016/as_queries/q03_outpatient.sql @@ -2,24 +2,24 @@ SELECT as_sum(p.mask_lo, p.weight) AS event_count_lo, as_sum(p.mask_hi, p.weight) AS event_count_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.self), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_self_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_self_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.self), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_self_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_self_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.private), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_private_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_private_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.private), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_private_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_private_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.medicare), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicare_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicare_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.medicare), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicare_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicare_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.medicaid), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicaid_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicaid_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.medicaid), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicaid_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicaid_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.other), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_other_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_other_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.other), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_other_hi + lambda x: x[1] / nullif(x[2], 0)) AS mean_other_hi FROM persons_as p JOIN events e USING (person_id) WHERE e.kind = 'OP'; diff --git a/benchmark/meps_brr/ahrq2016/as_queries/q04_outpatient_physician.sql b/benchmark/meps_brr/ahrq2016/as_queries/q04_outpatient_physician.sql index 9c528f12..41564706 100644 --- a/benchmark/meps_brr/ahrq2016/as_queries/q04_outpatient_physician.sql +++ b/benchmark/meps_brr/ahrq2016/as_queries/q04_outpatient_physician.sql @@ -2,24 +2,24 @@ SELECT as_sum(p.mask_lo, p.weight) AS event_count_lo, as_sum(p.mask_hi, p.weight) AS event_count_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.self), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_self_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_self_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.self), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_self_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_self_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.private), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_private_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_private_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.private), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_private_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_private_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.medicare), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicare_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicare_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.medicare), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicare_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicare_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.medicaid), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicaid_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicaid_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.medicaid), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_medicaid_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_medicaid_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * e.other), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_other_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_other_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * e.other), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_other_hi + lambda x: x[1] / nullif(x[2], 0)) AS mean_other_hi FROM persons_as p JOIN events e USING (person_id) WHERE e.kind = 'OP' AND e.physician = 1; diff --git a/benchmark/meps_brr/ahrq2016/as_queries/q05_visits_per_person.sql b/benchmark/meps_brr/ahrq2016/as_queries/q05_visits_per_person.sql index c53e1d8a..e01bce2e 100644 --- a/benchmark/meps_brr/ahrq2016/as_queries/q05_visits_per_person.sql +++ b/benchmark/meps_brr/ahrq2016/as_queries/q05_visits_per_person.sql @@ -5,12 +5,12 @@ WITH visits AS ( FROM events WHERE kind = 'OB' GROUP BY person_id ) SELECT list_transform(list_zip(as_sum(p.mask_lo, p.weight * coalesce(v.n, 0)), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_office_visits_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_office_visits_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * coalesce(v.n, 0)), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_office_visits_hi, + lambda x: x[1] / nullif(x[2], 0)) AS mean_office_visits_hi, list_transform(list_zip(as_sum(p.mask_lo, p.weight * coalesce(v.n_physician, 0)), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_office_physician_visits_lo, + lambda x: x[1] / nullif(x[2], 0)) AS mean_office_physician_visits_lo, list_transform(list_zip(as_sum(p.mask_hi, p.weight * coalesce(v.n_physician, 0)), as_sum(p.mask_hi, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_office_physician_visits_hi + lambda x: x[1] / nullif(x[2], 0)) AS mean_office_physician_visits_hi FROM persons_as p LEFT JOIN visits v USING (person_id) WHERE true; diff --git a/benchmark/meps_brr/damico2006/as_queries/q01_adults.sql b/benchmark/meps_brr/damico2006/as_queries/q01_adults.sql index 1f519414..e792657e 100644 --- a/benchmark/meps_brr/damico2006/as_queries/q01_adults.sql +++ b/benchmark/meps_brr/damico2006/as_queries/q01_adults.sql @@ -1,5 +1,5 @@ -- Damico (2009), Table 2: explicit AS rewrite; masks constructed in the timed query. SELECT list_transform(list_zip(as_sum(p.mask_lo, p.weight * p.expenditure), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_expenditure_lo + lambda x: x[1] / nullif(x[2], 0)) AS mean_expenditure_lo FROM persons_as p WHERE p.age > 17; diff --git a/benchmark/meps_brr/damico2006/as_queries/q02_nonelderly.sql b/benchmark/meps_brr/damico2006/as_queries/q02_nonelderly.sql index a138f0fd..5661a874 100644 --- a/benchmark/meps_brr/damico2006/as_queries/q02_nonelderly.sql +++ b/benchmark/meps_brr/damico2006/as_queries/q02_nonelderly.sql @@ -1,5 +1,5 @@ -- Damico (2009), Table 2: explicit AS rewrite; masks constructed in the timed query. SELECT list_transform(list_zip(as_sum(p.mask_lo, p.weight * p.expenditure), as_sum(p.mask_lo, p.weight)), - x -> x[1] / nullif(x[2], 0)) AS mean_expenditure_lo + lambda x: x[1] / nullif(x[2], 0)) AS mean_expenditure_lo FROM persons_as p WHERE p.age > 17 AND p.age < 65; diff --git a/benchmark/microbench/run_graviton.sh b/benchmark/microbench/run_graviton.sh index 174a8c96..4d68d855 100644 --- a/benchmark/microbench/run_graviton.sh +++ b/benchmark/microbench/run_graviton.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Build variants without deleting the analytical build or its measured binaries. set -euo pipefail +# The current rerun retries analytical setup failures before timing microbenchmarks. +while systemctl is-active --quiet asa-analytical-retry-20261110.service; do sleep 10; done results="${1:?Usage: run_graviton.sh NEW_RESULTS_DIR}" repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" test ! -e "$results" || { echo "Refusing existing results: $results" >&2; exit 1; } diff --git a/benchmark/retry_graviton_analytical.sh b/benchmark/retry_graviton_analytical.sh new file mode 100644 index 00000000..c6d58c49 --- /dev/null +++ b/benchmark/retry_graviton_analytical.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Retry the two setup failures after DP, before the queued microbenchmarks. +set -euo pipefail +root="${1:?Usage: retry_graviton_analytical.sh ROOT RESULTS_DIR}" +results="${2:?RESULTS_DIR is required}" +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +exec > >(tee -a "$results/repro/analytical_retry.log") 2>&1 +printf 'running\n' > "$results/repro/analytical_retry_exit_code" +trap 'printf "%s\n" "$?" > "$results/repro/analytical_retry_exit_code"' EXIT +while [ "$(cat "$results/repro/stage")" != microbench ]; do + systemctl is-active --quiet asa-full-revision-retry-20261110.service || exit 1 + sleep 10 +done +cd "$repo" +git rev-parse HEAD > "$results/repro/analytical_retry_source_commit.txt" +failures=0 +for workload in ssb meps; do + output="$results/meps" + data="$root/data/meps" + script=benchmark/meps_brr/run.sh + if [ "$workload" = ssb ]; then + output="$results/ssb_sf30" + data="$root/data/ssb_sf30" + script=benchmark/ssb/run.sh + fi + if [ -e "$output" ]; then + test ! -e "${output}_failed_setup" + mv "$output" "${output}_failed_setup" + fi + date -u + if SSB_MEMORY_LIMIT=30GB bash "$script" \ + "$data" "$output" 16 5 > "$results/repro/${workload}_retry.log" 2>&1; then + printf '%s_retry,0\n' "$workload" >> "$results/repro/stage_status.csv" + else + code=$? + printf '%s_retry,%s\n' "$workload" "$code" >> "$results/repro/stage_status.csv" + failures=$((failures+1)) + tail -25 "$results/repro/${workload}_retry.log" + fi +done +test "$failures" = 0 diff --git a/benchmark/ssb/as_ssb_benchmark.cpp b/benchmark/ssb/as_ssb_benchmark.cpp index ec223c73..aa148ca6 100644 --- a/benchmark/ssb/as_ssb_benchmark.cpp +++ b/benchmark/ssb/as_ssb_benchmark.cpp @@ -267,8 +267,8 @@ int main(int argc, char **argv) { double import_ms = Ms(start); Query(con, "CHECKPOINT"); // Explicit column protection permits grouping by customer city/nation while retaining key protection. - Query(con, "ALTER TABLE customer ADD PRIVACY_KEY(c_custkey); ALTER TABLE customer SET PU; " - "ALTER TABLE customer ADD PROTECTED(c_custkey); " + Query(con, "ALTER TABLE customer ADD PRIVACY_KEY(c_custkey); " + "ALTER TABLE customer ADD PROTECTED(c_custkey); ALTER TABLE customer SET PU; " "ALTER TABLE lineorder ADD PRIVACY_LINK(lo_custkey) REFERENCES customer(c_custkey)"); for (auto name : {"q11", "q12", "q13", "q21", "q22", "q23", "q31", "q32", "q33", "q34", "q41", "q42", "q43"}) { string sql = Read(ssb + "/queries/" + name + ".sql"); From 2dc137b1000cc4add4f5474e4b8650dda8e1dc19 Mon Sep 17 00:00:00 2001 From: ila Date: Tue, 10 Nov 2026 22:06:51 +0100 Subject: [PATCH 25/32] Fix microbenchmark profiling filenames and DuckDB 2.0 JSON parsing --- benchmark/microbench/common.sh | 112 ++++++++++++++++----------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/benchmark/microbench/common.sh b/benchmark/microbench/common.sh index 21e10ec1..d73adc7a 100755 --- a/benchmark/microbench/common.sh +++ b/benchmark/microbench/common.sh @@ -22,6 +22,38 @@ median_seconds() { sort -n | awk '{a[NR]=$1} END {if (NR % 2) print a[(NR+1)/2]; else if (NR) print (a[NR/2]+a[NR/2+1])/2}' } +aggregate_profile_seconds() { + python3 - "$1" <<'PY' +import json +import sys + +with open(sys.argv[1]) as file: + profile = json.load(file) + +def aggregate_time(node): + kind = node.get("operator_type", node.get("type", "")) + matched = any(name in kind for name in ("UNGROUPED", "GROUP_BY", "AGGREGATE")) + seconds = node.get("operator_timing", node.get("timing", 0.0)) if matched else 0.0 + count = int(matched) + for child in node.get("children", []): + child_seconds, child_count = aggregate_time(child) + seconds += child_seconds + count += child_count + return seconds, count + +seconds = 0.0 +count = 0 +# DuckDB 2.0 places the operator tree under "operator" and renames its fields. +for root in profile.get("operator", [profile]): + root_seconds, root_count = aggregate_time(root) + seconds += root_seconds + count += root_count +if not count: + raise ValueError("No aggregate operator found in the query profile") +print(f"{seconds:.3f}") +PY +} + # Run a benchmark query and extract timing # Args: binary, query, [runs] # Returns: wall_median,agg_median,wall_times,agg_times @@ -37,13 +69,14 @@ run_bench() { # Warmup for ((i=0; i/dev/null 2>&1 + "$binary" -c "$QUERY_SETUP $query" >/dev/null 2>> "$RESULTS_DIR/errors.log" done # Benchmark runs with profiling local wall_times=() local agg_times=() - local profile_file=$(mktemp) + local profile_dir=$(mktemp -d) + local profile_file="$profile_dir/profile.json" local query_failed=0 local timeout_threshold=25.0 @@ -51,10 +84,10 @@ run_bench() { local start=$(python3 -c 'import time; print(time.time())') if ! "$binary" -c " $QUERY_SETUP - PRAGMA enable_profiling='json'; - PRAGMA profiling_output='$profile_file'; + SET enable_profiling='json'; + SET profiling_output='$profile_file'; $query - " >/dev/null 2>&1; then + " >/dev/null 2>> "$RESULTS_DIR/errors.log"; then query_failed=1 break fi @@ -62,29 +95,11 @@ run_bench() { local elapsed=$(python3 -c "print(f'{$end - $start:.3f}')") wall_times+=("$elapsed") - # Extract aggregate operator time from JSON profile - local agg_time=$(python3 -c " -import json -import sys -try: - with open('$profile_file') as f: - data = json.load(f) - - def find_agg_time(node): - total = 0.0 - op_type = node.get('operator_type', '') - # Match aggregate operators - if 'UNGROUPED' in op_type or 'GROUP_BY' in op_type or 'AGGREGATE' in op_type: - total += node.get('operator_timing', 0.0) - for child in node.get('children', []): - total += find_agg_time(child) - return total - - agg_sec = find_agg_time(data) - print(f'{agg_sec:.3f}') -except: - print('0.000') -" 2>/dev/null) + local agg_time + if ! agg_time=$(aggregate_profile_seconds "$profile_file" 2>> "$RESULTS_DIR/errors.log"); then + query_failed=1 + break + fi agg_times+=("$agg_time") # Stop repeating if query took longer than threshold @@ -94,6 +109,7 @@ except: done rm -f "$profile_file" + rmdir "$profile_dir" # If any benchmark run failed, return error indicator if [[ $query_failed -eq 1 ]]; then @@ -204,7 +220,7 @@ run_bench_db() { # Warmup - check if query succeeds local warmup_failed=0 for ((i=0; i/dev/null 2>&1; then + if ! "$binary" "$TEST_DB" -c "$QUERY_SETUP $query" >/dev/null 2>> "$RESULTS_DIR/errors.log"; then warmup_failed=1 break fi @@ -219,7 +235,8 @@ run_bench_db() { # Benchmark runs with profiling local wall_times=() local agg_times=() - local profile_file=$(mktemp) + local profile_dir=$(mktemp -d) + local profile_file="$profile_dir/profile.json" local query_failed=0 local timeout_threshold=25.0 @@ -227,10 +244,10 @@ run_bench_db() { local start=$(python3 -c 'import time; print(time.time())') if ! "$binary" "$TEST_DB" -c " $QUERY_SETUP - PRAGMA enable_profiling='json'; - PRAGMA profiling_output='$profile_file'; + SET enable_profiling='json'; + SET profiling_output='$profile_file'; $query - " >/dev/null 2>&1; then + " >/dev/null 2>> "$RESULTS_DIR/errors.log"; then query_failed=1 break fi @@ -238,29 +255,11 @@ run_bench_db() { local elapsed=$(python3 -c "print(f'{$end - $start:.3f}')") wall_times+=("$elapsed") - # Extract aggregate operator time from JSON profile - local agg_time=$(python3 -c " -import json -import sys -try: - with open('$profile_file') as f: - data = json.load(f) - - def find_agg_time(node): - total = 0.0 - op_type = node.get('operator_type', '') - # Match aggregate operators - if 'UNGROUPED' in op_type or 'GROUP_BY' in op_type or 'AGGREGATE' in op_type: - total += node.get('operator_timing', 0.0) - for child in node.get('children', []): - total += find_agg_time(child) - return total - - agg_sec = find_agg_time(data) - print(f'{agg_sec:.3f}') -except: - print('0.000') -" 2>/dev/null) + local agg_time + if ! agg_time=$(aggregate_profile_seconds "$profile_file" 2>> "$RESULTS_DIR/errors.log"); then + query_failed=1 + break + fi agg_times+=("$agg_time") # Stop repeating if query took longer than threshold @@ -270,6 +269,7 @@ except: done rm -f "$profile_file" + rmdir "$profile_dir" # If any benchmark run failed, return error indicator if [[ $query_failed -eq 1 ]]; then From 72f8fc5dab654e2256efd5cd722525f7107778c6 Mon Sep 17 00:00:00 2001 From: ila Date: Tue, 10 Nov 2026 22:59:28 +0100 Subject: [PATCH 26/32] Update clustered SUM directly from input selections --- src/aggregates/as_sum.cpp | 91 ++++++++++++++++++++++----------------- test/sql/pac_sum.test | 17 ++++++++ 2 files changed, 68 insertions(+), 40 deletions(-) diff --git a/src/aggregates/as_sum.cpp b/src/aggregates/as_sum.cpp index 2a3fe11d..ead72feb 100644 --- a/src/aggregates/as_sum.cpp +++ b/src/aggregates/as_sum.cpp @@ -289,27 +289,29 @@ inline void PacSumUpdateOne(PacSumIntStateWrapper &agg, uint64_t key_has #endif // PAC_NOBUFFERING -// PacSumUpdate - unified for both int and double states -template -static void PacSumUpdate(Vector inputs[], State &state, idx_t count, ArenaAllocator &allocator, int sample_lanes = 0) { - UnifiedVectorFormat hash_data, value_data; - inputs[0].ToUnifiedFormat(hash_data); - inputs[1].ToUnifiedFormat(value_data); +// Update a single state from the original input vectors and an optional group selection. +template +static void PacSumUpdateValues(const UnifiedVectorFormat &hash_data, const UnifiedVectorFormat &value_data, + State &state, idx_t count, ArenaAllocator &allocator, int sample_lanes, + const sel_t *selection) { auto hashes = UnifiedVectorFormat::GetData(hash_data); auto values = UnifiedVectorFormat::GetData(value_data); if (hash_data.validity.CannotHaveNull() && value_data.validity.CannotHaveNull()) { for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); + auto row = SELECTED ? selection[i] : i; + auto h_idx = hash_data.sel->get_index(row); + auto v_idx = value_data.sel->get_index(row); PacSumUpdateOne(state, TransformPacUpdateHash(HASH_TRANSFORM::Transform(hashes[h_idx]), sample_lanes), ConvertValue::convert(values[v_idx]), allocator); } } else { for (idx_t i = 0; i < count; i++) { - auto h_idx = hash_data.sel->get_index(i); - auto v_idx = value_data.sel->get_index(i); + auto row = SELECTED ? selection[i] : i; + auto h_idx = hash_data.sel->get_index(row); + auto v_idx = value_data.sel->get_index(row); if (!hash_data.validity.RowIsValid(h_idx) || !value_data.validity.RowIsValid(v_idx)) { continue; } @@ -320,6 +322,33 @@ static void PacSumUpdate(Vector inputs[], State &state, idx_t count, ArenaAlloca } } +template +static void PacSumUpdate(Vector inputs[], State &state, idx_t count, ArenaAllocator &allocator, int sample_lanes = 0) { + UnifiedVectorFormat hash_data, value_data; + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + PacSumUpdateValues( + hash_data, value_data, state, count, allocator, sample_lanes, nullptr); +} + +template +static void PacSumUpdateClusters(Vector inputs[], const ClusteredAggr &clusters, ArenaAllocator &allocator, + int sample_lanes) { + UnifiedVectorFormat hash_data, value_data; + inputs[0].ToUnifiedFormat(hash_data); + inputs[1].ToUnifiedFormat(value_data); + for (auto &run : clusters.runs()) { + auto &state = *reinterpret_cast(run.state); + if (run.sel) { + PacSumUpdateValues(hash_data, value_data, state, run.count, + allocator, sample_lanes, run.sel); + } else { + PacSumUpdateValues(hash_data, value_data, state, run.count, + allocator, sample_lanes, nullptr); + } + } +} + template static void PacSumScatterUpdate(Vector inputs[], Vector &states, idx_t count, ArenaAllocator &allocator, int sample_lanes = 0) { @@ -573,11 +602,8 @@ void PacSumFinalize(Vector &states, AggregateFinalizeInputData &input, Vector &r #define X(NAME, VALUE_T, INPUT_T, SIGNED) \ void PacSumUpdate##NAME(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, \ const ClusteredAggr &clusters, idx_t count) { \ - ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { \ - auto &state = *reinterpret_cast *>(state_p); \ - PacSumUpdate, SIGNED, VALUE_T, INPUT_T>( \ - inputs, state, count, aggr_input_data.allocator, GetPrivSampleLanes(aggr_input_data)); \ - }); \ + PacSumUpdateClusters, SIGNED, VALUE_T, INPUT_T>( \ + inputs, clusters, aggr_input_data.allocator, GetPrivSampleLanes(aggr_input_data)); \ } \ void PacSumScatterUpdate##NAME(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, Vector &states, \ idx_t count) { \ @@ -591,35 +617,23 @@ PAC_INT_TYPES_UNSIGNED // Double/Float/[U]HugeInt use double state (not generated by X-macro) void PacSumUpdateHugeIntDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, const ClusteredAggr &clusters, idx_t count) { - ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { - auto &state = *reinterpret_cast(state_p); - PacSumUpdate(inputs, state, count, aggr_input_data.allocator, - GetPrivSampleLanes(aggr_input_data)); - }); + PacSumUpdateClusters(inputs, clusters, aggr_input_data.allocator, + GetPrivSampleLanes(aggr_input_data)); } void PacSumUpdateUHugeInt(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, const ClusteredAggr &clusters, idx_t count) { - ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { - auto &state = *reinterpret_cast(state_p); - PacSumUpdate(inputs, state, count, aggr_input_data.allocator, - GetPrivSampleLanes(aggr_input_data)); - }); + PacSumUpdateClusters(inputs, clusters, aggr_input_data.allocator, + GetPrivSampleLanes(aggr_input_data)); } void PacSumUpdateFloat(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, const ClusteredAggr &clusters, idx_t count) { - ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { - auto &state = *reinterpret_cast(state_p); - PacSumUpdate(inputs, state, count, aggr_input_data.allocator, - GetPrivSampleLanes(aggr_input_data)); - }); + PacSumUpdateClusters(inputs, clusters, aggr_input_data.allocator, + GetPrivSampleLanes(aggr_input_data)); } void PacSumUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, const ClusteredAggr &clusters, idx_t count) { - ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { - auto &state = *reinterpret_cast(state_p); - PacSumUpdate(inputs, state, count, aggr_input_data.allocator, - GetPrivSampleLanes(aggr_input_data)); - }); + PacSumUpdateClusters(inputs, clusters, aggr_input_data.allocator, + GetPrivSampleLanes(aggr_input_data)); } void PacSumScatterUpdateHugeIntDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, Vector &states, idx_t count) { @@ -644,11 +658,8 @@ void PacSumScatterUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_d static void DpSampleSumUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, const ClusteredAggr &clusters, idx_t count) { - ExecutePrivClusters(inputs, input_count, clusters, [&](Vector inputs[], data_ptr_t state_p, idx_t count) { - auto &state = *reinterpret_cast(state_p); - PacSumUpdate(inputs, state, count, aggr_input_data.allocator, - GetPrivSampleLanes(aggr_input_data)); - }); + PacSumUpdateClusters( + inputs, clusters, aggr_input_data.allocator, GetPrivSampleLanes(aggr_input_data)); } static void DpSampleSumScatterUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, Vector &states, diff --git a/test/sql/pac_sum.test b/test/sql/pac_sum.test index c3cd83fd..fdcf0bb0 100644 --- a/test/sql/pac_sum.test +++ b/test/sql/pac_sum.test @@ -668,6 +668,23 @@ FROM s,range(2)t(i) ORDER BY i true true +# Clustered updates reuse their selected views across groups. Exercise a partial +# chunk, nullable values, dictionary selections, and constant hash/value inputs. +query II +SELECT count(*),count(*) FILTER (WHERE list_extract(s,1)=2*expected + AND list_min(k)=2*3*n AND list_max(k)=2*3*n AND list_min(z)=0 AND list_max(z)=0) +FROM ( + SELECT i%13 g, + as_sum(CASE WHEN i%2=0 THEN 18446744073709551615::UBIGINT ELSE 1::UBIGINT END, + CASE WHEN i%5=0 THEN NULL ELSE (i%101)::INTEGER END) s, + as_sum(18446744073709551615::UBIGINT,3::INTEGER) k, + as_sum(18446744073709551615::UBIGINT,NULL::INTEGER) z, + sum(CASE WHEN i%5=0 THEN NULL ELSE (i%101)::INTEGER END) expected,count(*) n + FROM range(10000)t(i) WHERE i%7<>0 GROUP BY g +) +---- +13 13 + statement ok SET pac_sample_diversity_check=true From f122d3ba239a289a95c0c2d70016f55f31408d30 Mon Sep 17 00:00:00 2001 From: ila Date: Wed, 11 Nov 2026 11:24:26 +0100 Subject: [PATCH 27/32] Avoid redundant correlation and joins in TPC-H AS queries Keep Q20 nation filtering in the outer query, matching the control formulation. Drop Q21 inner orders joins: the outer orders join already provides a matching row for every correlated order key. Both transformations preserve filtered output, including NULL and duplicate-key cases. Verified exact SF30 result equivalence with noise disabled at 1 and 16 threads. Five measured runs at 16 threads cover m=64,128,256,512 on both DuckDB 1.5 and 2.0; Q20 no longer exhausts memory at larger m. --- benchmark/tpch/tpch_as_queries/q20.sql | 4 ++-- benchmark/tpch/tpch_as_queries/q21.sql | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/benchmark/tpch/tpch_as_queries/q20.sql b/benchmark/tpch/tpch_as_queries/q20.sql index 0e7aba4a..645b754c 100755 --- a/benchmark/tpch/tpch_as_queries/q20.sql +++ b/benchmark/tpch/tpch_as_queries/q20.sql @@ -6,6 +6,6 @@ SELECT s_name, s_address AND priv_filter_gt(ps_availqty * 2, (SELECT as_sum(priv_hash(hash(o_custkey)), l_quantity) FROM lineitem JOIN orders ON l_orderkey = o_orderkey WHERE l_partkey = ps_partkey AND l_suppkey = ps_suppkey - AND l_shipdate >= DATE '1994-01-01' AND l_shipdate < DATE '1995-01-01')) - AND n_name = 'CANADA') + AND l_shipdate >= DATE '1994-01-01' AND l_shipdate < DATE '1995-01-01'))) + AND n_name = 'CANADA' ORDER BY s_name; diff --git a/benchmark/tpch/tpch_as_queries/q21.sql b/benchmark/tpch/tpch_as_queries/q21.sql index fedc2c37..a70ed23f 100755 --- a/benchmark/tpch/tpch_as_queries/q21.sql +++ b/benchmark/tpch/tpch_as_queries/q21.sql @@ -3,9 +3,10 @@ SELECT s_name, as_noised_count(priv_hash(hash(o_custkey))) AS numwait JOIN orders ON o_orderkey = l1.l_orderkey JOIN nation ON s_nationkey = n_nationkey WHERE o_orderstatus = 'F' AND l1.l_receiptdate > l1.l_commitdate AND n_name = 'SAUDI ARABIA' - AND EXISTS (SELECT 1 FROM lineitem l2 JOIN orders o2 ON l2.l_orderkey = o2.o_orderkey + -- The outer orders join supplies an order row for both correlated existence checks. + AND EXISTS (SELECT 1 FROM lineitem l2 WHERE l2.l_orderkey = l1.l_orderkey AND l2.l_suppkey <> l1.l_suppkey) - AND NOT EXISTS (SELECT 1 FROM lineitem l3 JOIN orders o3 ON l3.l_orderkey = o3.o_orderkey + AND NOT EXISTS (SELECT 1 FROM lineitem l3 WHERE l3.l_orderkey = l1.l_orderkey AND l3.l_suppkey <> l1.l_suppkey AND l3.l_receiptdate > l3.l_commitdate) From 895a563be5fcb6cdadf8f343690f64a8067b7465 Mon Sep 17 00:00:00 2001 From: ila Date: Wed, 11 Nov 2026 14:55:46 +0100 Subject: [PATCH 28/32] Preserve scalar noise independence and optimize AS correlation plans --- CMakeLists.txt | 1 + benchmark/tpch/tpch_as_queries/q21.sql | 5 +- src/core/privacy_extension.cpp | 3 + .../as_correlation_optimizer.hpp | 8 + .../as_correlation_optimizer.cpp | 469 ++++++++++++++++++ test/sql/pac_categorical_subqueries.test | 76 +++ test/sql/pac_noise_independence.test | 111 +++++ 7 files changed, 670 insertions(+), 3 deletions(-) create mode 100644 src/include/query_processing/as_correlation_optimizer.hpp create mode 100644 src/query_processing/as_correlation_optimizer.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f1509fd8..6635e89b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,7 @@ set(EXTENSION_SOURCES src/compiler/privacy_mechanisms.cpp src/compiler/privacy_compiler_helpers.cpp src/query_processing/pac_avg_rewriter.cpp + src/query_processing/as_correlation_optimizer.cpp src/query_processing/privacy_expression_builder.cpp src/query_processing/privacy_plan_traversal.cpp src/query_processing/pac_projection_propagation.cpp diff --git a/benchmark/tpch/tpch_as_queries/q21.sql b/benchmark/tpch/tpch_as_queries/q21.sql index a70ed23f..fedc2c37 100755 --- a/benchmark/tpch/tpch_as_queries/q21.sql +++ b/benchmark/tpch/tpch_as_queries/q21.sql @@ -3,10 +3,9 @@ SELECT s_name, as_noised_count(priv_hash(hash(o_custkey))) AS numwait JOIN orders ON o_orderkey = l1.l_orderkey JOIN nation ON s_nationkey = n_nationkey WHERE o_orderstatus = 'F' AND l1.l_receiptdate > l1.l_commitdate AND n_name = 'SAUDI ARABIA' - -- The outer orders join supplies an order row for both correlated existence checks. - AND EXISTS (SELECT 1 FROM lineitem l2 + AND EXISTS (SELECT 1 FROM lineitem l2 JOIN orders o2 ON l2.l_orderkey = o2.o_orderkey WHERE l2.l_orderkey = l1.l_orderkey AND l2.l_suppkey <> l1.l_suppkey) - AND NOT EXISTS (SELECT 1 FROM lineitem l3 + AND NOT EXISTS (SELECT 1 FROM lineitem l3 JOIN orders o3 ON l3.l_orderkey = o3.o_orderkey WHERE l3.l_orderkey = l1.l_orderkey AND l3.l_suppkey <> l1.l_suppkey AND l3.l_receiptdate > l3.l_commitdate) diff --git a/src/core/privacy_extension.cpp b/src/core/privacy_extension.cpp index 0afc2849..7c54ef48 100644 --- a/src/core/privacy_extension.cpp +++ b/src/core/privacy_extension.cpp @@ -32,6 +32,7 @@ #include "table_functions/dp_sass_stability_query.hpp" #include "query_processing/pac_topk_rewriter.hpp" #include "query_processing/pac_avg_rewriter.hpp" +#include "query_processing/as_correlation_optimizer.hpp" #include "privacy_debug.hpp" namespace duckdb { @@ -287,6 +288,8 @@ static void LoadInternal(ExtensionLoader &loader) { #endif } + RegisterASCorrelationOptimizer(db); + // Register PAC optimizer rule auto priv_rewrite_rule = PACRewriteRule(); // attach PAC-specific optimizer info so the extension can coordinate replan state diff --git a/src/include/query_processing/as_correlation_optimizer.hpp b/src/include/query_processing/as_correlation_optimizer.hpp new file mode 100644 index 00000000..b31f759c --- /dev/null +++ b/src/include/query_processing/as_correlation_optimizer.hpp @@ -0,0 +1,8 @@ +#pragma once + +namespace duckdb { +class DatabaseInstance; + +void RegisterASCorrelationOptimizer(DatabaseInstance &db); + +} // namespace duckdb diff --git a/src/query_processing/as_correlation_optimizer.cpp b/src/query_processing/as_correlation_optimizer.cpp new file mode 100644 index 00000000..43be87ac --- /dev/null +++ b/src/query_processing/as_correlation_optimizer.cpp @@ -0,0 +1,469 @@ +#include "query_processing/as_correlation_optimizer.hpp" + +#include "duckdb/common/enums/optimizer_type.hpp" +#include "duckdb/execution/operator/join/physical_comparison_join.hpp" +#include "duckdb/execution/expression_executor.hpp" +#include "duckdb/function/builtin_function_lookup.hpp" +#include "duckdb/function/function_binder.hpp" +#include "duckdb/main/config.hpp" +#include "duckdb/main/settings.hpp" +#include "duckdb/optimizer/column_binding_replacer.hpp" +#include "duckdb/optimizer/join_filter_pushdown_optimizer.hpp" +#include "duckdb/optimizer/optimizer.hpp" +#include "duckdb/optimizer/optimizer_extension.hpp" +#include "duckdb/planner/expression/bound_aggregate_expression.hpp" +#include "duckdb/planner/expression/bound_columnref_expression.hpp" +#include "duckdb/planner/expression/bound_constant_expression.hpp" +#include "duckdb/planner/expression/bound_cast_expression.hpp" +#include "duckdb/planner/expression/bound_function_expression.hpp" +#include "duckdb/planner/expression/expression_barrier.hpp" +#include "duckdb/planner/expression_iterator.hpp" +#include "duckdb/planner/logical_operator_visitor.hpp" +#include "duckdb/planner/operator/logical_comparison_join.hpp" +#include "duckdb/planner/operator/logical_aggregate.hpp" +#include "duckdb/planner/operator/logical_dependent_join.hpp" +#include "duckdb/planner/operator/logical_filter.hpp" +#include "duckdb/planner/operator/logical_get.hpp" +#include "duckdb/planner/operator/logical_projection.hpp" +#include "duckdb/planner/planner_extension.hpp" +#include "privacy_debug.hpp" + +namespace duckdb { + +static bool ContainsASAggregate(LogicalOperator &op) { + bool found = false; + LogicalOperatorVisitor::EnumerateExpressions(op, [&](unique_ptr *expr) { + ExpressionIterator::VisitExpression(**expr, [&](const BoundAggregateExpression &agg) { + found |= StringUtil::StartsWith(agg.Function().GetName().GetIdentifierName(), "as_"); + }); + }); + for (auto &child : op.children) { + found |= ContainsASAggregate(*child); + } + return found; +} + +static bool ContainsReorderingBarrier(LogicalOperator &op) { + bool found = false; + LogicalOperatorVisitor::EnumerateExpressions(op, [&](unique_ptr *expr) { + found |= (*expr)->IsVolatile() || ExpressionBarrier::Contains(**expr); + }); + for (auto &child : op.children) { + found |= ContainsReorderingBarrier(*child); + } + return found; +} + +// The outer filter may still be one AND expression containing the noisy +// predicate. Separate safe predicates on passthrough columns before the input +// becomes a materialized correlation domain, so noise is drawn only after them. +static void PushCorrelatedInputFilters(ClientContext &context, LogicalFilter &filter) { + if (filter.children[0]->type != LogicalOperatorType::LOGICAL_PROJECTION) { + return; + } + auto &projection = filter.children[0]->Cast(); + if (projection.children[0]->type != LogicalOperatorType::LOGICAL_DEPENDENT_JOIN || projection.expressions.empty() || + projection.expressions[0]->GetExpressionClass() != ExpressionClass::BOUND_FUNCTION || + !projection.expressions[0]->IsVolatile()) { + return; + } + auto release_name = + projection.expressions[0]->Cast().Function().GetName().GetIdentifierName(); + if (release_name != "as_noised" && release_name != "as_noised_div") { + return; + } + auto &join = projection.children[0]->Cast(); + if (join.join_type != JoinType::SINGLE || !join.perform_delim || join.HasProjectionMap()) { + return; + } + auto input_bindings = join.children[0]->GetColumnBindings(); + filter.SplitPredicates(); + vector> remaining; + vector> selected; + for (auto &expression : filter.expressions) { + // Literal casts have not passed through constant folding at post-bind. + // Evaluate only these casts before checking whether a predicate can throw. + ExpressionIterator::EnumerateExpression(expression, [&](unique_ptr &node) { + if (BoundCastExpression::IsCast(*node) && node->IsFoldable() && + BoundCastExpression::Child(node->Cast()).GetExpressionClass() == + ExpressionClass::BOUND_CONSTANT) { + Value value; + if (ExpressionExecutor::TryEvaluateScalar(context, *node, value)) { + node = make_uniq(std::move(value)); + } + } + }); + bool safe = !expression->IsVolatile() && !expression->CanThrow() && !ExpressionBarrier::Contains(*expression); + ExpressionIterator::VisitExpression( + *expression, [&](const BoundColumnRefExpression &ref) { + if (ref.Depth() != 0 || ref.Binding().table_index != projection.table_index || + projection.GetExpression(ref.Binding()).GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF) { + safe = false; + return; + } + auto binding = projection.GetExpression(ref.Binding()).Cast().Binding(); + safe &= std::find(input_bindings.begin(), input_bindings.end(), binding) != input_bindings.end(); + }); + if (!safe) { + remaining.push_back(std::move(expression)); + continue; + } + ExpressionIterator::VisitExpressionMutable( + expression, [&](BoundColumnRefExpression &ref, unique_ptr &expr) { + expr = projection.GetExpression(ref.Binding()).Copy(); + }); + selected.push_back(std::move(expression)); + } + filter.expressions = std::move(remaining); + if (!selected.empty()) { + auto input_filter = make_uniq(); + input_filter->expressions = std::move(selected); + input_filter->children.push_back(std::move(join.children[0])); + join.children[0] = std::move(input_filter); + PRIVACY_DEBUG_PRINT("[AS] Applied deterministic outer filters before shared counter aggregation"); + } +} + +// A volatile scalar release belongs to each outer row, but the counter +// aggregates underneath it are deterministic and can be shared by correlation +// key. Move only the release above SINGLE before DuckDB chooses its domain. +static void LiftCorrelatedNoise(PlannerExtensionInput &input, unique_ptr &op, + BindingReplacementGraph &replacements) { + for (idx_t i = 0; i < op->children.size(); i++) { + BindingReplacementGraph child_replacements; + ColumnBindingRewrite::RewriteChild( + op, i, [&](unique_ptr &child, BindingReplacementGraph &child_changes) { + LiftCorrelatedNoise(input, child, child_changes); + child_replacements.Merge(child_changes); + }); + replacements.Merge(child_replacements); + } + if (op->type == LogicalOperatorType::LOGICAL_FILTER) { + PushCorrelatedInputFilters(input.context, op->Cast()); + } + if (op->type != LogicalOperatorType::LOGICAL_DEPENDENT_JOIN || op->HasProjectionMap()) { + return; + } + auto &join = op->Cast(); + if (join.join_type != JoinType::SINGLE || join.condition || join.correlated_columns.empty() || + join.children[1]->type != LogicalOperatorType::LOGICAL_PROJECTION) { + return; + } + auto &projection = join.children[1]->Cast(); + if (projection.expressions.size() != 1 || + projection.expressions[0]->GetExpressionClass() != ExpressionClass::BOUND_FUNCTION || + projection.children[0]->type != LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY || + !projection.children[0]->Cast().groups.empty() || + ContainsReorderingBarrier(*projection.children[0])) { + return; + } + auto &release = projection.expressions[0]->Cast(); + auto name = release.Function().GetName().GetIdentifierName(); + if ((name != "as_noised" && name != "as_noised_div") || + release.Function().GetStability() != FunctionStability::VOLATILE) { + return; + } + for (auto &argument : release.GetChildren()) { + if (argument->IsVolatile() || ExpressionBarrier::Contains(*argument)) { + return; + } + bool correlated = false; + ExpressionIterator::VisitExpression( + *argument, [&](const BoundColumnRefExpression &ref) { correlated |= ref.Depth() != 0; }); + if (correlated) { + return; + } + } + CorrelatedColumns shared_columns; + for (idx_t i = 0; i < join.correlated_columns.size(); i++) { + if (!join.perform_delim && i == join.correlated_columns.GetDelimIndex()) { + continue; + } + auto &column = join.correlated_columns[i]; + // Keep DuckDB's row-identity strategy for nested, non-deduplicable keys. + if (column.type.IsNested()) { + return; + } + shared_columns.AddColumnToBack(column); + } + if (shared_columns.empty()) { + return; + } + join.ResolveOperatorTypes(); + auto left_bindings = join.children[0]->GetColumnBindings(); + auto release_index = projection.table_index; + projection.table_index = input.binder.GenerateTableIndex(); + auto expression = std::move(projection.expressions[0]); + projection.expressions = std::move(expression->Cast().GetChildrenMutable()); + for (idx_t i = 0; i < projection.expressions.size(); i++) { + expression->Cast().GetChildrenMutable().push_back(make_uniq( + projection.expressions[i]->GetReturnType(), ColumnBinding(projection.table_index, ProjectionIndex(i)))); + } + vector> expressions; + expressions.push_back(std::move(expression)); + for (idx_t i = 0; i < left_bindings.size(); i++) { + expressions.push_back(make_uniq(join.children[0]->types[i], left_bindings[i])); + replacements.Add(left_bindings[i], ColumnBinding(release_index, ProjectionIndex(i + 1))); + } + join.correlated_columns = std::move(shared_columns); + join.perform_delim = true; + auto lifted = make_uniq(release_index, std::move(expressions)); + lifted->children.push_back(std::move(op)); + op = std::move(lifted); + op->ResolveOperatorTypes(); + PRIVACY_DEBUG_PRINT( + "[AS] Shared deterministic correlated counters while retaining one volatile release per outer row"); +} + +static void ShareCorrelatedNoiseInputs(PlannerExtensionInput &input, BoundStatement &statement) { + if (!statement.plan || !Settings::Get(input.context) || + DBConfig::GetConfig(input.context).IsOptimizerDisabled(OptimizerType::EXTENSION) || + !ContainsASAggregate(*statement.plan)) { + return; + } + BindingReplacementGraph replacements; + LiftCorrelatedNoise(input, statement.plan, replacements); +} + +// Unlike an ordinary anti join, RIGHT_ANTI preserves the build side. Probe rows +// with no possible build-side match may therefore be filtered without changing +// any output row. Carry that restriction to both sides of an inner ownership join. +static void GetOwnershipFilterTargets(LogicalOperator &op, vector columns, + vector &targets) { + if (columns.empty()) { + return; + } + if (op.type == LogicalOperatorType::LOGICAL_PROJECTION) { + auto &projection = op.Cast(); + vector projected; + for (auto &column : columns) { + if (column.probe_column_index.table_index == projection.table_index && + JoinFilterPushdownUtil::PushdownJoinFilterExpression( + projection.GetExpression(column.probe_column_index), column)) { + projected.push_back(std::move(column)); + } + } + if (!projected.empty()) { + GetOwnershipFilterTargets(*op.children[0], std::move(projected), targets); + } + return; + } + if (op.type == LogicalOperatorType::LOGICAL_FILTER) { + GetOwnershipFilterTargets(*op.children[0], std::move(columns), targets); + return; + } + if (op.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + auto &join = op.Cast(); + if (join.join_type != JoinType::INNER) { + return; + } + for (auto &condition : join.conditions) { + if (!condition.IsComparison() || condition.GetComparisonType() != ExpressionType::COMPARE_EQUAL || + condition.GetLHS().GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF || + condition.GetRHS().GetExpressionClass() != ExpressionClass::BOUND_COLUMN_REF || + condition.GetLHS().GetReturnType() != condition.GetRHS().GetReturnType()) { + continue; + } + auto left = condition.GetLHS().Cast().Binding(); + auto right = condition.GetRHS().Cast().Binding(); + auto count = columns.size(); + for (idx_t i = 0; i < count; i++) { + if (columns[i].probe_column_index != left && columns[i].probe_column_index != right) { + continue; + } + auto other = columns[i]; + other.probe_column_index = other.probe_column_index == left ? right : left; + bool present = false; + for (auto &column : columns) { + present |= column.probe_column_index == other.probe_column_index && + column.join_filter_idx == other.join_filter_idx; + } + if (!present) { + columns.push_back(std::move(other)); + } + } + } + for (auto &child : op.children) { + auto bindings = child->GetColumnBindings(); + vector child_columns; + for (auto &column : columns) { + for (auto &binding : bindings) { + if (binding == column.probe_column_index) { + child_columns.push_back(column); + break; + } + } + } + if (!child_columns.empty()) { + GetOwnershipFilterTargets(*child, std::move(child_columns), targets); + } + } + return; + } + if (op.type == LogicalOperatorType::LOGICAL_GET) { + JoinFilterPushdownOptimizer::GetPushdownFilterTargets(op, std::move(columns), targets); + } +} + +// A scan on the build side can finish before its parent anti join publishes +// runtime filters. Keep a wide, weakly filtered scan on the probe side when a +// smaller key-only source can build the ownership lookup instead. +static idx_t ScanCardinality(LogicalOperator &op) { + if (op.type == LogicalOperatorType::LOGICAL_GET) { + return op.has_estimated_cardinality ? op.estimated_cardinality : 0; + } + if ((op.type == LogicalOperatorType::LOGICAL_PROJECTION || op.type == LogicalOperatorType::LOGICAL_FILTER) && + op.children.size() == 1) { + return ScanCardinality(*op.children[0]); + } + return 0; +} + +static void PrepareOwnershipProbe(OptimizerExtensionInput &input, unique_ptr &op) { + if (op->type == LogicalOperatorType::LOGICAL_PROJECTION || op->type == LogicalOperatorType::LOGICAL_FILTER) { + PrepareOwnershipProbe(input, op->children[0]); + return; + } + if (op->type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + return; + } + auto &join = op->Cast(); + if (join.join_type != JoinType::INNER || join.children[0]->types.size() != 1 || + join.children[1]->types.size() <= 1 || !JoinFilterPushdownOptimizer::IsFiltering(join.children[1]) || + ContainsReorderingBarrier(*join.children[0])) { + return; + } + auto left_rows = ScanCardinality(*join.children[0]); + auto right_rows = ScanCardinality(*join.children[1]); + if (!left_rows || left_rows >= right_rows / 2 || !join.children[1]->has_estimated_cardinality || + join.children[1]->estimated_cardinality < left_rows / 2) { + return; + } + auto old_bindings = join.GetColumnBindings(); + std::swap(join.children[0], join.children[1]); + std::swap(join.left_projection_map, join.right_projection_map); + for (auto &condition : join.conditions) { + if (condition.IsComparison()) { + condition.Swap(); + } + } + join.filter_pushdown.reset(); + JoinFilterPushdownOptimizer filters(input.optimizer); + filters.VisitOperator(join); + auto bindings = join.GetColumnBindings(); + auto layout = make_uniq(); + for (auto &old_binding : old_bindings) { + for (idx_t i = 0; i < bindings.size(); i++) { + if (bindings[i] == old_binding) { + layout->projection_map.emplace_back(i); + break; + } + } + } + D_ASSERT(layout->projection_map.size() == old_bindings.size()); + layout->children.push_back(std::move(op)); + op = std::move(layout); + op->ResolveOperatorTypes(); + PRIVACY_DEBUG_PRINT("[AS] Kept the filtered ownership scan on the anti-join probe pipeline"); +} + +static void AddAntiJoinFilters(OptimizerExtensionInput &input, LogicalOperator &op) { + for (auto &child : op.children) { + AddAntiJoinFilters(input, *child); + } + if (op.type != LogicalOperatorType::LOGICAL_COMPARISON_JOIN) { + return; + } + auto &join = op.Cast(); + if (join.join_type != JoinType::RIGHT_ANTI || join.filter_pushdown || + !JoinFilterPushdownOptimizer::IsFiltering(join.children[1]) || ContainsReorderingBarrier(*join.children[0])) { + return; + } + // Null-safe equality can be normalized only with a non-null proof. Without + // that proof, filtering NULL probe keys could remove a real NULL=NULL match. + for (auto &condition : join.conditions) { + if (!condition.IsComparison() || condition.GetComparisonType() != ExpressionType::COMPARE_NOT_DISTINCT_FROM) { + continue; + } + if ((condition.GetLeftStats() && !condition.GetLeftStats()->CanHaveNull()) || + (condition.GetRightStats() && !condition.GetRightStats()->CanHaveNull())) { + condition = + JoinCondition(condition.GetLHS().Copy(), condition.GetRHS().Copy(), ExpressionType::COMPARE_EQUAL); + } + } + PhysicalComparisonJoin::ReorderConditions(join.conditions); + auto info = make_uniq(); + vector columns; + for (idx_t i = 0; i < join.conditions.size(); i++) { + auto &condition = join.conditions[i]; + if (!condition.IsComparison() || condition.GetComparisonType() != ExpressionType::COMPARE_EQUAL) { + continue; + } + JoinFilterPushdownColumn column; + if (!JoinFilterPushdownUtil::PushdownJoinFilterExpression(condition.GetLHS(), column)) { + continue; + } + column.join_filter_idx = info->join_condition.size(); + columns.push_back(std::move(column)); + info->join_condition.push_back(i); + } + if (columns.empty()) { + return; + } + if (!DBConfig::GetConfig(input.context).IsOptimizerDisabled(OptimizerType::BUILD_SIDE_PROBE_SIDE)) { + PrepareOwnershipProbe(input, join.children[0]); + } + vector targets; + GetOwnershipFilterTargets(*join.children[0], std::move(columns), targets); + if (targets.empty()) { + return; + } + for (auto &index : info->join_condition) { + for (auto name : {"min", "max"}) { + vector> arguments; + arguments.push_back(join.conditions[index].GetRHS().Copy()); + auto function = GetBuiltinAggregateFunction(input.context, name, {arguments[0]->GetReturnType()}); + FunctionBinder binder(input.context); + auto aggregate = binder.BindAggregateFunction(std::move(function), std::move(arguments), nullptr, + AggregateType::NON_DISTINCT); + if (aggregate->GetChildren().size() != 1) { + return; + } + info->min_max_aggregates.push_back(std::move(aggregate)); + } + } + for (auto &target : targets) { + auto &get = target.get; + if (!get.dynamic_filters) { + get.dynamic_filters = make_shared_ptr(); + } + JoinFilterPushdownFilter filter; + filter.dynamic_filters = get.dynamic_filters; + filter.columns = std::move(target.columns); + if (get.has_estimated_cardinality) { + filter.estimated_scan_cardinality = get.estimated_cardinality; + } + info->probe_info.push_back(std::move(filter)); + } + info->build_side_has_filter = true; + join.filter_pushdown = std::move(info); + PRIVACY_DEBUG_PRINT("[AS] Pushed right-anti membership keys into ownership join inputs"); +} + +static void OptimizeASCorrelations(OptimizerExtensionInput &input, unique_ptr &plan) { + if (!DBConfig::GetConfig(input.context).IsOptimizerDisabled(OptimizerType::JOIN_FILTER_PUSHDOWN) && + ContainsASAggregate(*plan)) { + AddAntiJoinFilters(input, *plan); + } +} + +void RegisterASCorrelationOptimizer(DatabaseInstance &db) { + PlannerExtension planner; + planner.post_bind_function = ShareCorrelatedNoiseInputs; + PlannerExtension::Register(DBConfig::GetConfig(db), std::move(planner)); + OptimizerExtension extension; + extension.optimize_function = OptimizeASCorrelations; + OptimizerExtension::Register(DBConfig::GetConfig(db), std::move(extension)); +} + +} // namespace duckdb diff --git a/test/sql/pac_categorical_subqueries.test b/test/sql/pac_categorical_subqueries.test index da665632..5e2e7e82 100644 --- a/test/sql/pac_categorical_subqueries.test +++ b/test/sql/pac_categorical_subqueries.test @@ -870,3 +870,79 @@ DROP TABLE scalar_case_reason; statement ok DROP TABLE scalar_case_pu; + + +# Right-anti membership filters must preserve ownership joins, duplicates, +# NULL-safe matches, and the output layout when the build/probe sides change. +# as_count scales each world by two for 50% sampling; divide out that scale. +statement ok +PRAGMA clear_privacy_metadata; + +statement ok +SET priv_rewrite=false; + +statement ok +SET pac_sample_diversity_check=false; + +statement ok +CREATE TABLE anti_owners AS SELECT i::BIGINT k FROM range(1000) r(i); + +statement ok +CREATE TABLE anti_facts AS +SELECT (i % 1000)::BIGINT k, (i % 7)::INTEGER v FROM range(10000) r(i); + +statement ok +CREATE TABLE anti_outer(k BIGINT, v INTEGER); + +statement ok +INSERT INTO anti_outer VALUES (10, 3), (10, 3), (11, 4), (2000, 1), (NULL, 1), (12, 0); + +query I +SELECT list_extract(as_count(3::UBIGINT), 1) / 2 +FROM anti_outer a +WHERE a.v > 0 AND NOT EXISTS ( + SELECT 1 FROM anti_owners o JOIN anti_facts f ON o.k = f.k + WHERE f.k = a.k AND f.v <> a.v AND f.v > 1 +); +---- +2 + +# NULL=NULL must remain a match when neither join key is proven non-null. +statement ok +CREATE TABLE anti_nullable(k BIGINT, v INTEGER); + +statement ok +INSERT INTO anti_nullable VALUES (NULL, 2), (10, 3), (11, 4); + +query I +SELECT list_extract(as_count(3::UBIGINT), 1) / 2 +FROM anti_outer a +WHERE a.v > 0 AND NOT EXISTS ( + SELECT 1 FROM anti_nullable n + WHERE n.k IS NOT DISTINCT FROM a.k AND n.v <> a.v +); +---- +4 + +# A LIMIT is a barrier: filtering below it can change which evidence row exists. +query I +SELECT list_extract(as_count(3::UBIGINT), 1) / 2 +FROM anti_outer a +WHERE a.v > 0 AND NOT EXISTS ( + SELECT 1 FROM (SELECT * FROM anti_nullable ORDER BY k NULLS FIRST LIMIT 1) n + WHERE n.k IS NOT DISTINCT FROM a.k AND n.v <> a.v +); +---- +4 + +statement ok +DROP TABLE anti_nullable; + +statement ok +DROP TABLE anti_outer; + +statement ok +DROP TABLE anti_facts; + +statement ok +DROP TABLE anti_owners; diff --git a/test/sql/pac_noise_independence.test b/test/sql/pac_noise_independence.test index 2e845584..eb939366 100644 --- a/test/sql/pac_noise_independence.test +++ b/test/sql/pac_noise_independence.test @@ -197,3 +197,114 @@ WHERE a.rowid < 1000 AND b.rowid >= 1000 AND a.c IS DISTINCT FROM b.c statement ok DEALLOCATE seeded_count + +# Correlated scalar releases must draw independently for repeated outer keys, +# even when their deterministic aggregate counters are shared. +statement ok +SET priv_rewrite=false + +statement ok +CREATE TABLE correlated_noise_input AS +SELECT 0::INTEGER AS k, i AS e, 1 + i % 7 AS v FROM range(128) t(i) + +statement ok +CREATE TABLE correlated_noise_outer AS +SELECT i AS id, 0::INTEGER AS k, CASE WHEN i % 2 = 0 THEN 'KEEP' ELSE 'DROP' END AS label +FROM range(4096) t(i) + +statement ok +CREATE TABLE correlated_noise_result AS +SELECT o.id, + (SELECT as_noised(as_sum(priv_hash(hash(i.e)), i.v)) + FROM correlated_noise_input i WHERE i.k = o.k) AS s, + (SELECT as_noised_div(as_sum(priv_hash(hash(i.e)), i.v), + as_count(priv_hash(hash(i.e)), i.v)) + FROM correlated_noise_input i WHERE i.k = o.k) AS d +FROM correlated_noise_outer o + +query II +SELECT count(DISTINCT s) > 3000, count(DISTINCT d) > 3000 +FROM correlated_noise_result +---- +true true + +# With noise suppressed, the rewrite must exactly preserve empty inputs, +# NULL correlation keys, duplicate keys, and multiple scalar subqueries. +statement ok +INSERT INTO correlated_noise_outer VALUES (4096, 1, 'KEEP'), (4097, NULL, 'DROP') + +statement ok +SET pac_mi=0 + +statement ok +SET disabled_optimizers='extension' + +statement ok +CREATE TABLE correlated_noise_reference AS +SELECT o.id, + (SELECT as_noised(as_sum(priv_hash(hash(i.e)), i.v)) + FROM correlated_noise_input i WHERE i.k = o.k) AS s, + (SELECT as_noised_div(as_sum(priv_hash(hash(i.e)), i.v), + as_count(priv_hash(hash(i.e)), i.v)) + FROM correlated_noise_input i WHERE i.k = o.k) AS d +FROM correlated_noise_outer o + +statement ok +SET disabled_optimizers='' + +statement ok +CREATE TABLE correlated_noise_actual AS +SELECT o.id, + (SELECT as_noised(as_sum(priv_hash(hash(i.e)), i.v)) + FROM correlated_noise_input i WHERE i.k = o.k) AS s, + (SELECT as_noised_div(as_sum(priv_hash(hash(i.e)), i.v), + as_count(priv_hash(hash(i.e)), i.v)) + FROM correlated_noise_input i WHERE i.k = o.k) AS d +FROM correlated_noise_outer o + +query I +SELECT count(*) FROM ( + (SELECT * FROM correlated_noise_actual EXCEPT ALL SELECT * FROM correlated_noise_reference) + UNION ALL + (SELECT * FROM correlated_noise_reference EXCEPT ALL SELECT * FROM correlated_noise_actual) +) differences +---- +0 + + +# Pure outer predicates can move before the shared counters, while the noisy +# comparison and its potentially throwing arithmetic retain their output layout. +statement ok +SET disabled_optimizers='extension' + +statement ok +CREATE TABLE correlated_filtered_reference AS +SELECT o.id FROM correlated_noise_outer o +WHERE o.label = 'KEEP' AND o.id * 5 < ( + SELECT as_noised_div(as_sum(priv_hash(hash(i.e)), i.v), + as_count(priv_hash(hash(i.e)), i.v)) + FROM correlated_noise_input i WHERE i.k = o.k +) + +statement ok +SET disabled_optimizers='' + +query I +SELECT count(*) FROM ( + (SELECT o.id FROM correlated_noise_outer o + WHERE o.label = 'KEEP' AND o.id * 5 < ( + SELECT as_noised_div(as_sum(priv_hash(hash(i.e)), i.v), + as_count(priv_hash(hash(i.e)), i.v)) + FROM correlated_noise_input i WHERE i.k = o.k + ) EXCEPT ALL SELECT * FROM correlated_filtered_reference) + UNION ALL + (SELECT * FROM correlated_filtered_reference EXCEPT ALL + SELECT o.id FROM correlated_noise_outer o + WHERE o.label = 'KEEP' AND o.id * 5 < ( + SELECT as_noised_div(as_sum(priv_hash(hash(i.e)), i.v), + as_count(priv_hash(hash(i.e)), i.v)) + FROM correlated_noise_input i WHERE i.k = o.k + )) +) differences +---- +0 From 4f06701b5bbe9ca56ae280fd29fadf0d9d5d715a Mon Sep 17 00:00:00 2001 From: ila Date: Wed, 11 Nov 2026 15:00:27 +0100 Subject: [PATCH 29/32] Separate TPC-DS benchmark additions and remove one-off rerun tooling --- CMakeLists.txt | 61 ++- benchmark/audit_revision_results.py | 76 ---- benchmark/configs/utility/as_tpcds_sf30.json | 12 - benchmark/microbench/run_accuracy.py | 28 -- benchmark/microbench/run_graviton.sh | 35 -- benchmark/prepare_graviton_revision_data.sh | 33 -- benchmark/prepare_revision_configs.py | 49 --- benchmark/retry_graviton_analytical.sh | 41 -- benchmark/run_graviton_full_revision.sh | 63 --- benchmark/tpcds/as_queries/q01.sql | 23 - benchmark/tpcds/as_queries/q03.sql | 14 - benchmark/tpcds/as_queries/q04.sql | 120 ------ benchmark/tpcds/as_queries/q06.sql | 26 -- benchmark/tpcds/as_queries/q07.sql | 24 -- benchmark/tpcds/as_queries/q09.sql | 8 - benchmark/tpcds/as_queries/q10.sql | 66 --- benchmark/tpcds/as_queries/q11.sql | 80 ---- benchmark/tpcds/as_queries/q13.sql | 43 -- benchmark/tpcds/as_queries/q15.sql | 28 -- benchmark/tpcds/as_queries/q16.sql | 24 -- benchmark/tpcds/as_queries/q17.sql | 48 --- benchmark/tpcds/as_queries/q18.sql | 50 --- benchmark/tpcds/as_queries/q19.sql | 31 -- benchmark/tpcds/as_queries/q23.sql | 83 ---- benchmark/tpcds/as_queries/q24.sql | 53 --- benchmark/tpcds/as_queries/q25.sql | 42 -- benchmark/tpcds/as_queries/q26.sql | 24 -- benchmark/tpcds/as_queries/q27.sql | 61 --- benchmark/tpcds/as_queries/q28.sql | 45 -- benchmark/tpcds/as_queries/q29.sql | 42 -- benchmark/tpcds/as_queries/q30.sql | 49 --- benchmark/tpcds/as_queries/q31.sql | 69 --- benchmark/tpcds/as_queries/q32.sql | 22 - benchmark/tpcds/as_queries/q33.sql | 64 --- benchmark/tpcds/as_queries/q34.sql | 42 -- benchmark/tpcds/as_queries/q35.sql | 61 --- benchmark/tpcds/as_queries/q40.sql | 27 -- benchmark/tpcds/as_queries/q42.sql | 15 - benchmark/tpcds/as_queries/q43.sql | 13 - benchmark/tpcds/as_queries/q45.sql | 42 -- benchmark/tpcds/as_queries/q46.sql | 48 --- benchmark/tpcds/as_queries/q48.sql | 40 -- benchmark/tpcds/as_queries/q50.sql | 68 --- benchmark/tpcds/as_queries/q52.sql | 15 - benchmark/tpcds/as_queries/q55.sql | 14 - benchmark/tpcds/as_queries/q56.sql | 71 ---- benchmark/tpcds/as_queries/q58.sql | 68 --- benchmark/tpcds/as_queries/q59.sql | 85 ---- benchmark/tpcds/as_queries/q60.sql | 65 --- benchmark/tpcds/as_queries/q61.sql | 44 -- benchmark/tpcds/as_queries/q62.sql | 46 -- benchmark/tpcds/as_queries/q64.sql | 124 ------ benchmark/tpcds/as_queries/q65.sql | 23 - benchmark/tpcds/as_queries/q66.sql | 221 ---------- benchmark/tpcds/as_queries/q68.sql | 45 -- benchmark/tpcds/as_queries/q69.sql | 52 --- benchmark/tpcds/as_queries/q72.sql | 40 -- benchmark/tpcds/as_queries/q73.sql | 42 -- benchmark/tpcds/as_queries/q74.sql | 64 --- benchmark/tpcds/as_queries/q77.sql | 91 ---- benchmark/tpcds/as_queries/q78.sql | 75 ---- benchmark/tpcds/as_queries/q79.sql | 39 -- benchmark/tpcds/as_queries/q80.sql | 85 ---- benchmark/tpcds/as_queries/q81.sql | 55 --- benchmark/tpcds/as_queries/q83.sql | 68 --- benchmark/tpcds/as_queries/q85.sql | 60 --- benchmark/tpcds/as_queries/q88.sql | 139 ------ benchmark/tpcds/as_queries/q90.sql | 26 -- benchmark/tpcds/as_queries/q91.sql | 31 -- benchmark/tpcds/as_queries/q92.sql | 22 - benchmark/tpcds/as_queries/q93.sql | 20 - benchmark/tpcds/as_queries/q94.sql | 24 -- benchmark/tpcds/as_queries/q95.sql | 30 -- benchmark/tpcds/as_queries/q96.sql | 15 - benchmark/tpcds/as_queries/q99.sql | 45 -- benchmark/tpcds/as_tpcds_benchmark.cpp | 426 +------------------ benchmark/tpcds/q17_fixture.sql | 22 - benchmark/tpcds/reference_queries/q01.sql | 30 -- benchmark/tpcds/reference_queries/q03.sql | 30 -- benchmark/tpcds/reference_queries/q04.sql | 135 ------ benchmark/tpcds/reference_queries/q06.sql | 36 -- benchmark/tpcds/reference_queries/q07.sql | 38 -- benchmark/tpcds/reference_queries/q09.sql | 17 - benchmark/tpcds/reference_queries/q10.sql | 59 --- benchmark/tpcds/reference_queries/q11.sql | 93 ---- benchmark/tpcds/reference_queries/q13.sql | 55 --- benchmark/tpcds/reference_queries/q15.sql | 39 -- benchmark/tpcds/reference_queries/q16.sql | 10 - benchmark/tpcds/reference_queries/q17.sql | 67 --- benchmark/tpcds/reference_queries/q18.sql | 68 --- benchmark/tpcds/reference_queries/q19.sql | 41 -- benchmark/tpcds/reference_queries/q23.sql | 96 ----- benchmark/tpcds/reference_queries/q24.sql | 58 --- benchmark/tpcds/reference_queries/q25.sql | 54 --- benchmark/tpcds/reference_queries/q26.sql | 38 -- benchmark/tpcds/reference_queries/q27.sql | 25 -- benchmark/tpcds/reference_queries/q28.sql | 9 - benchmark/tpcds/reference_queries/q29.sql | 55 --- benchmark/tpcds/reference_queries/q30.sql | 68 --- benchmark/tpcds/reference_queries/q31.sql | 85 ---- benchmark/tpcds/reference_queries/q32.sql | 14 - benchmark/tpcds/reference_queries/q33.sql | 90 ---- benchmark/tpcds/reference_queries/q34.sql | 51 --- benchmark/tpcds/reference_queries/q35.sql | 111 ----- benchmark/tpcds/reference_queries/q40.sql | 16 - benchmark/tpcds/reference_queries/q42.sql | 31 -- benchmark/tpcds/reference_queries/q43.sql | 39 -- benchmark/tpcds/reference_queries/q45.sql | 53 --- benchmark/tpcds/reference_queries/q46.sql | 61 --- benchmark/tpcds/reference_queries/q48.sql | 50 --- benchmark/tpcds/reference_queries/q50.sql | 65 --- benchmark/tpcds/reference_queries/q52.sql | 31 -- benchmark/tpcds/reference_queries/q55.sql | 28 -- benchmark/tpcds/reference_queries/q56.sql | 98 ----- benchmark/tpcds/reference_queries/q58.sql | 93 ---- benchmark/tpcds/reference_queries/q59.sql | 94 ---- benchmark/tpcds/reference_queries/q60.sql | 92 ---- benchmark/tpcds/reference_queries/q61.sql | 49 --- benchmark/tpcds/reference_queries/q62.sql | 43 -- benchmark/tpcds/reference_queries/q64.sql | 133 ------ benchmark/tpcds/reference_queries/q65.sql | 17 - benchmark/tpcds/reference_queries/q66.sql | 227 ---------- benchmark/tpcds/reference_queries/q68.sql | 56 --- benchmark/tpcds/reference_queries/q69.sql | 49 --- benchmark/tpcds/reference_queries/q72.sql | 20 - benchmark/tpcds/reference_queries/q73.sql | 50 --- benchmark/tpcds/reference_queries/q74.sql | 74 ---- benchmark/tpcds/reference_queries/q77.sql | 114 ----- benchmark/tpcds/reference_queries/q78.sql | 80 ---- benchmark/tpcds/reference_queries/q79.sql | 53 --- benchmark/tpcds/reference_queries/q80.sql | 58 --- benchmark/tpcds/reference_queries/q81.sql | 77 ---- benchmark/tpcds/reference_queries/q83.sql | 93 ---- benchmark/tpcds/reference_queries/q85.sql | 68 --- benchmark/tpcds/reference_queries/q88.sql | 25 -- benchmark/tpcds/reference_queries/q90.sql | 30 -- benchmark/tpcds/reference_queries/q91.sql | 44 -- benchmark/tpcds/reference_queries/q92.sql | 14 - benchmark/tpcds/reference_queries/q93.sql | 12 - benchmark/tpcds/reference_queries/q94.sql | 10 - benchmark/tpcds/reference_queries/q95.sql | 10 - benchmark/tpcds/reference_queries/q96.sql | 24 -- benchmark/tpcds/reference_queries/q99.sql | 42 -- benchmark/tpcds/run.py | 67 +-- docs/benchmark/graviton_duckdb2_rerun.md | 42 -- src/aggregates/as_sum.cpp | 4 +- src/compiler/privacy_compiler.cpp | 17 +- 147 files changed, 68 insertions(+), 7942 deletions(-) delete mode 100644 benchmark/audit_revision_results.py delete mode 100644 benchmark/configs/utility/as_tpcds_sf30.json delete mode 100644 benchmark/microbench/run_accuracy.py delete mode 100644 benchmark/microbench/run_graviton.sh delete mode 100644 benchmark/prepare_graviton_revision_data.sh delete mode 100644 benchmark/prepare_revision_configs.py delete mode 100644 benchmark/retry_graviton_analytical.sh delete mode 100644 benchmark/run_graviton_full_revision.sh delete mode 100644 benchmark/tpcds/as_queries/q01.sql delete mode 100644 benchmark/tpcds/as_queries/q03.sql delete mode 100644 benchmark/tpcds/as_queries/q04.sql delete mode 100644 benchmark/tpcds/as_queries/q06.sql delete mode 100644 benchmark/tpcds/as_queries/q07.sql delete mode 100644 benchmark/tpcds/as_queries/q09.sql delete mode 100644 benchmark/tpcds/as_queries/q10.sql delete mode 100644 benchmark/tpcds/as_queries/q11.sql delete mode 100644 benchmark/tpcds/as_queries/q13.sql delete mode 100644 benchmark/tpcds/as_queries/q15.sql delete mode 100644 benchmark/tpcds/as_queries/q16.sql delete mode 100644 benchmark/tpcds/as_queries/q17.sql delete mode 100644 benchmark/tpcds/as_queries/q18.sql delete mode 100644 benchmark/tpcds/as_queries/q19.sql delete mode 100644 benchmark/tpcds/as_queries/q23.sql delete mode 100644 benchmark/tpcds/as_queries/q24.sql delete mode 100644 benchmark/tpcds/as_queries/q25.sql delete mode 100644 benchmark/tpcds/as_queries/q26.sql delete mode 100644 benchmark/tpcds/as_queries/q27.sql delete mode 100644 benchmark/tpcds/as_queries/q28.sql delete mode 100644 benchmark/tpcds/as_queries/q29.sql delete mode 100644 benchmark/tpcds/as_queries/q30.sql delete mode 100644 benchmark/tpcds/as_queries/q31.sql delete mode 100644 benchmark/tpcds/as_queries/q32.sql delete mode 100644 benchmark/tpcds/as_queries/q33.sql delete mode 100644 benchmark/tpcds/as_queries/q34.sql delete mode 100644 benchmark/tpcds/as_queries/q35.sql delete mode 100644 benchmark/tpcds/as_queries/q40.sql delete mode 100644 benchmark/tpcds/as_queries/q42.sql delete mode 100644 benchmark/tpcds/as_queries/q43.sql delete mode 100644 benchmark/tpcds/as_queries/q45.sql delete mode 100644 benchmark/tpcds/as_queries/q46.sql delete mode 100644 benchmark/tpcds/as_queries/q48.sql delete mode 100644 benchmark/tpcds/as_queries/q50.sql delete mode 100644 benchmark/tpcds/as_queries/q52.sql delete mode 100644 benchmark/tpcds/as_queries/q55.sql delete mode 100644 benchmark/tpcds/as_queries/q56.sql delete mode 100644 benchmark/tpcds/as_queries/q58.sql delete mode 100644 benchmark/tpcds/as_queries/q59.sql delete mode 100644 benchmark/tpcds/as_queries/q60.sql delete mode 100644 benchmark/tpcds/as_queries/q61.sql delete mode 100644 benchmark/tpcds/as_queries/q62.sql delete mode 100644 benchmark/tpcds/as_queries/q64.sql delete mode 100644 benchmark/tpcds/as_queries/q65.sql delete mode 100644 benchmark/tpcds/as_queries/q66.sql delete mode 100644 benchmark/tpcds/as_queries/q68.sql delete mode 100644 benchmark/tpcds/as_queries/q69.sql delete mode 100644 benchmark/tpcds/as_queries/q72.sql delete mode 100644 benchmark/tpcds/as_queries/q73.sql delete mode 100644 benchmark/tpcds/as_queries/q74.sql delete mode 100644 benchmark/tpcds/as_queries/q77.sql delete mode 100644 benchmark/tpcds/as_queries/q78.sql delete mode 100644 benchmark/tpcds/as_queries/q79.sql delete mode 100644 benchmark/tpcds/as_queries/q80.sql delete mode 100644 benchmark/tpcds/as_queries/q81.sql delete mode 100644 benchmark/tpcds/as_queries/q83.sql delete mode 100644 benchmark/tpcds/as_queries/q85.sql delete mode 100644 benchmark/tpcds/as_queries/q88.sql delete mode 100644 benchmark/tpcds/as_queries/q90.sql delete mode 100644 benchmark/tpcds/as_queries/q91.sql delete mode 100644 benchmark/tpcds/as_queries/q92.sql delete mode 100644 benchmark/tpcds/as_queries/q93.sql delete mode 100644 benchmark/tpcds/as_queries/q94.sql delete mode 100644 benchmark/tpcds/as_queries/q95.sql delete mode 100644 benchmark/tpcds/as_queries/q96.sql delete mode 100644 benchmark/tpcds/as_queries/q99.sql delete mode 100644 benchmark/tpcds/q17_fixture.sql delete mode 100644 benchmark/tpcds/reference_queries/q01.sql delete mode 100644 benchmark/tpcds/reference_queries/q03.sql delete mode 100644 benchmark/tpcds/reference_queries/q04.sql delete mode 100644 benchmark/tpcds/reference_queries/q06.sql delete mode 100644 benchmark/tpcds/reference_queries/q07.sql delete mode 100644 benchmark/tpcds/reference_queries/q09.sql delete mode 100644 benchmark/tpcds/reference_queries/q10.sql delete mode 100644 benchmark/tpcds/reference_queries/q11.sql delete mode 100644 benchmark/tpcds/reference_queries/q13.sql delete mode 100644 benchmark/tpcds/reference_queries/q15.sql delete mode 100644 benchmark/tpcds/reference_queries/q16.sql delete mode 100644 benchmark/tpcds/reference_queries/q17.sql delete mode 100644 benchmark/tpcds/reference_queries/q18.sql delete mode 100644 benchmark/tpcds/reference_queries/q19.sql delete mode 100644 benchmark/tpcds/reference_queries/q23.sql delete mode 100644 benchmark/tpcds/reference_queries/q24.sql delete mode 100644 benchmark/tpcds/reference_queries/q25.sql delete mode 100644 benchmark/tpcds/reference_queries/q26.sql delete mode 100644 benchmark/tpcds/reference_queries/q27.sql delete mode 100644 benchmark/tpcds/reference_queries/q28.sql delete mode 100644 benchmark/tpcds/reference_queries/q29.sql delete mode 100644 benchmark/tpcds/reference_queries/q30.sql delete mode 100644 benchmark/tpcds/reference_queries/q31.sql delete mode 100644 benchmark/tpcds/reference_queries/q32.sql delete mode 100644 benchmark/tpcds/reference_queries/q33.sql delete mode 100644 benchmark/tpcds/reference_queries/q34.sql delete mode 100644 benchmark/tpcds/reference_queries/q35.sql delete mode 100644 benchmark/tpcds/reference_queries/q40.sql delete mode 100644 benchmark/tpcds/reference_queries/q42.sql delete mode 100644 benchmark/tpcds/reference_queries/q43.sql delete mode 100644 benchmark/tpcds/reference_queries/q45.sql delete mode 100644 benchmark/tpcds/reference_queries/q46.sql delete mode 100644 benchmark/tpcds/reference_queries/q48.sql delete mode 100644 benchmark/tpcds/reference_queries/q50.sql delete mode 100644 benchmark/tpcds/reference_queries/q52.sql delete mode 100644 benchmark/tpcds/reference_queries/q55.sql delete mode 100644 benchmark/tpcds/reference_queries/q56.sql delete mode 100644 benchmark/tpcds/reference_queries/q58.sql delete mode 100644 benchmark/tpcds/reference_queries/q59.sql delete mode 100644 benchmark/tpcds/reference_queries/q60.sql delete mode 100644 benchmark/tpcds/reference_queries/q61.sql delete mode 100644 benchmark/tpcds/reference_queries/q62.sql delete mode 100644 benchmark/tpcds/reference_queries/q64.sql delete mode 100644 benchmark/tpcds/reference_queries/q65.sql delete mode 100644 benchmark/tpcds/reference_queries/q66.sql delete mode 100644 benchmark/tpcds/reference_queries/q68.sql delete mode 100644 benchmark/tpcds/reference_queries/q69.sql delete mode 100644 benchmark/tpcds/reference_queries/q72.sql delete mode 100644 benchmark/tpcds/reference_queries/q73.sql delete mode 100644 benchmark/tpcds/reference_queries/q74.sql delete mode 100644 benchmark/tpcds/reference_queries/q77.sql delete mode 100644 benchmark/tpcds/reference_queries/q78.sql delete mode 100644 benchmark/tpcds/reference_queries/q79.sql delete mode 100644 benchmark/tpcds/reference_queries/q80.sql delete mode 100644 benchmark/tpcds/reference_queries/q81.sql delete mode 100644 benchmark/tpcds/reference_queries/q83.sql delete mode 100644 benchmark/tpcds/reference_queries/q85.sql delete mode 100644 benchmark/tpcds/reference_queries/q88.sql delete mode 100644 benchmark/tpcds/reference_queries/q90.sql delete mode 100644 benchmark/tpcds/reference_queries/q91.sql delete mode 100644 benchmark/tpcds/reference_queries/q92.sql delete mode 100644 benchmark/tpcds/reference_queries/q93.sql delete mode 100644 benchmark/tpcds/reference_queries/q94.sql delete mode 100644 benchmark/tpcds/reference_queries/q95.sql delete mode 100644 benchmark/tpcds/reference_queries/q96.sql delete mode 100644 benchmark/tpcds/reference_queries/q99.sql delete mode 100644 docs/benchmark/graviton_duckdb2_rerun.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 6635e89b..6862aad9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,7 +86,6 @@ install( # Test runners and benchmarks - only build for native platforms, not WASM or # Windows (Windows lacks unistd.h and other POSIX features used by the benchmark # code) -set(PRIVACY_LINK_LIBS duckdb_static) if(NOT EMSCRIPTEN AND NOT WIN32) # Build a single unified test executable that invokes all tests via @@ -96,56 +95,49 @@ if(NOT EMSCRIPTEN AND NOT WIN32) src/test/test_runner.cpp src/test/test_plan_traversal.cpp src/test/test_schema_metadata.cpp src/test/test_compiler_functions.cpp src/test/test_privacy_parser.cpp) - target_link_libraries(pac_test_runner PRIVATE ${PRIVACY_LINK_LIBS}) # TPCH + AS benchmark runner add_executable(as_tpch_benchmark benchmark/tpch/as_tpch_benchmark.cpp) - target_link_libraries(as_tpch_benchmark PRIVATE ${PRIVACY_LINK_LIBS}) # WIP rebuttal experiment: four-way MEPS balanced repeated replication # benchmark add_executable(meps_brr_poc benchmark/meps_brr/meps_brr_poc.cpp) - target_link_libraries(meps_brr_poc PRIVATE ${PRIVACY_LINK_LIBS}) # WIP rebuttal experiment: standard joined SSB queries and 64 sampled worlds add_executable(as_ssb_benchmark benchmark/ssb/as_ssb_benchmark.cpp) - target_link_libraries(as_ssb_benchmark PRIVATE ${PRIVACY_LINK_LIBS}) # ClickBench + AS benchmark runner add_executable(as_clickbench_benchmark benchmark/clickbench/as_clickbench_benchmark.cpp) - target_link_libraries(as_clickbench_benchmark PRIVATE ${PRIVACY_LINK_LIBS}) - - # TPC-DS automatic-rewriter coverage and end-to-end performance - add_executable(as_tpcds_benchmark benchmark/tpcds/as_tpcds_benchmark.cpp) - target_link_libraries(as_tpcds_benchmark PRIVATE ${PRIVACY_LINK_LIBS}) - - # Unified DP utility benchmark runner - set(TPCH_STOCK_DP_QUERY_NAMES q01 q05 q06 q14 q19) - set(TPCH_STOCK_DP_QUERY_FILES) - foreach(QUERY_NAME IN LISTS TPCH_STOCK_DP_QUERY_NAMES) - string(TOUPPER ${QUERY_NAME} QUERY_NAME_UPPER) - set(QUERY_FILE "${CMAKE_CURRENT_SOURCE_DIR}/benchmark/dp/tpch_queries/${QUERY_NAME}.sql") - list(APPEND TPCH_STOCK_DP_QUERY_FILES "${QUERY_FILE}") - file(READ - "${QUERY_FILE}" - "TPCH_${QUERY_NAME_UPPER}_SQL") - endforeach() - set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS - ${TPCH_STOCK_DP_QUERY_FILES}) - set(TPCH_QUERY_GENERATED_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated") - file(MAKE_DIRECTORY "${TPCH_QUERY_GENERATED_DIR}") - configure_file( - benchmark/include/tpch_stock_queries.hpp.in - "${TPCH_QUERY_GENERATED_DIR}/tpch_stock_queries.hpp" - @ONLY) - add_executable(dp_benchmark_runner benchmark/dp/dp_benchmark_runner.cpp) - target_include_directories(dp_benchmark_runner PRIVATE "${TPCH_QUERY_GENERATED_DIR}") - target_link_libraries(dp_benchmark_runner PRIVATE ${PRIVACY_LINK_LIBS}) + + # TPC-DS automatic-rewriter coverage and end-to-end performance + add_executable(as_tpcds_benchmark benchmark/tpcds/as_tpcds_benchmark.cpp) + + # Unified DP utility benchmark runner + set(TPCH_STOCK_DP_QUERY_NAMES q01 q05 q06 q14 q19) + set(TPCH_STOCK_DP_QUERY_FILES) + foreach(QUERY_NAME IN LISTS TPCH_STOCK_DP_QUERY_NAMES) + string(TOUPPER ${QUERY_NAME} QUERY_NAME_UPPER) + set(QUERY_FILE + "${CMAKE_CURRENT_SOURCE_DIR}/benchmark/dp/tpch_queries/${QUERY_NAME}.sql" + ) + list(APPEND TPCH_STOCK_DP_QUERY_FILES "${QUERY_FILE}") + file(READ "${QUERY_FILE}" "TPCH_${QUERY_NAME_UPPER}_SQL") + endforeach() + set_property( + DIRECTORY + APPEND + PROPERTY CMAKE_CONFIGURE_DEPENDS ${TPCH_STOCK_DP_QUERY_FILES}) + set(TPCH_QUERY_GENERATED_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated") + file(MAKE_DIRECTORY "${TPCH_QUERY_GENERATED_DIR}") + configure_file(benchmark/include/tpch_stock_queries.hpp.in + "${TPCH_QUERY_GENERATED_DIR}/tpch_stock_queries.hpp" @ONLY) + add_executable(dp_benchmark_runner benchmark/dp/dp_benchmark_runner.cpp) + target_include_directories(dp_benchmark_runner + PRIVATE "${TPCH_QUERY_GENERATED_DIR}") # IMDb/JOB benchmark runner add_executable(imdb_benchmark benchmark/imdb/imdb_benchmark.cpp) - target_link_libraries(imdb_benchmark PRIVATE ${PRIVACY_LINK_LIBS}) foreach( PRIVACY_RUNNER IN @@ -157,6 +149,7 @@ if(NOT EMSCRIPTEN AND NOT WIN32) as_tpcds_benchmark dp_benchmark_runner imdb_benchmark) + target_link_libraries(${PRIVACY_RUNNER} PRIVATE duckdb_static) link_extension_libraries(${PRIVACY_RUNNER} PRIVATE) endforeach() endif() diff --git a/benchmark/audit_revision_results.py b/benchmark/audit_revision_results.py deleted file mode 100644 index 726263da..00000000 --- a/benchmark/audit_revision_results.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Collect runtime comparisons and execution anomalies without generating figures.""" -import argparse -import csv -import json -import statistics -from pathlib import Path - -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument("collection", type=Path) -args = parser.parse_args() -fresh = args.collection / "results/duckdb2_revision_20261110" -previous = args.collection / "previous" -comparisons, anomalies = [], [] - -def read(path): - if not path.exists(): - return [] - return list(csv.DictReader(path.open())) - -def medians(path, value="median_ms", warmup=False): - grouped = {} - record_anomalies = path.is_relative_to(fresh) - for row in read(path): - if warmup and row.get("run") == "0": - continue - if row.get("status") not in (None, "ok", "executed_not_validated"): - if record_anomalies: - anomalies.append(dict(file=str(path.relative_to(args.collection)), query=row.get("query", ""), - mode=row.get("mode", ""), status=row["status"], error=row.get("error", ""))) - continue - if not row.get(value): - continue - number = float(row[value]) - if number < 0: - if record_anomalies: - anomalies.append(dict(file=str(path.relative_to(args.collection)), query=row.get("query", ""), - mode=row.get("mode", ""), status="negative_runtime", error="")) - continue - mode = "".join(c for c in row.get("mode", "").lower() if c.isalnum()) - m = row.get("m") or ("0" if mode == "baseline" else "64") - key = row.get("query", ""), mode, m - grouped.setdefault(key, []).append(number) - return {key: statistics.median(values) for key, values in grouped.items()} - -def compare(name, old, new, value="median_ms", warmup=False): - reference, measured = medians(old, value, warmup), medians(new, value, warmup) - for key in sorted(reference.keys() & measured.keys()): - before, after = reference[key], measured[key] - comparisons.append(dict(benchmark=name, query=key[0], mode=key[1], m=key[2], - reference_ms=before, new_ms=after, - runtime_ratio=after / before if before else "", - inspect=bool(before and after / before > 1.2))) - -compare("tpch_matched_main", fresh / "csv/as_tpch_sf30_main.csv", fresh / "csv/as_tpch_sf30.csv") -compare("tpch_previous_paper", previous / "as_tpch_m64.csv", fresh / "csv/as_tpch_sf30.csv") -compare("tpch_m_previous_paper", previous / "as_tpch_m_sweep.csv", fresh / "csv/as_tpch_sf30_m_sweep.csv") -compare("clickbench_previous_paper", previous / "as_clickbench_m64_m512.csv", fresh / "csv/as_clickbench_graviton.csv") -compare("tpcds_previous_paper", previous / "tpcds_sf30_plot_data.csv", fresh / "tpcds_sf30/summary.csv") -for kernel in ("approximate", "exact"): - subpath = Path("ssb_sf30") / kernel / "threads_16_timings.csv" - compare("ssb_" + kernel, previous / "graviton_5runs" / subpath, fresh / subpath, "query_ms") - for workload in ("damico2006", "ahrq2016"): - subpath = Path("meps") / kernel / workload / "threads_16_timings.csv" - compare("meps_" + kernel + "_" + workload, previous / "graviton_5runs" / subpath, - fresh / subpath, "query_ms", warmup=True) - -for name, rows, fields in ( - ("runtime_comparison.csv", comparisons, ["benchmark", "query", "mode", "m", "reference_ms", "new_ms", "runtime_ratio", "inspect"]), - ("execution_anomalies.csv", anomalies, ["file", "query", "mode", "status", "error"]), -): - with (args.collection / name).open("w", newline="") as file: - writer = csv.DictWriter(file, fieldnames=fields) - writer.writeheader() - writer.writerows(rows) -print(json.dumps(dict(comparisons=len(comparisons), inspect=sum(r["inspect"] for r in comparisons), - execution_anomalies=len(anomalies)))) diff --git a/benchmark/configs/utility/as_tpcds_sf30.json b/benchmark/configs/utility/as_tpcds_sf30.json deleted file mode 100644 index ba422483..00000000 --- a/benchmark/configs/utility/as_tpcds_sf30.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "sf": 30, - "db": "benchmark/results/tpcds_sf30/data.db", - "queries_dir": "benchmark/results/tpcds_sf30/queries", - "out_dir": "benchmark/results/tpcds_sf30", - "schema": "benchmark/tpcds/schema.sql", - "threads": 16, - "memory_limit": "30GB", - "repetitions": 5, - "timeout_seconds": 3600, - "extension_directory": "benchmark/results/tpcds_extensions" -} diff --git a/benchmark/microbench/run_accuracy.py b/benchmark/microbench/run_accuracy.py deleted file mode 100644 index 1fa07200..00000000 --- a/benchmark/microbench/run_accuracy.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Repeat the existing ten-distribution SUM accuracy experiment on a common scale.""" -import argparse -import subprocess -from pathlib import Path - -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument("binary", type=Path) -parser.add_argument("output", type=Path) -args = parser.parse_args() -source = (Path(__file__).parent / "approx_sum_stability.sql").read_text() -marker = source.index("-- test query:") -setup = source[:marker] -start = source.index("WITH bit_positions", marker) -query = source[start:source.index(";", start)] -# as_sum returns 2x the sampled totals. Rescale once, before computing both -# squared errors and the approximate variance, so every statistic uses the -# same units as independently computed sample sums. -query = query.replace("as_sum(priv_hash(hash(rowid)), val) as approx_counters", - "list_transform(as_sum(priv_hash(hash(rowid)), val), lambda x: x / 2.0) as approx_counters") -query = query.replace("a.approx_counters[x+1] / 2", "a.approx_counters[x+1]") -query = query.replace("x -> abs(x)", "lambda x: abs(x)") -path = str(args.output.resolve()).replace("'", "''") -sql = ("SET threads=4; SET memory_limit='30GB'; SET priv_rewrite=false; " - "SET pac_mi=1; SET privacy_seed=42; SELECT setseed(0.42);\n" + setup - + "\nCOPY (" + query + ") TO '" + path + "' (HEADER, DELIMITER ',');") -args.output.parent.mkdir(parents=True, exist_ok=True) -args.output.with_suffix(".sql").write_text(sql) -subprocess.run([str(args.binary.resolve()), "-batch"], input=sql, text=True, check=True) diff --git a/benchmark/microbench/run_graviton.sh b/benchmark/microbench/run_graviton.sh deleted file mode 100644 index 4d68d855..00000000 --- a/benchmark/microbench/run_graviton.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# Build variants without deleting the analytical build or its measured binaries. -set -euo pipefail -# The current rerun retries analytical setup failures before timing microbenchmarks. -while systemctl is-active --quiet asa-analytical-retry-20261110.service; do sleep 10; done -results="${1:?Usage: run_graviton.sh NEW_RESULTS_DIR}" -repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -test ! -e "$results" || { echo "Refusing existing results: $results" >&2; exit 1; } -mkdir -p "$results/builds" "$repo/benchmark/microbench/binaries" -variants=(default noboundopt nobuffering signedsum exactsum nocascading nosimd) -flags=('' '-DPAC_NOBOUNDOPT' '-DPAC_NOBUFFERING' '-DPAC_SIGNEDSUM' '-DPAC_EXACTSUM' '-DPAC_NOCASCADING' '-DPAC_NOCASCADING -DPAC_NOSIMD -fno-vectorize -fno-slp-vectorize') -for i in "${!variants[@]}"; do - name="${variants[$i]}" - cmake -S "$repo/duckdb" -B "$repo/build/graviton_micro" -G Ninja \ - -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang-22 -DCMAKE_CXX_COMPILER=clang++-22 \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DDUCKDB_EXTENSION_CONFIGS="$repo/extension_config.cmake" -DBUILD_UNITTESTS=OFF \ - -DPAC_NOAPPROX=OFF -DCMAKE_CXX_FLAGS="${flags[$i]}" \ - > "$results/builds/$name.configure.log" 2>&1 - cmake --build "$repo/build/graviton_micro" --target duckdb -j 4 > "$results/builds/$name.build.log" 2>&1 - cp "$repo/build/graviton_micro/duckdb" "$repo/benchmark/microbench/binaries/duckdb_$name" - cp "$repo/build/graviton_micro/CMakeCache.txt" "$results/builds/$name.CMakeCache.txt" -done -bash "$repo/benchmark/microbench/create_test_db.sh" > "$results/create_data.log" 2>&1 -export RESULTS_DIR="$results/csv" -export QUERY_SETUP="SET pac_mi=0.0; SET threads=4; SET max_memory='30G'; SET max_temp_directory_size='90GB'; SET temp_directory='$repo/benchmark/microbench/tmp';" -export WARMUP_RUNS=1 BENCH_RUNS=3 -for benchmark in count sum_avg min_max; do - bash "$repo/benchmark/microbench/bench_$benchmark.sh" > "$results/$benchmark.log" 2>&1 -done -for variant in default signedsum exactsum; do - python3 "$repo/benchmark/microbench/run_accuracy.py" \ - "$repo/benchmark/microbench/binaries/duckdb_$variant" "$results/csv/accuracy_$variant.csv" \ - > "$results/accuracy_$variant.log" 2>&1 -done diff --git a/benchmark/prepare_graviton_revision_data.sh b/benchmark/prepare_graviton_revision_data.sh deleted file mode 100644 index df283724..00000000 --- a/benchmark/prepare_graviton_revision_data.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -root="${1:?Usage: prepare_graviton_revision_data.sh ROOT}" -repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -mkdir -p "$root/data" "$root/repro/data_prepare" -exec > >(tee -a "$root/repro/data_prepare/run.log") 2>&1 -printf 'running\n' > "$root/repro/data_prepare/exit_code" -trap 'printf "%s\n" "$?" > "$root/repro/data_prepare/exit_code"' EXIT -cd "$repo" -if [ ! -d "$root/ssb-dbgen" ]; then - git clone https://github.com/electrum/ssb-dbgen.git "$root/ssb-dbgen" - git -C "$root/ssb-dbgen" checkout 219403ad7d1dd32ae1f97b5553abf92129fccd7f - git -C "$root/ssb-dbgen" apply "$repo/benchmark/ssb/dbgen-macos.patch" -fi -make -C "$root/ssb-dbgen" dbgen CC=clang-22 \ - CFLAGS='-O2 -std=gnu89 -include unistd.h -D_GNU_SOURCE -DDBNAME=\"dss\" -DLINUX -DDB2 -DSSBM' -if [ ! -f "$root/data/ssb_sf30/generation_complete" ]; then - bash benchmark/ssb/prepare.sh "$root/ssb-dbgen" "$root/data/ssb_sf30" 30 -fi -while pgrep -x apt-get >/dev/null; do sleep 5; done -python3 -m venv --system-site-packages "$root/prepare-env" -"$root/prepare-env/bin/python" -m pip freeze > "$root/repro/data_prepare/python_requirements.txt" -for workload in damico2006 ahrq2016; do - if [ ! -f "$root/data/meps/$workload/manifest.json" ]; then - "$root/prepare-env/bin/python" benchmark/meps_brr/prepare.py "$root/data/meps/$workload" --workload "$workload" - fi -done -if [ ! -f "$repo/hits.parquet" ]; then - curl --fail --location --retry 3 https://datasets.clickhouse.com/hits_compatible/hits.parquet \ - --output "$repo/hits.parquet.part" - mv "$repo/hits.parquet.part" "$repo/hits.parquet" -fi -sha256sum "$repo/hits.parquet" > "$root/repro/data_prepare/clickbench.sha256" diff --git a/benchmark/prepare_revision_configs.py b/benchmark/prepare_revision_configs.py deleted file mode 100644 index 94eb194a..00000000 --- a/benchmark/prepare_revision_configs.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Prepare fresh Graviton run configurations without changing paper configurations.""" -import argparse -import json -from pathlib import Path - -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument("root", type=Path) -parser.add_argument("output", type=Path) -args = parser.parse_args() -repo = Path(__file__).resolve().parent.parent -args.output.mkdir(parents=True, exist_ok=True) -tpch_db = str(args.root / "data/tpch_sf30_duckdb2.db") -for source in sorted((repo / "benchmark/configs/utility").glob("*.json")): - if source.name.startswith("as_tpcds"): - continue - config = json.loads(source.read_text()) - config["out"] = str(args.output.parent / "csv" / (source.stem + ".csv")) - if source.name.startswith("as_tpch"): - config.update(db=tpch_db, queries_dir=str(repo / "benchmark")) - elif source.name.startswith("as_clickbench"): - config.update(db=str(args.root / "data/clickbench_duckdb2.db"), - queries_dir=str(repo / "benchmark/clickbench/clickbench_queries")) - elif source.name.startswith("dp_tpch"): - for dataset in config.get("datasets", []): - dataset["db"] = tpch_db - else: - continue - (args.output / source.name).write_text(json.dumps(config, indent=2) + "\n") - -config = json.loads((repo / "benchmark/configs/utility/as_tpcds_sf30.json").read_text()) -config.update(db=str(args.root / "data/tpcds_sf30_duckdb2/data.db"), - queries_dir=str(args.root / "data/tpcds_sf30_duckdb2/queries"), - out_dir=str(args.output.parent / "tpcds_sf30"), - schema=str(repo / "benchmark/tpcds/schema.sql"), - as_queries_dir=str(repo / "benchmark/tpcds/as_queries"), - reference_queries_dir=str(repo / "benchmark/tpcds/reference_queries"), - extension_directory=str(args.root / "data/tpcds_extensions_duckdb2")) -(args.output / "as_tpcds_sf30.json").write_text(json.dumps(config, indent=2) + "\n") -config.update(sf=1, threads=1, repetitions=1, - db=str(args.root / "data/tpcds_sf1_duckdb2/data.db"), - queries_dir=str(args.root / "data/tpcds_sf1_duckdb2/queries"), - out_dir=str(args.output.parent / "tpcds_sf1_validation")) -(args.output / "as_tpcds_validate_sf1.json").write_text(json.dumps(config, indent=2) + "\n") - -main = json.loads((repo / "benchmark/configs/utility/as_tpch_sf30.json").read_text()) -main.update(db=str(args.root / "data/tpch_sf30_main.db"), - queries_dir=str(args.root / "privacy/benchmark"), - out=str(args.output.parent / "csv/as_tpch_sf30_main.csv")) -(args.output / "as_tpch_sf30_main.json").write_text(json.dumps(main, indent=2) + "\n") diff --git a/benchmark/retry_graviton_analytical.sh b/benchmark/retry_graviton_analytical.sh deleted file mode 100644 index c6d58c49..00000000 --- a/benchmark/retry_graviton_analytical.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -# Retry the two setup failures after DP, before the queued microbenchmarks. -set -euo pipefail -root="${1:?Usage: retry_graviton_analytical.sh ROOT RESULTS_DIR}" -results="${2:?RESULTS_DIR is required}" -repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -exec > >(tee -a "$results/repro/analytical_retry.log") 2>&1 -printf 'running\n' > "$results/repro/analytical_retry_exit_code" -trap 'printf "%s\n" "$?" > "$results/repro/analytical_retry_exit_code"' EXIT -while [ "$(cat "$results/repro/stage")" != microbench ]; do - systemctl is-active --quiet asa-full-revision-retry-20261110.service || exit 1 - sleep 10 -done -cd "$repo" -git rev-parse HEAD > "$results/repro/analytical_retry_source_commit.txt" -failures=0 -for workload in ssb meps; do - output="$results/meps" - data="$root/data/meps" - script=benchmark/meps_brr/run.sh - if [ "$workload" = ssb ]; then - output="$results/ssb_sf30" - data="$root/data/ssb_sf30" - script=benchmark/ssb/run.sh - fi - if [ -e "$output" ]; then - test ! -e "${output}_failed_setup" - mv "$output" "${output}_failed_setup" - fi - date -u - if SSB_MEMORY_LIMIT=30GB bash "$script" \ - "$data" "$output" 16 5 > "$results/repro/${workload}_retry.log" 2>&1; then - printf '%s_retry,0\n' "$workload" >> "$results/repro/stage_status.csv" - else - code=$? - printf '%s_retry,%s\n' "$workload" "$code" >> "$results/repro/stage_status.csv" - failures=$((failures+1)) - tail -25 "$results/repro/${workload}_retry.log" - fi -done -test "$failures" = 0 diff --git a/benchmark/run_graviton_full_revision.sh b/benchmark/run_graviton_full_revision.sh deleted file mode 100644 index 11e767bf..00000000 --- a/benchmark/run_graviton_full_revision.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# Run analytical experiments first, then DP experiments, then microbenchmarks. -set -euo pipefail -root="${1:?Usage: run_graviton_full_revision.sh ROOT NEW_RESULTS_DIR}" -results="${2:?A fresh result directory is required}" -repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -test ! -e "$results" || { echo "Refusing existing results: $results" >&2; exit 1; } -mkdir -p "$results/repro" "$results/csv" -exec > >(tee -a "$results/repro/run.log") 2>&1 -printf 'running\n' > "$results/repro/exit_code" -trap 'code=$?; printf "%s\n" "$code" > "$results/repro/exit_code"; date -u > "$results/repro/finished_utc.txt"' EXIT -date -u > "$results/repro/started_utc.txt" -cd "$repo" -git rev-parse HEAD > "$results/repro/source_commit.txt" -git status --short > "$results/repro/source_status.txt" -git submodule status > "$results/repro/submodules.txt" -{ uname -a; lscpu; free -h; lsblk -o NAME,SIZE,MODEL,MOUNTPOINTS; df -h "$root"; clang-22 --version; } > "$results/repro/environment.txt" -for unit in asa-main-prepare-20261110 asa-duckdb2-build-20261110 asa-data-prepare-20261110 asa-data-prepare-retry-20261110 asa-duckdb2-validated-20261110; do - while systemctl is-active --quiet "$unit"; do sleep 10; done -done -test "$(cat "$root/repro/main_prepare/exit_code")" = 0 -test "$(cat "$root/repro/duckdb2_build/exit_code")" = 0 -test "$(cat "$root/repro/data_prepare/exit_code")" = 0 -cp build/release/CMakeCache.txt "$results/repro/CMakeCache.txt" -python3 benchmark/prepare_revision_configs.py "$root" "$results/configs" -cp --reflink=auto "$root/data/tpch_sf30_base.db" "$root/data/tpch_sf30_main.db" -cp --reflink=auto "$root/data/tpch_sf30_base.db" "$root/data/tpch_sf30_duckdb2.db" -failures=0 -run_stage() { - local stage="$1" - shift - printf '%s\n' "$stage" > "$results/repro/stage" - date -u - if "$@" > "$results/repro/$stage.log" 2>&1; then - printf '%s,0\n' "$stage" >> "$results/repro/stage_status.csv" - else - local code=$? - printf '%s,%s\n' "$stage" "$code" >> "$results/repro/stage_status.csv" - failures=$((failures+1)) - tail -25 "$results/repro/$stage.log" - fi -} -run_stage tpch_main "$root/privacy/build/release/extension/privacy/as_tpch_benchmark" --config "$results/configs/as_tpch_sf30_main.json" -run_stage tpch_2 "$repo/build/release/extension/privacy/as_tpch_benchmark" --config "$results/configs/as_tpch_sf30.json" -run_stage tpch_m_sweep "$repo/build/release/extension/privacy/as_tpch_benchmark" --config "$results/configs/as_tpch_sf30_m_sweep.json" -run_stage clickbench "$repo/build/release/extension/privacy/as_clickbench_benchmark" --config "$results/configs/as_clickbench_graviton.json" -run_stage ssb env SSB_MEMORY_LIMIT=30GB bash benchmark/ssb/run.sh "$root/data/ssb_sf30" "$results/ssb_sf30" 16 5 -run_stage meps bash benchmark/meps_brr/run.sh "$root/data/meps" "$results/meps" 16 5 -run_stage tpcds_validation python3 benchmark/tpcds/run.py --config "$results/configs/as_tpcds_validate_sf1.json" --validate -run_stage tpcds python3 benchmark/tpcds/run.py --config "$results/configs/as_tpcds_sf30.json" -for config in "$results"/configs/dp_tpch*.json; do - name="$(basename "$config" .json)" - if [[ "$name" == *stability* ]]; then - run_stage "$name" python3 benchmark/dp/run_tpch_sass_stability.py --config "$config" \ - --duckdb "$repo/build/release/duckdb" --out "$results/csv/$name.csv" \ - --summary-out "$results/csv/${name}_summary.csv" --strict - else - run_stage "$name" "$repo/build/release/extension/privacy/dp_benchmark_runner" --config "$config" - fi -done -run_stage microbench bash benchmark/microbench/run_graviton.sh "$results/microbench" -printf 'complete\n' > "$results/repro/stage" -test "$failures" = 0 diff --git a/benchmark/tpcds/as_queries/q01.sql b/benchmark/tpcds/as_queries/q01.sql deleted file mode 100644 index 1621b57e..00000000 --- a/benchmark/tpcds/as_queries/q01.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- The CTE exports customer totals unchanged; its correlated AVG is stochastic across customers. -WITH customer_total_return AS (SELECT sr_customer_sk AS ctr_customer_sk, - sr_store_sk AS ctr_store_sk, - sum(sr_return_amt) AS ctr_total_return - FROM store_returns, - date_dim - WHERE sr_returned_date_sk = d_date_sk - AND d_year = 2000 - GROUP BY sr_customer_sk, - sr_store_sk) -SELECT c_customer_id -FROM customer_total_return ctr1, - store, - customer -WHERE priv_filter_gt(CAST(CAST(ctr1.ctr_total_return AS DOUBLE) * 0.8333333333333334 AS FLOAT), (SELECT priv_div(as_sum(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return), as_count(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return)) - FROM customer_total_return ctr2 - WHERE ctr1.ctr_store_sk = ctr2.ctr_store_sk)) - AND s_store_sk = ctr1.ctr_store_sk - AND s_state = 'TN' - AND ctr1.ctr_customer_sk = c_customer_sk -ORDER BY c_customer_id -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q03.sql b/benchmark/tpcds/as_queries/q03.sql deleted file mode 100644 index 156d7171..00000000 --- a/benchmark/tpcds/as_queries/q03.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH asa_states AS (SELECT dt.d_year AS d_year, item.i_brand_id AS brand_id, item.i_brand AS brand, as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_sales_price AS DOUBLE)) AS sum_agg FROM date_dim dt, - store_sales, - item -WHERE dt.d_date_sk = store_sales.ss_sold_date_sk - AND store_sales.ss_item_sk = item.i_item_sk - AND item.i_manufact_id = 128 - AND dt.d_moy=11 -GROUP BY dt.d_year, - item.i_brand, - item.i_brand_id -), -asa_ranked AS (SELECT * FROM asa_states ORDER BY d_year, priv_mean(sum_agg) DESC, brand_id LIMIT 100) -SELECT d_year, brand_id, brand, CAST(as_noised(sum_agg) AS DECIMAL(38,2)) AS sum_agg FROM asa_ranked ORDER BY d_year, sum_agg DESC, brand_id; diff --git a/benchmark/tpcds/as_queries/q04.sql b/benchmark/tpcds/as_queries/q04.sql deleted file mode 100644 index b499a2b7..00000000 --- a/benchmark/tpcds/as_queries/q04.sql +++ /dev/null @@ -1,120 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Keep year_total scalar inside its CTE; outer customer/year joins retain ordinary PAC scalar semantics. -WITH year_total AS (SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - c_preferred_cust_flag customer_preferred_cust_flag, - c_birth_country customer_birth_country, - c_login customer_login, - c_email_address customer_email_address, - d_year dyear, - as_noised_sum(priv_hash(hash(c_customer_sk)), ((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total, - 's' sale_type - FROM customer, - store_sales, - date_dim - WHERE c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - GROUP BY c_customer_id, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_country, - c_login, - c_email_address, - d_year - UNION ALL SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - c_preferred_cust_flag customer_preferred_cust_flag, - c_birth_country customer_birth_country, - c_login customer_login, - c_email_address customer_email_address, - d_year dyear, - as_noised_sum(priv_hash(hash(c_customer_sk)), (((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2)) year_total, - 'c' sale_type - FROM customer, - catalog_sales, - date_dim - WHERE c_customer_sk = cs_bill_customer_sk - AND cs_sold_date_sk = d_date_sk - GROUP BY c_customer_id, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_country, - c_login, - c_email_address, - d_year - UNION ALL SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - c_preferred_cust_flag customer_preferred_cust_flag, - c_birth_country customer_birth_country, - c_login customer_login, - c_email_address customer_email_address, - d_year dyear, - as_noised_sum(priv_hash(hash(c_customer_sk)), (((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2)) year_total, - 'w' sale_type - FROM customer, - web_sales, - date_dim - WHERE c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - GROUP BY c_customer_id, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_country, - c_login, - c_email_address, - d_year) -SELECT t_s_secyear.customer_id, - t_s_secyear.customer_first_name, - t_s_secyear.customer_last_name, - t_s_secyear.customer_preferred_cust_flag -FROM year_total t_s_firstyear, - year_total t_s_secyear, - year_total t_c_firstyear, - year_total t_c_secyear, - year_total t_w_firstyear, - year_total t_w_secyear -WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id - AND t_s_firstyear.customer_id = t_c_secyear.customer_id - AND t_s_firstyear.customer_id = t_c_firstyear.customer_id - AND t_s_firstyear.customer_id = t_w_firstyear.customer_id - AND t_s_firstyear.customer_id = t_w_secyear.customer_id - AND t_s_firstyear.sale_type = 's' - AND t_c_firstyear.sale_type = 'c' - AND t_w_firstyear.sale_type = 'w' - AND t_s_secyear.sale_type = 's' - AND t_c_secyear.sale_type = 'c' - AND t_w_secyear.sale_type = 'w' - AND t_s_firstyear.dyear = 2001 - AND t_s_secyear.dyear = 2001+1 - AND t_c_firstyear.dyear = 2001 - AND t_c_secyear.dyear = 2001+1 - AND t_w_firstyear.dyear = 2001 - AND t_w_secyear.dyear = 2001+1 - AND t_s_firstyear.year_total > 0 - AND t_c_firstyear.year_total > 0 - AND t_w_firstyear.year_total > 0 - AND CASE - WHEN t_c_firstyear.year_total > 0 THEN t_c_secyear.year_total / t_c_firstyear.year_total - ELSE NULL - END > CASE - WHEN t_s_firstyear.year_total > 0 THEN t_s_secyear.year_total / t_s_firstyear.year_total - ELSE NULL - END - AND CASE - WHEN t_c_firstyear.year_total > 0 THEN t_c_secyear.year_total / t_c_firstyear.year_total - ELSE NULL - END > CASE - WHEN t_w_firstyear.year_total > 0 THEN t_w_secyear.year_total / t_w_firstyear.year_total - ELSE NULL - END -ORDER BY t_s_secyear.customer_id NULLS FIRST, - t_s_secyear.customer_first_name NULLS FIRST, - t_s_secyear.customer_last_name NULLS FIRST, - t_s_secyear.customer_preferred_cust_flag NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q06.sql b/benchmark/tpcds/as_queries/q06.sql deleted file mode 100644 index 849ef217..00000000 --- a/benchmark/tpcds/as_queries/q06.sql +++ /dev/null @@ -1,26 +0,0 @@ --- PAC COUNT with scalar HAVING, as in the compiled plan. -SELECT a.ca_state state, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt -FROM customer_address a , - customer c , - store_sales s , - date_dim d , - item i -WHERE a.ca_address_sk = c.c_current_addr_sk - AND c.c_customer_sk = s.ss_customer_sk - AND s.ss_sold_date_sk = d.d_date_sk - AND s.ss_item_sk = i.i_item_sk - AND d.d_month_seq = - (SELECT DISTINCT (d_month_seq) - FROM date_dim - WHERE d_year = 2001 - AND d_moy = 1 ) - AND i.i_current_price > 1.2 * - (SELECT avg(j.i_current_price) - FROM item j - WHERE j.i_category = i.i_category) -GROUP BY a.ca_state -HAVING as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) >= 10 -ORDER BY cnt NULLS FIRST, - a.ca_state NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q07.sql b/benchmark/tpcds/as_queries/q07.sql deleted file mode 100644 index 959198df..00000000 --- a/benchmark/tpcds/as_queries/q07.sql +++ /dev/null @@ -1,24 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT i_item_id, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE)) agg1, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_list_price) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_list_price) AS DOUBLE)) agg2, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) AS DOUBLE)) agg3, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_sales_price) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_sales_price) AS DOUBLE)) agg4 -FROM store_sales, - customer_demographics, - date_dim, - item, - promotion -WHERE ss_sold_date_sk = d_date_sk - AND ss_item_sk = i_item_sk - AND ss_cdemo_sk = cd_demo_sk - AND ss_promo_sk = p_promo_sk - AND cd_gender = 'M' - AND cd_marital_status = 'S' - AND cd_education_status = 'College' - AND (p_channel_email = 'N' - OR p_channel_event = 'N') - AND d_year = 2000 -GROUP BY i_item_id -ORDER BY i_item_id -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q09.sql b/benchmark/tpcds/as_queries/q09.sql deleted file mode 100644 index 352ff3cb..00000000 --- a/benchmark/tpcds/as_queries/q09.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Evaluate CASE lane-wise over counter lists and finalize once; all fifteen scalar subqueries remain. -SELECT CAST(as_noised(list_transform(list_zip((SELECT as_count(priv_hash(hash(ss_customer_sk)),1::BIGINT) FROM store_sales WHERE ss_quantity BETWEEN 1 AND 20), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_discount_amt AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_ext_discount_amt)) FROM store_sales WHERE ss_quantity BETWEEN 1 AND 20), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_net_paid AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_net_paid)) FROM store_sales WHERE ss_quantity BETWEEN 1 AND 20)), lambda x: CASE WHEN x[1] > 74129 THEN x[2] ELSE x[3] END)) AS DOUBLE) AS bucket1, -CAST(as_noised(list_transform(list_zip((SELECT as_count(priv_hash(hash(ss_customer_sk)),1::BIGINT) FROM store_sales WHERE ss_quantity BETWEEN 21 AND 40), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_discount_amt AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_ext_discount_amt)) FROM store_sales WHERE ss_quantity BETWEEN 21 AND 40), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_net_paid AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_net_paid)) FROM store_sales WHERE ss_quantity BETWEEN 21 AND 40)), lambda x: CASE WHEN x[1] > 122840 THEN x[2] ELSE x[3] END)) AS DOUBLE) AS bucket2, -CAST(as_noised(list_transform(list_zip((SELECT as_count(priv_hash(hash(ss_customer_sk)),1::BIGINT) FROM store_sales WHERE ss_quantity BETWEEN 41 AND 60), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_discount_amt AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_ext_discount_amt)) FROM store_sales WHERE ss_quantity BETWEEN 41 AND 60), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_net_paid AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_net_paid)) FROM store_sales WHERE ss_quantity BETWEEN 41 AND 60)), lambda x: CASE WHEN x[1] > 56580 THEN x[2] ELSE x[3] END)) AS DOUBLE) AS bucket3, -CAST(as_noised(list_transform(list_zip((SELECT as_count(priv_hash(hash(ss_customer_sk)),1::BIGINT) FROM store_sales WHERE ss_quantity BETWEEN 61 AND 80), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_discount_amt AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_ext_discount_amt)) FROM store_sales WHERE ss_quantity BETWEEN 61 AND 80), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_net_paid AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_net_paid)) FROM store_sales WHERE ss_quantity BETWEEN 61 AND 80)), lambda x: CASE WHEN x[1] > 10097 THEN x[2] ELSE x[3] END)) AS DOUBLE) AS bucket4, -CAST(as_noised(list_transform(list_zip((SELECT as_count(priv_hash(hash(ss_customer_sk)),1::BIGINT) FROM store_sales WHERE ss_quantity BETWEEN 81 AND 100), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_discount_amt AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_ext_discount_amt)) FROM store_sales WHERE ss_quantity BETWEEN 81 AND 100), (SELECT priv_div(as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_net_paid AS DOUBLE)),as_count(priv_hash(hash(ss_customer_sk)),ss_net_paid)) FROM store_sales WHERE ss_quantity BETWEEN 81 AND 100)), lambda x: CASE WHEN x[1] > 165306 THEN x[2] ELSE x[3] END)) AS DOUBLE) AS bucket5 -FROM reason WHERE r_reason_sk=1; diff --git a/benchmark/tpcds/as_queries/q10.sql b/benchmark/tpcds/as_queries/q10.sql deleted file mode 100644 index df71aa6a..00000000 --- a/benchmark/tpcds/as_queries/q10.sql +++ /dev/null @@ -1,66 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -SELECT cd_gender, - cd_marital_status, - cd_education_status, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt1, - cd_purchase_estimate, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt2, - cd_credit_rating, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt3, - cd_dep_count, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt4, - cd_dep_employed_count, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt5, - cd_dep_college_count, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt6 -FROM customer c, - customer_address ca, - customer_demographics -WHERE c.c_current_addr_sk = ca.ca_address_sk - AND ca_county IN ('Rush County', - 'Toole County', - 'Jefferson County', - 'Dona Ana County', - 'La Porte County') - AND cd_demo_sk = c.c_current_cdemo_sk - AND EXISTS - (SELECT * - FROM store_sales, - date_dim - WHERE c.c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_moy BETWEEN 1 AND 1+3) - AND (EXISTS - (SELECT * - FROM web_sales, - date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_moy BETWEEN 1 AND 1+3) - OR EXISTS - (SELECT * - FROM catalog_sales, - date_dim - WHERE c.c_customer_sk = cs_ship_customer_sk - AND cs_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_moy BETWEEN 1 AND 1+3)) -GROUP BY cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating, - cd_dep_count, - cd_dep_employed_count, - cd_dep_college_count -ORDER BY cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating, - cd_dep_count, - cd_dep_employed_count, - cd_dep_college_count -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q11.sql b/benchmark/tpcds/as_queries/q11.sql deleted file mode 100644 index c7067279..00000000 --- a/benchmark/tpcds/as_queries/q11.sql +++ /dev/null @@ -1,80 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH year_total AS (SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - c_preferred_cust_flag customer_preferred_cust_flag, - c_birth_country customer_birth_country, - c_login customer_login, - c_email_address customer_email_address, - d_year dyear, - as_noised_sum(priv_hash(hash(c_customer_sk)), ss_ext_list_price-ss_ext_discount_amt) year_total, - 's' sale_type - FROM customer, - store_sales, - date_dim - WHERE c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - GROUP BY c_customer_id, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_country, - c_login, - c_email_address, - d_year - UNION ALL SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - c_preferred_cust_flag customer_preferred_cust_flag, - c_birth_country customer_birth_country, - c_login customer_login, - c_email_address customer_email_address, - d_year dyear, - as_noised_sum(priv_hash(hash(c_customer_sk)), ws_ext_list_price-ws_ext_discount_amt) year_total, - 'w' sale_type - FROM customer, - web_sales, - date_dim - WHERE c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - GROUP BY c_customer_id, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_country, - c_login, - c_email_address, - d_year) -SELECT t_s_secyear.customer_id, - t_s_secyear.customer_first_name, - t_s_secyear.customer_last_name, - t_s_secyear.customer_preferred_cust_flag -FROM year_total t_s_firstyear, - year_total t_s_secyear, - year_total t_w_firstyear, - year_total t_w_secyear -WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id - AND t_s_firstyear.customer_id = t_w_secyear.customer_id - AND t_s_firstyear.customer_id = t_w_firstyear.customer_id - AND t_s_firstyear.sale_type = 's' - AND t_w_firstyear.sale_type = 'w' - AND t_s_secyear.sale_type = 's' - AND t_w_secyear.sale_type = 'w' - AND t_s_firstyear.dyear = 2001 - AND t_s_secyear.dyear = 2001+1 - AND t_w_firstyear.dyear = 2001 - AND t_w_secyear.dyear = 2001+1 - AND t_s_firstyear.year_total > 0 - AND t_w_firstyear.year_total > 0 - AND CASE - WHEN t_w_firstyear.year_total > 0 THEN (t_w_secyear.year_total*1.0000) / t_w_firstyear.year_total - ELSE 0.0 - END > CASE - WHEN t_s_firstyear.year_total > 0 THEN (t_s_secyear.year_total*1.0000) / t_s_firstyear.year_total - ELSE 0.0 - END -ORDER BY t_s_secyear.customer_id NULLS FIRST, - t_s_secyear.customer_first_name NULLS FIRST, - t_s_secyear.customer_last_name NULLS FIRST, - t_s_secyear.customer_preferred_cust_flag NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q13.sql b/benchmark/tpcds/as_queries/q13.sql deleted file mode 100644 index d666e013..00000000 --- a/benchmark/tpcds/as_queries/q13.sql +++ /dev/null @@ -1,43 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. - -SELECT (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE)) avg1, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS DOUBLE)) avg2, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_wholesale_cost) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_ext_wholesale_cost) AS DOUBLE)) avg3, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_wholesale_cost) -FROM store_sales , - store , - customer_demographics , - household_demographics , - customer_address , - date_dim -WHERE s_store_sk = ss_store_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2001 and((ss_hdemo_sk=hd_demo_sk - AND cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'M' - AND cd_education_status = 'Advanced Degree' - AND ss_sales_price BETWEEN 100.00 AND 150.00 - AND hd_dep_count = 3) - OR (ss_hdemo_sk=hd_demo_sk - AND cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'S' - AND cd_education_status = 'College' - AND ss_sales_price BETWEEN 50.00 AND 100.00 - AND hd_dep_count = 1 ) - OR (ss_hdemo_sk=hd_demo_sk - AND cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'W' - AND cd_education_status = '2 yr Degree' - AND ss_sales_price BETWEEN 150.00 AND 200.00 - AND hd_dep_count = 1)) and((ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('TX', 'OH', 'TX') - AND ss_net_profit BETWEEN 100 AND 200) - OR (ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('OR', 'NM', 'KY') - AND ss_net_profit BETWEEN 150 AND 300) - OR (ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('VA', 'TX', 'MS') - AND ss_net_profit BETWEEN 50 AND 250)) ; diff --git a/benchmark/tpcds/as_queries/q15.sql b/benchmark/tpcds/as_queries/q15.sql deleted file mode 100644 index 7d7198f6..00000000 --- a/benchmark/tpcds/as_queries/q15.sql +++ /dev/null @@ -1,28 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT ca_zip, - as_noised_sum(priv_hash(hash(c_customer_sk)), cs_sales_price) -FROM catalog_sales, - customer, - customer_address, - date_dim -WHERE cs_bill_customer_sk = c_customer_sk - AND c_current_addr_sk = ca_address_sk - AND (SUBSTRING(ca_zip, 1, 5) IN ('85669', - '86197', - '88274', - '83405', - '86475', - '85392', - '85460', - '80348', - '81792') - OR ca_state IN ('CA', - 'WA', - 'GA') - OR cs_sales_price > 500) - AND cs_sold_date_sk = d_date_sk - AND d_qoy = 2 - AND d_year = 2001 -GROUP BY ca_zip -ORDER BY ca_zip NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q16.sql b/benchmark/tpcds/as_queries/q16.sql deleted file mode 100644 index 757dcb1f..00000000 --- a/benchmark/tpcds/as_queries/q16.sql +++ /dev/null @@ -1,24 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- COUNT(DISTINCT) unions masks per order before counting; SUMs retain their own source scan. -WITH asa_input AS NOT MATERIALIZED (SELECT cs_order_number, cs_ext_ship_cost, cs_net_profit, priv_hash(hash(cs1.cs_bill_customer_sk)) AS membership FROM catalog_sales cs1, - date_dim, - customer_address, - call_center -WHERE d_date BETWEEN '2002-02-01' AND cast('2002-04-02' AS date) - AND cs1.cs_ship_date_sk = d_date_sk - AND cs1.cs_ship_addr_sk = ca_address_sk - AND ca_state = 'GA' - AND cs1.cs_call_center_sk = cc_call_center_sk - AND cc_county = 'Williamson County' - AND EXISTS - (SELECT * - FROM catalog_sales cs2 - WHERE cs1.cs_order_number = cs2.cs_order_number - AND cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) - AND NOT EXISTS - (SELECT * - FROM catalog_returns cr1 - WHERE cs1.cs_order_number = cr1.cr_order_number) -), -asa_distinct AS NOT MATERIALIZED (SELECT cs_order_number, bit_or(membership) FILTER (WHERE cs_order_number IS NOT NULL) AS membership FROM asa_input GROUP BY cs_order_number) -SELECT d.n AS "order count", s.shipping AS "total shipping cost", s.profit AS "total net profit" FROM (SELECT as_noised_count(membership, 1::BIGINT) AS n FROM asa_distinct) d CROSS JOIN (SELECT as_noised_sum(membership, cs_ext_ship_cost) AS shipping, as_noised_sum(membership, cs_net_profit) AS profit FROM asa_input) s ORDER BY "order count" LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q17.sql b/benchmark/tpcds/as_queries/q17.sql deleted file mode 100644 index ea247a6d..00000000 --- a/benchmark/tpcds/as_queries/q17.sql +++ /dev/null @@ -1,48 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT i_item_id, - i_item_desc, - s_state, - as_noised_count(priv_hash(hash(ss_customer_sk)), ss_quantity) AS store_sales_quantitycount, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE)) AS store_sales_quantityave, - as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), ss_quantity) AS store_sales_quantitystdev, - as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), ss_quantity)/(CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), ss_quantity) AS DOUBLE)) AS store_sales_quantitycov, - as_noised_count(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS store_returns_quantitycount, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS DOUBLE)) AS store_returns_quantityave, - as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS store_returns_quantitystdev, - as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), sr_return_quantity)/(CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS DOUBLE)) AS store_returns_quantitycov, - as_noised_count(priv_hash(hash(ss_customer_sk)), cs_quantity) AS catalog_sales_quantitycount, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), cs_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), cs_quantity) AS DOUBLE)) AS catalog_sales_quantityave, - as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), cs_quantity) AS catalog_sales_quantitystdev, - as_noised_stddev_samp(priv_hash(hash(ss_customer_sk)), cs_quantity)/(CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), cs_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), cs_quantity) AS DOUBLE)) AS catalog_sales_quantitycov -FROM store_sales, - store_returns, - catalog_sales, - date_dim d1, - date_dim d2, - date_dim d3, - store, - item -WHERE d1.d_quarter_name = '2001Q1' - AND d1.d_date_sk = ss_sold_date_sk - AND i_item_sk = ss_item_sk - AND s_store_sk = ss_store_sk - AND ss_customer_sk = sr_customer_sk - AND ss_item_sk = sr_item_sk - AND ss_ticket_number = sr_ticket_number - AND sr_returned_date_sk = d2.d_date_sk - AND d2.d_quarter_name IN ('2001Q1', - '2001Q2', - '2001Q3') - AND sr_customer_sk = cs_bill_customer_sk - AND sr_item_sk = cs_item_sk - AND cs_sold_date_sk = d3.d_date_sk - AND d3.d_quarter_name IN ('2001Q1', - '2001Q2', - '2001Q3') -GROUP BY i_item_id, - i_item_desc, - s_state -ORDER BY i_item_id NULLS FIRST, - i_item_desc NULLS FIRST, - s_state NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q18.sql b/benchmark/tpcds/as_queries/q18.sql deleted file mode 100644 index c04011b9..00000000 --- a/benchmark/tpcds/as_queries/q18.sql +++ /dev/null @@ -1,50 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT i_item_id, - ca_country, - ca_state, - ca_county, - (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cs_quantity AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cs_quantity AS decimal(12, 2))) AS DOUBLE)) agg1, - (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cs_list_price AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cs_list_price AS decimal(12, 2))) AS DOUBLE)) agg2, - (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cs_coupon_amt AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cs_coupon_amt AS decimal(12, 2))) AS DOUBLE)) agg3, - (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cs_sales_price AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cs_sales_price AS decimal(12, 2))) AS DOUBLE)) agg4, - (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cs_net_profit AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cs_net_profit AS decimal(12, 2))) AS DOUBLE)) agg5, - (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(c_birth_year AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(c_birth_year AS decimal(12, 2))) AS DOUBLE)) agg6, - (CAST(as_noised_sum(priv_hash(hash(c_customer_sk)), cast(cd1.cd_dep_count AS decimal(12, 2))) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c_customer_sk)), cast(cd1.cd_dep_count AS decimal(12, 2))) AS DOUBLE)) agg7 -FROM catalog_sales, - customer_demographics cd1, - customer_demographics cd2, - customer, - customer_address, - date_dim, - item -WHERE cs_sold_date_sk = d_date_sk - AND cs_item_sk = i_item_sk - AND cs_bill_cdemo_sk = cd1.cd_demo_sk - AND cs_bill_customer_sk = c_customer_sk - AND cd1.cd_gender = 'F' - AND cd1.cd_education_status = 'Unknown' - AND c_current_cdemo_sk = cd2.cd_demo_sk - AND c_current_addr_sk = ca_address_sk - AND c_birth_month IN (1, - 6, - 8, - 9, - 12, - 2) - AND d_year = 1998 - AND ca_state IN ('MS', - 'IN', - 'ND', - 'OK', - 'NM', - 'VA', - 'MS') -GROUP BY ROLLUP (i_item_id, - ca_country, - ca_state, - ca_county) -ORDER BY ca_country NULLS FIRST, - ca_state NULLS FIRST, - ca_county NULLS FIRST, - i_item_id NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q19.sql b/benchmark/tpcds/as_queries/q19.sql deleted file mode 100644 index 1488e632..00000000 --- a/benchmark/tpcds/as_queries/q19.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT i_brand_id brand_id, - i_brand brand, - i_manufact_id, - i_manufact, - as_noised_sum(priv_hash(hash(c_customer_sk)), ss_ext_sales_price) ext_price -FROM date_dim, - store_sales, - item, - customer, - customer_address, - store -WHERE d_date_sk = ss_sold_date_sk - AND ss_item_sk = i_item_sk - AND i_manager_id=8 - AND d_moy=11 - AND d_year=1998 - AND ss_customer_sk = c_customer_sk - AND c_current_addr_sk = ca_address_sk - AND SUBSTRING(ca_zip, 1, 5) <> SUBSTRING(s_zip, 1, 5) - AND ss_store_sk = s_store_sk -GROUP BY i_brand, - i_brand_id, - i_manufact_id, - i_manufact -ORDER BY ext_price DESC, - i_brand, - i_brand_id, - i_manufact_id, - i_manufact -LIMIT 100 ; diff --git a/benchmark/tpcds/as_queries/q23.sql b/benchmark/tpcds/as_queries/q23.sql deleted file mode 100644 index ffebd1d1..00000000 --- a/benchmark/tpcds/as_queries/q23.sql +++ /dev/null @@ -1,83 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Per-customer inner totals remain ordinary; the aggregate across those totals is stochastic. -WITH frequent_ss_items AS (SELECT itemdesc, - i_item_sk item_sk, - d_date solddate, - as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) cnt - FROM store_sales, - date_dim, - (SELECT SUBSTRING(i_item_desc, 1, 30) itemdesc, - * - FROM item) sq1 - WHERE ss_sold_date_sk = d_date_sk - AND ss_item_sk = i_item_sk - AND d_year IN (2000, - 2000+1, - 2000+2, - 2000+3) - GROUP BY itemdesc, - i_item_sk, - d_date - HAVING as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) >4), -max_store_sales AS (SELECT as_noised_max(priv_hash(hash(c_customer_sk)), csales) tpcds_cmax - FROM - (SELECT c_customer_sk, - sum(ss_quantity*ss_sales_price) csales - FROM store_sales, - customer, - date_dim - WHERE ss_customer_sk = c_customer_sk - AND ss_sold_date_sk = d_date_sk - AND d_year IN (2000, - 2000+1, - 2000+2, - 2000+3) - GROUP BY c_customer_sk) sq2), -best_ss_customer AS (SELECT c_customer_sk, - as_noised_sum(priv_hash(hash(c_customer_sk)), ss_quantity*ss_sales_price) ssales - FROM store_sales, - customer, - max_store_sales - WHERE ss_customer_sk = c_customer_sk - GROUP BY c_customer_sk - HAVING as_noised_sum(priv_hash(hash(c_customer_sk)), ss_quantity*ss_sales_price) > (50/100.0) * as_noised_max(priv_hash(hash(c_customer_sk)), tpcds_cmax)) -SELECT c_last_name, - c_first_name, - sales -FROM - (SELECT c_last_name, - c_first_name, - as_noised_sum(priv_hash(hash(customer.c_customer_sk)), cs_quantity*cs_list_price) sales - FROM catalog_sales, - customer, - date_dim, - frequent_ss_items, - best_ss_customer - WHERE d_year = 2000 - AND d_moy = 2 - AND cs_sold_date_sk = d_date_sk - AND cs_item_sk = item_sk - AND cs_bill_customer_sk = best_ss_customer.c_customer_sk - AND cs_bill_customer_sk = customer.c_customer_sk - GROUP BY c_last_name, - c_first_name - UNION ALL SELECT c_last_name, - c_first_name, - as_noised_sum(priv_hash(hash(customer.c_customer_sk)), ws_quantity*ws_list_price) sales - FROM web_sales, - customer, - date_dim, - frequent_ss_items, - best_ss_customer - WHERE d_year = 2000 - AND d_moy = 2 - AND ws_sold_date_sk = d_date_sk - AND ws_item_sk = item_sk - AND ws_bill_customer_sk = best_ss_customer.c_customer_sk - AND ws_bill_customer_sk = customer.c_customer_sk - GROUP BY c_last_name, - c_first_name) sq3 -ORDER BY c_last_name NULLS FIRST, - c_first_name NULLS FIRST, - sales NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q24.sql b/benchmark/tpcds/as_queries/q24.sql deleted file mode 100644 index 7e61d583..00000000 --- a/benchmark/tpcds/as_queries/q24.sql +++ /dev/null @@ -1,53 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- The ssales CTE scalarizes its SUM; later SUM/AVG and HAVING operate on those scalar values. -WITH ssales AS (SELECT c_last_name, - c_first_name, - s_store_name, - ca_state, - s_state, - i_color, - i_current_price, - i_manager_id, - i_units, - i_size, - as_noised_sum(priv_hash(hash(c_customer_sk)), ss_net_paid) netpaid - FROM store_sales, - store_returns, - store, - item, - customer, - customer_address - WHERE ss_ticket_number = sr_ticket_number - AND ss_item_sk = sr_item_sk - AND ss_customer_sk = c_customer_sk - AND ss_item_sk = i_item_sk - AND ss_store_sk = s_store_sk - AND c_current_addr_sk = ca_address_sk - AND c_birth_country <> upper(ca_country) - AND s_zip = ca_zip - AND s_market_id=8 - GROUP BY c_last_name, - c_first_name, - s_store_name, - ca_state, - s_state, - i_color, - i_current_price, - i_manager_id, - i_units, - i_size) -SELECT c_last_name, - c_first_name, - s_store_name, - sum(netpaid) paid -FROM ssales -WHERE i_color = 'peach' -GROUP BY c_last_name, - c_first_name, - s_store_name -HAVING sum(netpaid) > - (SELECT 0.05*avg(netpaid) - FROM ssales) -ORDER BY c_last_name, - c_first_name, - s_store_name ; diff --git a/benchmark/tpcds/as_queries/q25.sql b/benchmark/tpcds/as_queries/q25.sql deleted file mode 100644 index b4e038e6..00000000 --- a/benchmark/tpcds/as_queries/q25.sql +++ /dev/null @@ -1,42 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. - -SELECT i_item_id , - i_item_desc , - s_store_id , - s_store_name , - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) AS store_sales_profit , - as_noised_sum(priv_hash(hash(ss_customer_sk)), sr_net_loss) AS store_returns_loss , - as_noised_sum(priv_hash(hash(ss_customer_sk)), cs_net_profit) AS catalog_sales_profit -FROM store_sales , - store_returns , - catalog_sales , - date_dim d1 , - date_dim d2 , - date_dim d3 , - store , - item -WHERE d1.d_moy = 4 - AND d1.d_year = 2001 - AND d1.d_date_sk = ss_sold_date_sk - AND i_item_sk = ss_item_sk - AND s_store_sk = ss_store_sk - AND ss_customer_sk = sr_customer_sk - AND ss_item_sk = sr_item_sk - AND ss_ticket_number = sr_ticket_number - AND sr_returned_date_sk = d2.d_date_sk - AND d2.d_moy BETWEEN 4 AND 10 - AND d2.d_year = 2001 - AND sr_customer_sk = cs_bill_customer_sk - AND sr_item_sk = cs_item_sk - AND cs_sold_date_sk = d3.d_date_sk - AND d3.d_moy BETWEEN 4 AND 10 - AND d3.d_year = 2001 -GROUP BY i_item_id , - i_item_desc , - s_store_id , - s_store_name -ORDER BY i_item_id , - i_item_desc , - s_store_id , - s_store_name -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q26.sql b/benchmark/tpcds/as_queries/q26.sql deleted file mode 100644 index 36b06fce..00000000 --- a/benchmark/tpcds/as_queries/q26.sql +++ /dev/null @@ -1,24 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT i_item_id, - (CAST(as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(cs_bill_customer_sk)), cs_quantity) AS DOUBLE)) agg1, - (CAST(as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_list_price) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(cs_bill_customer_sk)), cs_list_price) AS DOUBLE)) agg2, - (CAST(as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_coupon_amt) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(cs_bill_customer_sk)), cs_coupon_amt) AS DOUBLE)) agg3, - (CAST(as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_sales_price) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(cs_bill_customer_sk)), cs_sales_price) AS DOUBLE)) agg4 -FROM catalog_sales, - customer_demographics, - date_dim, - item, - promotion -WHERE cs_sold_date_sk = d_date_sk - AND cs_item_sk = i_item_sk - AND cs_bill_cdemo_sk = cd_demo_sk - AND cs_promo_sk = p_promo_sk - AND cd_gender = 'M' - AND cd_marital_status = 'S' - AND cd_education_status = 'College' - AND (p_channel_email = 'N' - OR p_channel_event = 'N') - AND d_year = 2000 -GROUP BY i_item_id -ORDER BY i_item_id -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q27.sql b/benchmark/tpcds/as_queries/q27.sql deleted file mode 100644 index 93f09123..00000000 --- a/benchmark/tpcds/as_queries/q27.sql +++ /dev/null @@ -1,61 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Carry the source customer key through the row-level CTE into all three UNION aggregate scopes. -WITH results AS (SELECT ss_customer_sk, i_item_id, - s_state, - 0 AS g_state, - ss_quantity agg1, - ss_list_price agg2, - ss_coupon_amt agg3, - ss_sales_price agg4 - FROM store_sales, - customer_demographics, - date_dim, - store, - item - WHERE ss_sold_date_sk = d_date_sk - AND ss_item_sk = i_item_sk - AND ss_store_sk = s_store_sk - AND ss_cdemo_sk = cd_demo_sk - AND cd_gender = 'M' - AND cd_marital_status = 'S' - AND cd_education_status = 'College' - AND d_year = 2002 - AND s_state = 'TN' ) -SELECT i_item_id, - s_state, - g_state, - agg1, - agg2, - agg3, - agg4 -FROM - ( SELECT i_item_id, - s_state, - 0 AS g_state, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE)) agg1, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE)) agg2, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE)) agg3, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE)) agg4 - FROM results - GROUP BY i_item_id , - s_state - UNION ALL SELECT i_item_id, - NULL AS s_state, - 1 AS g_state, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE)) agg1, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE)) agg2, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE)) agg3, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE)) agg4 - FROM results - GROUP BY i_item_id - UNION ALL SELECT NULL AS i_item_id, - NULL AS s_state, - 1 AS g_state, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg1) AS DOUBLE)) agg1, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg2) AS DOUBLE)) agg2, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg3) AS DOUBLE)) agg3, - (CAST(as_noised_sum(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ss_customer_sk)), agg4) AS DOUBLE)) agg4 - FROM results ) foo -ORDER BY i_item_id NULLS FIRST, - s_state NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q28.sql b/benchmark/tpcds/as_queries/q28.sql deleted file mode 100644 index 2a5d29f8..00000000 --- a/benchmark/tpcds/as_queries/q28.sql +++ /dev/null @@ -1,45 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Each bucket has independent scalar aggregates and a mask-union DISTINCT-count branch. -WITH b1_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales - WHERE ss_quantity BETWEEN 0 AND 5 - AND (ss_list_price BETWEEN 8 AND 8+10 - OR ss_coupon_amt BETWEEN 459 AND 459+1000 - OR ss_wholesale_cost BETWEEN 57 AND 57+20)), -b1_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b1_input GROUP BY ss_list_price), -b1 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B1_LP, as_noised_count(membership,ss_list_price) AS B1_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b1_unique) AS B1_CNTD FROM b1_input), -b2_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales - WHERE ss_quantity BETWEEN 6 AND 10 - AND (ss_list_price BETWEEN 90 AND 90+10 - OR ss_coupon_amt BETWEEN 2323 AND 2323+1000 - OR ss_wholesale_cost BETWEEN 31 AND 31+20)), -b2_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b2_input GROUP BY ss_list_price), -b2 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B2_LP, as_noised_count(membership,ss_list_price) AS B2_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b2_unique) AS B2_CNTD FROM b2_input), -b3_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales - WHERE ss_quantity BETWEEN 11 AND 15 - AND (ss_list_price BETWEEN 142 AND 142+10 - OR ss_coupon_amt BETWEEN 12214 AND 12214+1000 - OR ss_wholesale_cost BETWEEN 79 AND 79+20)), -b3_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b3_input GROUP BY ss_list_price), -b3 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B3_LP, as_noised_count(membership,ss_list_price) AS B3_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b3_unique) AS B3_CNTD FROM b3_input), -b4_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales - WHERE ss_quantity BETWEEN 16 AND 20 - AND (ss_list_price BETWEEN 135 AND 135+10 - OR ss_coupon_amt BETWEEN 6071 AND 6071+1000 - OR ss_wholesale_cost BETWEEN 38 AND 38+20)), -b4_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b4_input GROUP BY ss_list_price), -b4 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B4_LP, as_noised_count(membership,ss_list_price) AS B4_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b4_unique) AS B4_CNTD FROM b4_input), -b5_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales - WHERE ss_quantity BETWEEN 21 AND 25 - AND (ss_list_price BETWEEN 122 AND 122+10 - OR ss_coupon_amt BETWEEN 836 AND 836+1000 - OR ss_wholesale_cost BETWEEN 17 AND 17+20)), -b5_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b5_input GROUP BY ss_list_price), -b5 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B5_LP, as_noised_count(membership,ss_list_price) AS B5_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b5_unique) AS B5_CNTD FROM b5_input), -b6_input AS NOT MATERIALIZED (SELECT ss_list_price, priv_hash(hash(ss_customer_sk)) AS membership FROM store_sales - WHERE ss_quantity BETWEEN 26 AND 30 - AND (ss_list_price BETWEEN 154 AND 154+10 - OR ss_coupon_amt BETWEEN 7326 AND 7326+1000 - OR ss_wholesale_cost BETWEEN 7 AND 7+20)), -b6_unique AS NOT MATERIALIZED (SELECT ss_list_price, bit_or(membership) FILTER (WHERE ss_list_price IS NOT NULL) AS membership FROM b6_input GROUP BY ss_list_price), -b6 AS NOT MATERIALIZED (SELECT CAST(as_noised_sum(membership,ss_list_price) AS DOUBLE)/CAST(as_noised_count(membership,ss_list_price) AS DOUBLE) AS B6_LP, as_noised_count(membership,ss_list_price) AS B6_CNT, (SELECT as_noised_count(membership,1::BIGINT) FROM b6_unique) AS B6_CNTD FROM b6_input) -SELECT * FROM b1, b2, b3, b4, b5, b6 LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q29.sql b/benchmark/tpcds/as_queries/q29.sql deleted file mode 100644 index a58132ca..00000000 --- a/benchmark/tpcds/as_queries/q29.sql +++ /dev/null @@ -1,42 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT i_item_id, - i_item_desc, - s_store_id, - s_store_name, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) AS store_sales_quantity, - as_noised_sum(priv_hash(hash(ss_customer_sk)), sr_return_quantity) AS store_returns_quantity, - as_noised_sum(priv_hash(hash(ss_customer_sk)), cs_quantity) AS catalog_sales_quantity -FROM store_sales, - store_returns, - catalog_sales, - date_dim d1, - date_dim d2, - date_dim d3, - store, - item -WHERE d1.d_moy = 9 - AND d1.d_year = 1999 - AND d1.d_date_sk = ss_sold_date_sk - AND i_item_sk = ss_item_sk - AND s_store_sk = ss_store_sk - AND ss_customer_sk = sr_customer_sk - AND ss_item_sk = sr_item_sk - AND ss_ticket_number = sr_ticket_number - AND sr_returned_date_sk = d2.d_date_sk - AND d2.d_moy BETWEEN 9 AND 9 + 3 - AND d2.d_year = 1999 - AND sr_customer_sk = cs_bill_customer_sk - AND sr_item_sk = cs_item_sk - AND cs_sold_date_sk = d3.d_date_sk - AND d3.d_year IN (1999, - 1999+1, - 1999+2) -GROUP BY i_item_id, - i_item_desc, - s_store_id, - s_store_name -ORDER BY i_item_id, - i_item_desc, - s_store_id, - s_store_name -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q30.sql b/benchmark/tpcds/as_queries/q30.sql deleted file mode 100644 index 7c835ab5..00000000 --- a/benchmark/tpcds/as_queries/q30.sql +++ /dev/null @@ -1,49 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Per-customer CTE totals stay scalar; only the correlated cross-customer AVG supplies a counter list. -WITH customer_total_return AS (SELECT wr_returning_customer_sk AS ctr_customer_sk, - ca_state AS ctr_state, - sum(wr_return_amt) AS ctr_total_return - FROM web_returns, - date_dim, - customer_address - WHERE wr_returned_date_sk = d_date_sk - AND d_year = 2002 - AND wr_returning_addr_sk = ca_address_sk - GROUP BY wr_returning_customer_sk, - ca_state) -SELECT c_customer_id, - c_salutation, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_day, - c_birth_month, - c_birth_year, - c_birth_country, - c_login, - c_email_address, - c_last_review_date_sk, - ctr_total_return -FROM customer_total_return ctr1, - customer_address, - customer -WHERE priv_filter_gt(CAST(CAST(ctr1.ctr_total_return AS DOUBLE) * 0.8333333333333334 AS FLOAT), (SELECT priv_div(as_sum(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return), as_count(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return)) - FROM customer_total_return ctr2 - WHERE ctr1.ctr_state = ctr2.ctr_state)) - AND ca_address_sk = c_current_addr_sk - AND ca_state = 'GA' - AND ctr1.ctr_customer_sk = c_customer_sk -ORDER BY c_customer_id NULLS FIRST, - c_salutation NULLS FIRST, - c_first_name NULLS FIRST, - c_last_name NULLS FIRST, - c_preferred_cust_flag NULLS FIRST, - c_birth_day NULLS FIRST, - c_birth_month NULLS FIRST, - c_birth_year NULLS FIRST, - c_birth_country NULLS FIRST, - c_login NULLS FIRST, - c_email_address NULLS FIRST, - c_last_review_date_sk NULLS FIRST, - ctr_total_return NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q31.sql b/benchmark/tpcds/as_queries/q31.sql deleted file mode 100644 index a232e042..00000000 --- a/benchmark/tpcds/as_queries/q31.sql +++ /dev/null @@ -1,69 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH ss AS (SELECT ca_county, - d_qoy, - d_year, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS store_sales - FROM store_sales, - date_dim, - customer_address - WHERE ss_sold_date_sk = d_date_sk - AND ss_addr_sk=ca_address_sk - GROUP BY ca_county, - d_qoy, - d_year), -ws AS (SELECT ca_county, - d_qoy, - d_year, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) AS web_sales - FROM web_sales, - date_dim, - customer_address - WHERE ws_sold_date_sk = d_date_sk - AND ws_bill_addr_sk=ca_address_sk - GROUP BY ca_county, - d_qoy, - d_year) -SELECT ss1.ca_county , - ss1.d_year , - (ws2.web_sales*1.0000)/ws1.web_sales web_q1_q2_increase , - (ss2.store_sales*1.0000)/ss1.store_sales store_q1_q2_increase , - (ws3.web_sales*1.0000)/ws2.web_sales web_q2_q3_increase , - (ss3.store_sales*1.0000)/ss2.store_sales store_q2_q3_increase -FROM ss ss1 , - ss ss2 , - ss ss3 , - ws ws1 , - ws ws2 , - ws ws3 -WHERE ss1.d_qoy = 1 - AND ss1.d_year = 2000 - AND ss1.ca_county = ss2.ca_county - AND ss2.d_qoy = 2 - AND ss2.d_year = 2000 - AND ss2.ca_county = ss3.ca_county - AND ss3.d_qoy = 3 - AND ss3.d_year = 2000 - AND ss1.ca_county = ws1.ca_county - AND ws1.d_qoy = 1 - AND ws1.d_year = 2000 - AND ws1.ca_county = ws2.ca_county - AND ws2.d_qoy = 2 - AND ws2.d_year = 2000 - AND ws1.ca_county = ws3.ca_county - AND ws3.d_qoy = 3 - AND ws3.d_year = 2000 - AND CASE - WHEN ws1.web_sales > 0 THEN (ws2.web_sales*1.0000)/ws1.web_sales - ELSE NULL - END > CASE - WHEN ss1.store_sales > 0 THEN (ss2.store_sales*1.0000)/ss1.store_sales - ELSE NULL - END - AND CASE - WHEN ws2.web_sales > 0 THEN (ws3.web_sales*1.0000)/ws2.web_sales - ELSE NULL - END > CASE - WHEN ss2.store_sales > 0 THEN (ss3.store_sales*1.0000)/ss2.store_sales - ELSE NULL - END -ORDER BY ss1.ca_county; diff --git a/benchmark/tpcds/as_queries/q32.sql b/benchmark/tpcds/as_queries/q32.sql deleted file mode 100644 index 9c4ae6b7..00000000 --- a/benchmark/tpcds/as_queries/q32.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- The correlated threshold narrows the tuple mask before the outer stochastic SUM. -SELECT as_noised_sum(priv_select_gt(priv_hash(hash(cs_bill_customer_sk)), CAST(cs_ext_discount_amt AS FLOAT), ( SELECT list_transform(priv_div(as_sum(priv_hash(hash(cs_bill_customer_sk)), CAST(cs_ext_discount_amt AS DOUBLE)), as_count(priv_hash(hash(cs_bill_customer_sk)), cs_ext_discount_amt)), lambda x: CAST(1.3 * x AS FLOAT)) - FROM catalog_sales , - date_dim - WHERE cs_item_sk = i_item_sk - AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) - AND d_date_sk = cs_sold_date_sk )), cs_ext_discount_amt) AS "excess discount amount" -FROM catalog_sales , - item , - date_dim -WHERE i_manufact_id = 977 - AND i_item_sk = cs_item_sk - AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) - AND d_date_sk = cs_sold_date_sk - AND priv_select_gt(priv_hash(hash(cs_bill_customer_sk)), CAST(cs_ext_discount_amt AS FLOAT), ( SELECT list_transform(priv_div(as_sum(priv_hash(hash(cs_bill_customer_sk)), CAST(cs_ext_discount_amt AS DOUBLE)), as_count(priv_hash(hash(cs_bill_customer_sk)), cs_ext_discount_amt)), lambda x: CAST(1.3 * x AS FLOAT)) - FROM catalog_sales , - date_dim - WHERE cs_item_sk = i_item_sk - AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) - AND d_date_sk = cs_sold_date_sk )) <> 0 -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q33.sql b/benchmark/tpcds/as_queries/q33.sql deleted file mode 100644 index c4288e8c..00000000 --- a/benchmark/tpcds/as_queries/q33.sql +++ /dev/null @@ -1,64 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH ss AS ( SELECT i_manufact_id, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales - FROM store_sales, - date_dim, - customer_address, - item - WHERE i_manufact_id IN - (SELECT i_manufact_id - FROM item - WHERE i_category IN ('Electronics')) - AND ss_item_sk = i_item_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 5 - AND ss_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_manufact_id), -cs AS ( SELECT i_manufact_id, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales - FROM catalog_sales, - date_dim, - customer_address, - item - WHERE i_manufact_id IN - (SELECT i_manufact_id - FROM item - WHERE i_category IN ('Electronics')) - AND cs_item_sk = i_item_sk - AND cs_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 5 - AND cs_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_manufact_id), -ws AS ( SELECT i_manufact_id, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales - FROM web_sales, - date_dim, - customer_address, - item - WHERE i_manufact_id IN - (SELECT i_manufact_id - FROM item - WHERE i_category IN ('Electronics')) - AND ws_item_sk = i_item_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 5 - AND ws_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_manufact_id) -SELECT i_manufact_id, - sum(total_sales) total_sales -FROM - (SELECT * - FROM ss - UNION ALL SELECT * - FROM cs - UNION ALL SELECT * - FROM ws) tmp1 -GROUP BY i_manufact_id -ORDER BY total_sales -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q34.sql b/benchmark/tpcds/as_queries/q34.sql deleted file mode 100644 index e7cdd84b..00000000 --- a/benchmark/tpcds/as_queries/q34.sql +++ /dev/null @@ -1,42 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- The terminal BETWEEN uses two separate PAC majority comparisons, not one per-world conjunction. -SELECT c_last_name , - c_first_name , - c_salutation , - c_preferred_cust_flag , - ss_ticket_number , - CAST(as_noised(cnt) AS BIGINT) AS cnt -FROM - (SELECT ss_ticket_number , - ss_customer_sk , - as_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) cnt - FROM store_sales, - date_dim, - store, - household_demographics - WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk - AND store_sales.ss_store_sk = store.s_store_sk - AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - AND (date_dim.d_dom BETWEEN 1 AND 3 - OR date_dim.d_dom BETWEEN 25 AND 28) - AND (household_demographics.hd_buy_potential = '>10000' - OR household_demographics.hd_buy_potential = 'Unknown') - AND household_demographics.hd_vehicle_count > 0 - AND (CASE - WHEN household_demographics.hd_vehicle_count > 0 THEN (household_demographics.hd_dep_count*1.000)/ household_demographics.hd_vehicle_count - ELSE NULL - END) > 1.2 - AND date_dim.d_year IN (1999, - 1999+1, - 1999+2) - AND store.s_county = 'Williamson County' - GROUP BY ss_ticket_number, - ss_customer_sk) dn, - customer -WHERE ss_customer_sk = c_customer_sk - AND priv_filter_lte(15::FLOAT, cnt) AND priv_filter_gte(20::FLOAT, cnt) -ORDER BY c_last_name NULLS FIRST, - c_first_name NULLS FIRST, - c_salutation NULLS FIRST, - c_preferred_cust_flag DESC NULLS FIRST, - ss_ticket_number NULLS FIRST; diff --git a/benchmark/tpcds/as_queries/q35.sql b/benchmark/tpcds/as_queries/q35.sql deleted file mode 100644 index a6a41304..00000000 --- a/benchmark/tpcds/as_queries/q35.sql +++ /dev/null @@ -1,61 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -SELECT ca_state, - cd_gender, - cd_marital_status, - cd_dep_count, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt1, - as_noised_min(priv_hash(hash(c.c_customer_sk)), cd_dep_count) min1, - as_noised_max(priv_hash(hash(c.c_customer_sk)), cd_dep_count) max1, - (CAST(as_noised_sum(priv_hash(hash(c.c_customer_sk)), cd_dep_count) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c.c_customer_sk)), cd_dep_count) AS DOUBLE)) avg1, - cd_dep_employed_count, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt2, - as_noised_min(priv_hash(hash(c.c_customer_sk)), cd_dep_employed_count) min2, - as_noised_max(priv_hash(hash(c.c_customer_sk)), cd_dep_employed_count) max2, - (CAST(as_noised_sum(priv_hash(hash(c.c_customer_sk)), cd_dep_employed_count) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c.c_customer_sk)), cd_dep_employed_count) AS DOUBLE)) avg2, - cd_dep_college_count, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt3, - as_noised_min(priv_hash(hash(c.c_customer_sk)), cd_dep_college_count), - as_noised_max(priv_hash(hash(c.c_customer_sk)), cd_dep_college_count), - (CAST(as_noised_sum(priv_hash(hash(c.c_customer_sk)), cd_dep_college_count) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(c.c_customer_sk)), cd_dep_college_count) AS DOUBLE)) -FROM customer c, - customer_address ca, - customer_demographics -WHERE c.c_current_addr_sk = ca.ca_address_sk - AND cd_demo_sk = c.c_current_cdemo_sk - AND EXISTS - (SELECT * - FROM store_sales, - date_dim - WHERE c.c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) - AND (EXISTS - (SELECT * - FROM web_sales, - date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) - OR EXISTS - (SELECT * - FROM catalog_sales, - date_dim - WHERE c.c_customer_sk = cs_ship_customer_sk - AND cs_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4)) -GROUP BY ca_state, - cd_gender, - cd_marital_status, - cd_dep_count, - cd_dep_employed_count, - cd_dep_college_count -ORDER BY ca_state NULLS FIRST, - cd_gender NULLS FIRST, - cd_marital_status NULLS FIRST, - cd_dep_count NULLS FIRST, - cd_dep_employed_count NULLS FIRST, - cd_dep_college_count NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q40.sql b/benchmark/tpcds/as_queries/q40.sql deleted file mode 100644 index 18e49653..00000000 --- a/benchmark/tpcds/as_queries/q40.sql +++ /dev/null @@ -1,27 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- LEFT JOIN uses the catalog-sales owner once; do not intersect the return customer mask. -SELECT w_state, - i_item_id, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE - WHEN (cast(d_date AS date) < CAST ('2000-03-11' AS date)) THEN cs_sales_price - coalesce(cr_refunded_cash,0) - ELSE 0 - END) AS sales_before, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE - WHEN (cast(d_date AS date) >= CAST ('2000-03-11' AS date)) THEN cs_sales_price - coalesce(cr_refunded_cash,0) - ELSE 0 - END) AS sales_after -FROM catalog_sales -LEFT OUTER JOIN catalog_returns ON (cs_order_number = cr_order_number - AND cs_item_sk = cr_item_sk) ,warehouse, - item, - date_dim -WHERE i_current_price BETWEEN 0.99 AND 1.49 - AND i_item_sk = cs_item_sk - AND cs_warehouse_sk = w_warehouse_sk - AND cs_sold_date_sk = d_date_sk - AND d_date BETWEEN CAST ('2000-02-10' AS date) AND CAST ('2000-04-10' AS date) -GROUP BY w_state, - i_item_id -ORDER BY w_state, - i_item_id -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q42.sql b/benchmark/tpcds/as_queries/q42.sql deleted file mode 100644 index c0bc6259..00000000 --- a/benchmark/tpcds/as_queries/q42.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH asa_states AS (SELECT dt.d_year AS d_year, item.i_category_id AS i_category_id, item.i_category AS i_category, as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_sales_price AS DOUBLE)) AS sum_agg FROM date_dim dt, - store_sales, - item -WHERE dt.d_date_sk = store_sales.ss_sold_date_sk - AND store_sales.ss_item_sk = item.i_item_sk - AND item.i_manager_id = 1 - AND dt.d_moy=11 - AND dt.d_year=2000 -GROUP BY dt.d_year, - item.i_category_id, - item.i_category -), -asa_ranked AS (SELECT * FROM asa_states ORDER BY priv_mean(sum_agg) DESC, d_year, i_category_id, i_category LIMIT 100) -SELECT d_year, i_category_id, i_category, CAST(as_noised(sum_agg) AS DECIMAL(38,2)) AS sum_agg FROM asa_ranked ORDER BY sum_agg DESC, d_year, i_category_id, i_category; diff --git a/benchmark/tpcds/as_queries/q43.sql b/benchmark/tpcds/as_queries/q43.sql deleted file mode 100644 index caafc1e3..00000000 --- a/benchmark/tpcds/as_queries/q43.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH asa_states AS (SELECT s_store_name, s_store_id, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Sunday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS sun_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Monday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS mon_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Tuesday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS tue_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Wednesday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS wed_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Thursday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS thu_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Friday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS fri_sales, as_sum(priv_hash(hash(ss_customer_sk)), CAST(CASE WHEN d_day_name='Saturday' THEN ss_sales_price ELSE NULL END AS DOUBLE)) AS sat_sales FROM date_dim, - store_sales, - store -WHERE d_date_sk = ss_sold_date_sk - AND s_store_sk = ss_store_sk - AND s_gmt_offset = -5 - AND d_year = 2000 -GROUP BY s_store_name, - s_store_id -), -asa_ranked AS (SELECT * FROM asa_states ORDER BY s_store_name, s_store_id, priv_mean(sun_sales), priv_mean(mon_sales), priv_mean(tue_sales), priv_mean(wed_sales), priv_mean(thu_sales), priv_mean(fri_sales), priv_mean(sat_sales) LIMIT 100) -SELECT s_store_name, s_store_id, CAST(as_noised(sun_sales) AS DECIMAL(38,2)) AS sun_sales, CAST(as_noised(mon_sales) AS DECIMAL(38,2)) AS mon_sales, CAST(as_noised(tue_sales) AS DECIMAL(38,2)) AS tue_sales, CAST(as_noised(wed_sales) AS DECIMAL(38,2)) AS wed_sales, CAST(as_noised(thu_sales) AS DECIMAL(38,2)) AS thu_sales, CAST(as_noised(fri_sales) AS DECIMAL(38,2)) AS fri_sales, CAST(as_noised(sat_sales) AS DECIMAL(38,2)) AS sat_sales FROM asa_ranked ORDER BY s_store_name, s_store_id, sun_sales, mon_sales, tue_sales, wed_sales, thu_sales, fri_sales, sat_sales; diff --git a/benchmark/tpcds/as_queries/q45.sql b/benchmark/tpcds/as_queries/q45.sql deleted file mode 100644 index 1fa8f442..00000000 --- a/benchmark/tpcds/as_queries/q45.sql +++ /dev/null @@ -1,42 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT ca_zip, - ca_city, - as_noised_sum(priv_hash(hash(c_customer_sk)), ws_sales_price) -FROM web_sales, - customer, - customer_address, - date_dim, - item -WHERE ws_bill_customer_sk = c_customer_sk - AND c_current_addr_sk = ca_address_sk - AND ws_item_sk = i_item_sk - AND (SUBSTRING(ca_zip,1,5) IN ('85669', - '86197', - '88274', - '83405', - '86475', - '85392', - '85460', - '80348', - '81792') - OR i_item_id IN - (SELECT i_item_id - FROM item - WHERE i_item_sk IN (2, - 3, - 5, - 7, - 11, - 13, - 17, - 19, - 23, - 29) )) - AND ws_sold_date_sk = d_date_sk - AND d_qoy = 2 - AND d_year = 2001 -GROUP BY ca_zip, - ca_city -ORDER BY ca_zip, - ca_city -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q46.sql b/benchmark/tpcds/as_queries/q46.sql deleted file mode 100644 index 839b21fa..00000000 --- a/benchmark/tpcds/as_queries/q46.sql +++ /dev/null @@ -1,48 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. - -SELECT c_last_name, - c_first_name, - ca_city, - bought_city, - ss_ticket_number, - amt, - profit -FROM - (SELECT ss_ticket_number, - ss_customer_sk, - ca_city bought_city, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) amt, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) profit - FROM store_sales, - date_dim, - store, - household_demographics, - customer_address - WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk - AND store_sales.ss_store_sk = store.s_store_sk - AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - AND store_sales.ss_addr_sk = customer_address.ca_address_sk - AND (household_demographics.hd_dep_count = 4 - OR household_demographics.hd_vehicle_count= 3) - AND date_dim.d_dow IN (6, - 0) - AND date_dim.d_year IN (1999, - 1999+1, - 1999+2) - AND store.s_city IN ('Fairview', - 'Midway') - GROUP BY ss_ticket_number, - ss_customer_sk, - ss_addr_sk, - ca_city) dn, - customer, - customer_address current_addr -WHERE ss_customer_sk = c_customer_sk - AND customer.c_current_addr_sk = current_addr.ca_address_sk - AND current_addr.ca_city <> bought_city -ORDER BY c_last_name NULLS FIRST, - c_first_name NULLS FIRST, - ca_city NULLS FIRST, - bought_city NULLS FIRST, - ss_ticket_number NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q48.sql b/benchmark/tpcds/as_queries/q48.sql deleted file mode 100644 index b2796c71..00000000 --- a/benchmark/tpcds/as_queries/q48.sql +++ /dev/null @@ -1,40 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) -FROM store_sales, - store, - customer_demographics, - customer_address, - date_dim -WHERE s_store_sk = ss_store_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2000 - AND ((cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'M' - AND cd_education_status = '4 yr Degree' - AND ss_sales_price BETWEEN 100.00 AND 150.00) - OR (cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'D' - AND cd_education_status = '2 yr Degree' - AND ss_sales_price BETWEEN 50.00 AND 100.00) - OR (cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'S' - AND cd_education_status = 'College' - AND ss_sales_price BETWEEN 150.00 AND 200.00)) - AND ((ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('CO', - 'OH', - 'TX') - AND ss_net_profit BETWEEN 0 AND 2000) - OR (ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('OR', - 'MN', - 'KY') - AND ss_net_profit BETWEEN 150 AND 3000) - OR (ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('VA', - 'CA', - 'MS') - AND ss_net_profit BETWEEN 50 AND 25000)) ; diff --git a/benchmark/tpcds/as_queries/q50.sql b/benchmark/tpcds/as_queries/q50.sql deleted file mode 100644 index 07fc3b8b..00000000 --- a/benchmark/tpcds/as_queries/q50.sql +++ /dev/null @@ -1,68 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT s_store_name, - s_company_id, - s_street_number, - s_street_name, - s_street_type, - s_suite_number, - s_city, - s_county, - s_state, - s_zip, - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (sr_returned_date_sk - ss_sold_date_sk <= 30) THEN 1 - ELSE 0 - END) AS "30 days", - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (sr_returned_date_sk - ss_sold_date_sk > 30) - AND (sr_returned_date_sk - ss_sold_date_sk <= 60) THEN 1 - ELSE 0 - END) AS "31-60 days", - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (sr_returned_date_sk - ss_sold_date_sk > 60) - AND (sr_returned_date_sk - ss_sold_date_sk <= 90) THEN 1 - ELSE 0 - END) AS "61-90 days", - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (sr_returned_date_sk - ss_sold_date_sk > 90) - AND (sr_returned_date_sk - ss_sold_date_sk <= 120) THEN 1 - ELSE 0 - END) AS "91-120 days", - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (sr_returned_date_sk - ss_sold_date_sk > 120) THEN 1 - ELSE 0 - END) AS ">120 days" -FROM store_sales, - store_returns, - store, - date_dim d1, - date_dim d2 -WHERE d2.d_year = 2001 - AND d2.d_moy = 8 - AND ss_ticket_number = sr_ticket_number - AND ss_item_sk = sr_item_sk - AND ss_sold_date_sk = d1.d_date_sk - AND sr_returned_date_sk = d2.d_date_sk - AND ss_customer_sk = sr_customer_sk - AND ss_store_sk = s_store_sk -GROUP BY s_store_name, - s_company_id, - s_street_number, - s_street_name, - s_street_type, - s_suite_number, - s_city, - s_county, - s_state, - s_zip -ORDER BY s_store_name, - s_company_id, - s_street_number, - s_street_name, - s_street_type, - s_suite_number, - s_city, - s_county, - s_state, - s_zip -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q52.sql b/benchmark/tpcds/as_queries/q52.sql deleted file mode 100644 index adad0f2a..00000000 --- a/benchmark/tpcds/as_queries/q52.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH asa_states AS (SELECT dt.d_year AS d_year, item.i_brand_id AS brand_id, item.i_brand AS brand, as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_sales_price AS DOUBLE)) AS ext_price FROM date_dim dt, - store_sales, - item -WHERE dt.d_date_sk = store_sales.ss_sold_date_sk - AND store_sales.ss_item_sk = item.i_item_sk - AND item.i_manager_id = 1 - AND dt.d_moy=11 - AND dt.d_year=2000 -GROUP BY dt.d_year, - item.i_brand, - item.i_brand_id -), -asa_ranked AS (SELECT * FROM asa_states ORDER BY d_year, priv_mean(ext_price) DESC, brand_id LIMIT 100) -SELECT d_year, brand_id, brand, CAST(as_noised(ext_price) AS DECIMAL(38,2)) AS ext_price FROM asa_ranked ORDER BY d_year, ext_price DESC, brand_id; diff --git a/benchmark/tpcds/as_queries/q55.sql b/benchmark/tpcds/as_queries/q55.sql deleted file mode 100644 index a93b0782..00000000 --- a/benchmark/tpcds/as_queries/q55.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH asa_states AS (SELECT i_brand_id AS brand_id, i_brand AS brand, as_sum(priv_hash(hash(ss_customer_sk)), CAST(ss_ext_sales_price AS DOUBLE)) AS ext_price FROM date_dim, - store_sales, - item -WHERE d_date_sk = ss_sold_date_sk - AND ss_item_sk = i_item_sk - AND i_manager_id=28 - AND d_moy=11 - AND d_year=1999 -GROUP BY i_brand, - i_brand_id -), -asa_ranked AS (SELECT * FROM asa_states ORDER BY priv_mean(ext_price) DESC, brand_id LIMIT 100) -SELECT brand_id, brand, CAST(as_noised(ext_price) AS DECIMAL(38,2)) AS ext_price FROM asa_ranked ORDER BY ext_price DESC, brand_id; diff --git a/benchmark/tpcds/as_queries/q56.sql b/benchmark/tpcds/as_queries/q56.sql deleted file mode 100644 index 7e10f771..00000000 --- a/benchmark/tpcds/as_queries/q56.sql +++ /dev/null @@ -1,71 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH ss AS (SELECT i_item_id, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales - FROM store_sales, - date_dim, - customer_address, - item - WHERE i_item_id IN - (SELECT i_item_id - FROM item - WHERE i_color IN ('slate', - 'blanched', - 'burnished')) - AND ss_item_sk = i_item_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy = 2 - AND ss_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id), -cs AS (SELECT i_item_id, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales - FROM catalog_sales, - date_dim, - customer_address, - item - WHERE i_item_id IN - (SELECT i_item_id - FROM item - WHERE i_color IN ('slate', - 'blanched', - 'burnished')) - AND cs_item_sk = i_item_sk - AND cs_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy = 2 - AND cs_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id), -ws AS (SELECT i_item_id, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales - FROM web_sales, - date_dim, - customer_address, - item - WHERE i_item_id IN - (SELECT i_item_id - FROM item - WHERE i_color IN ('slate', - 'blanched', - 'burnished')) - AND ws_item_sk = i_item_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy = 2 - AND ws_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id) -SELECT i_item_id, - sum(total_sales) total_sales -FROM - (SELECT * - FROM ss - UNION ALL SELECT * - FROM cs - UNION ALL SELECT * - FROM ws) tmp1 -GROUP BY i_item_id -ORDER BY total_sales NULLS FIRST, - i_item_id NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q58.sql b/benchmark/tpcds/as_queries/q58.sql deleted file mode 100644 index cbc64b88..00000000 --- a/benchmark/tpcds/as_queries/q58.sql +++ /dev/null @@ -1,68 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH ss_items AS (SELECT i_item_id item_id, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) ss_item_rev - FROM store_sales, - item, - date_dim - WHERE ss_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM date_dim - WHERE d_week_seq = - (SELECT d_week_seq - FROM date_dim - WHERE d_date = '2000-01-03')) - AND ss_sold_date_sk = d_date_sk - GROUP BY i_item_id), -cs_items AS (SELECT i_item_id item_id, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) cs_item_rev - FROM catalog_sales, - item, - date_dim - WHERE cs_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM date_dim - WHERE d_week_seq = - (SELECT d_week_seq - FROM date_dim - WHERE d_date = '2000-01-03')) - AND cs_sold_date_sk = d_date_sk - GROUP BY i_item_id), -ws_items AS (SELECT i_item_id item_id, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) ws_item_rev - FROM web_sales, - item, - date_dim - WHERE ws_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM date_dim - WHERE d_week_seq = - (SELECT d_week_seq - FROM date_dim - WHERE d_date = '2000-01-03')) - AND ws_sold_date_sk = d_date_sk - GROUP BY i_item_id) -SELECT ss_items.item_id, - ss_item_rev, - ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev, - cs_item_rev, - cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev, - ws_item_rev, - ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev, - (ss_item_rev+cs_item_rev+ws_item_rev)/3 average -FROM ss_items, - cs_items, - ws_items -WHERE ss_items.item_id=cs_items.item_id - AND ss_items.item_id=ws_items.item_id - AND ss_item_rev BETWEEN 0.9 * cs_item_rev AND 1.1 * cs_item_rev - AND ss_item_rev BETWEEN 0.9 * ws_item_rev AND 1.1 * ws_item_rev - AND cs_item_rev BETWEEN 0.9 * ss_item_rev AND 1.1 * ss_item_rev - AND cs_item_rev BETWEEN 0.9 * ws_item_rev AND 1.1 * ws_item_rev - AND ws_item_rev BETWEEN 0.9 * ss_item_rev AND 1.1 * ss_item_rev - AND ws_item_rev BETWEEN 0.9 * cs_item_rev AND 1.1 * cs_item_rev -ORDER BY ss_items.item_id NULLS FIRST, - ss_item_rev NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q59.sql b/benchmark/tpcds/as_queries/q59.sql deleted file mode 100644 index 4dd90e46..00000000 --- a/benchmark/tpcds/as_queries/q59.sql +++ /dev/null @@ -1,85 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH wss AS (SELECT d_week_seq, - ss_store_sk, - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Sunday') THEN ss_sales_price - ELSE NULL - END) sun_sales, - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Monday') THEN ss_sales_price - ELSE NULL - END) mon_sales, - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Tuesday') THEN ss_sales_price - ELSE NULL - END) tue_sales, - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Wednesday') THEN ss_sales_price - ELSE NULL - END) wed_sales, - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Thursday') THEN ss_sales_price - ELSE NULL - END) thu_sales, - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Friday') THEN ss_sales_price - ELSE NULL - END) fri_sales, - as_noised_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Saturday') THEN ss_sales_price - ELSE NULL - END) sat_sales - FROM store_sales, - date_dim - WHERE d_date_sk = ss_sold_date_sk - GROUP BY d_week_seq, - ss_store_sk) -SELECT s_store_name1, - s_store_id1, - d_week_seq1, - sun_sales1/sun_sales2 AS sun_sales_ratio, - mon_sales1/mon_sales2 AS mon_sales_ratio, - tue_sales1/tue_sales2 AS tue_sales_ratio, - wed_sales1/wed_sales2 AS wed_sales_ratio, - thu_sales1/thu_sales2 AS thu_sales_ratio, - fri_sales1/fri_sales2 AS fri_sales_ratio, - sat_sales1/sat_sales2 AS sat_sales_ratio -FROM - (SELECT s_store_name s_store_name1, - wss.d_week_seq d_week_seq1, - s_store_id s_store_id1, - sun_sales sun_sales1, - mon_sales mon_sales1, - tue_sales tue_sales1, - wed_sales wed_sales1, - thu_sales thu_sales1, - fri_sales fri_sales1, - sat_sales sat_sales1 - FROM wss, - store, - date_dim d - WHERE d.d_week_seq = wss.d_week_seq - AND ss_store_sk = s_store_sk - AND d_month_seq BETWEEN 1212 AND 1212 + 11) y, - (SELECT s_store_name s_store_name2, - wss.d_week_seq d_week_seq2, - s_store_id s_store_id2, - sun_sales sun_sales2, - mon_sales mon_sales2, - tue_sales tue_sales2, - wed_sales wed_sales2, - thu_sales thu_sales2, - fri_sales fri_sales2, - sat_sales sat_sales2 - FROM wss, - store, - date_dim d - WHERE d.d_week_seq = wss.d_week_seq - AND ss_store_sk = s_store_sk - AND d_month_seq BETWEEN 1212 + 12 AND 1212 + 23) x -WHERE s_store_id1=s_store_id2 - AND d_week_seq1=d_week_seq2-52 -ORDER BY s_store_name1 NULLS FIRST, - s_store_id1 NULLS FIRST, - d_week_seq1 NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q60.sql b/benchmark/tpcds/as_queries/q60.sql deleted file mode 100644 index e2430099..00000000 --- a/benchmark/tpcds/as_queries/q60.sql +++ /dev/null @@ -1,65 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH ss AS (SELECT i_item_id, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales - FROM store_sales, - date_dim, - customer_address, - item - WHERE i_item_id IN - (SELECT i_item_id - FROM item - WHERE i_category ='Music') - AND ss_item_sk = i_item_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 9 - AND ss_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id), -cs AS (SELECT i_item_id, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales - FROM catalog_sales, - date_dim, - customer_address, - item - WHERE i_item_id IN - (SELECT i_item_id - FROM item - WHERE i_category ='Music') - AND cs_item_sk = i_item_sk - AND cs_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 9 - AND cs_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id), -ws AS (SELECT i_item_id, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales - FROM web_sales, - date_dim, - customer_address, - item - WHERE i_item_id IN - (SELECT i_item_id - FROM item - WHERE i_category = 'Music') - AND ws_item_sk = i_item_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 9 - AND ws_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id) -SELECT i_item_id, - sum(total_sales) total_sales -FROM - (SELECT * - FROM ss - UNION ALL SELECT * - FROM cs - UNION ALL SELECT * - FROM ws) tmp1 -GROUP BY i_item_id -ORDER BY i_item_id, - total_sales -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q61.sql b/benchmark/tpcds/as_queries/q61.sql deleted file mode 100644 index 4f924fc8..00000000 --- a/benchmark/tpcds/as_queries/q61.sql +++ /dev/null @@ -1,44 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Ratio arithmetic is lane-wise; only final outputs are scalarized, matching categorical compilation. -SELECT CAST(as_noised(promotions) AS DECIMAL(38,2)) AS promotions, CAST(as_noised(total) AS DECIMAL(38,2)) AS total, CAST(as_noised(list_transform(list_zip(promotions,total), lambda x: CAST(x[1] AS DECIMAL(15,4)) / CAST(x[2] AS DECIMAL(15,4)) * 100)::FLOAT[]) AS DOUBLE) -FROM (SELECT as_sum(priv_hash(hash(c_customer_sk)), ss_ext_sales_price) promotions - FROM store_sales, - store, - promotion, - date_dim, - customer, - customer_address, - item - WHERE ss_sold_date_sk = d_date_sk - AND ss_store_sk = s_store_sk - AND ss_promo_sk = p_promo_sk - AND ss_customer_sk= c_customer_sk - AND ca_address_sk = c_current_addr_sk - AND ss_item_sk = i_item_sk - AND ca_gmt_offset = -5 - AND i_category = 'Jewelry' - AND (p_channel_dmail = 'Y' - OR p_channel_email = 'Y' - OR p_channel_tv = 'Y') - AND s_gmt_offset = -5 - AND d_year = 1998 - AND d_moy = 11) promotional_sales, (SELECT as_sum(priv_hash(hash(c_customer_sk)), ss_ext_sales_price) total - FROM store_sales, - store, - date_dim, - customer, - customer_address, - item - WHERE ss_sold_date_sk = d_date_sk - AND ss_store_sk = s_store_sk - AND ss_customer_sk= c_customer_sk - AND ca_address_sk = c_current_addr_sk - AND ss_item_sk = i_item_sk - AND ca_gmt_offset = -5 - AND i_category = 'Jewelry' - AND s_gmt_offset = -5 - AND d_year = 1998 - AND d_moy = 11) all_sales -ORDER BY promotions, - total -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q62.sql b/benchmark/tpcds/as_queries/q62.sql deleted file mode 100644 index a95aa874..00000000 --- a/benchmark/tpcds/as_queries/q62.sql +++ /dev/null @@ -1,46 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT w_substr, - sm_type, - web_name, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), CASE - WHEN (ws_ship_date_sk - ws_sold_date_sk <= 30) THEN 1 - ELSE 0 - END) AS "30 days", - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), CASE - WHEN (ws_ship_date_sk - ws_sold_date_sk > 30) - AND (ws_ship_date_sk - ws_sold_date_sk <= 60) THEN 1 - ELSE 0 - END) AS "31-60 days", - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), CASE - WHEN (ws_ship_date_sk - ws_sold_date_sk > 60) - AND (ws_ship_date_sk - ws_sold_date_sk <= 90) THEN 1 - ELSE 0 - END) AS "61-90 days", - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), CASE - WHEN (ws_ship_date_sk - ws_sold_date_sk > 90) - AND (ws_ship_date_sk - ws_sold_date_sk <= 120) THEN 1 - ELSE 0 - END) AS "91-120 days", - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), CASE - WHEN (ws_ship_date_sk - ws_sold_date_sk > 120) THEN 1 - ELSE 0 - END) AS ">120 days" -FROM web_sales, - (SELECT SUBSTRING(w_warehouse_name,1,20) w_substr, - * - FROM warehouse) sq1, - ship_mode, - web_site, - date_dim -WHERE d_month_seq BETWEEN 1200 AND 1200 + 11 - AND ws_ship_date_sk = d_date_sk - AND ws_warehouse_sk = w_warehouse_sk - AND ws_ship_mode_sk = sm_ship_mode_sk - AND ws_web_site_sk = web_site_sk -GROUP BY w_substr, - sm_type, - web_name -ORDER BY 1 NULLS FIRST, - 2 NULLS FIRST, - 3 NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q64.sql b/benchmark/tpcds/as_queries/q64.sql deleted file mode 100644 index bed6a7ea..00000000 --- a/benchmark/tpcds/as_queries/q64.sql +++ /dev/null @@ -1,124 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Each CTE retains scalar aggregates and HAVING; source ownership differs between its two scopes. -WITH cs_ui AS (SELECT cs_item_sk, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_list_price) AS sale, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cr_refunded_cash+cr_reversed_charge+cr_store_credit) AS refund - FROM catalog_sales, - catalog_returns - WHERE cs_item_sk = cr_item_sk - AND cs_order_number = cr_order_number - GROUP BY cs_item_sk - HAVING as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_list_price)>2*as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cr_refunded_cash+cr_reversed_charge+cr_store_credit)), -cross_sales AS (SELECT i_product_name product_name, - i_item_sk item_sk, - s_store_name store_name, - s_zip store_zip, - ad1.ca_street_number b_street_number, - ad1.ca_street_name b_street_name, - ad1.ca_city b_city, - ad1.ca_zip b_zip, - ad2.ca_street_number c_street_number, - ad2.ca_street_name c_street_name, - ad2.ca_city c_city, - ad2.ca_zip c_zip, - d1.d_year AS syear, - d2.d_year AS fsyear, - d3.d_year s2year, - as_noised_count(priv_hash(hash(c_customer_sk)), 1::BIGINT) cnt, - as_noised_sum(priv_hash(hash(c_customer_sk)), ss_wholesale_cost) s1, - as_noised_sum(priv_hash(hash(c_customer_sk)), ss_list_price) s2, - as_noised_sum(priv_hash(hash(c_customer_sk)), ss_coupon_amt) s3 - FROM store_sales, - store_returns, - cs_ui, - date_dim d1, - date_dim d2, - date_dim d3, - store, - customer, - customer_demographics cd1, - customer_demographics cd2, - promotion, - household_demographics hd1, - household_demographics hd2, - customer_address ad1, - customer_address ad2, - income_band ib1, - income_band ib2, - item - WHERE ss_store_sk = s_store_sk - AND ss_sold_date_sk = d1.d_date_sk - AND ss_customer_sk = c_customer_sk - AND ss_cdemo_sk= cd1.cd_demo_sk - AND ss_hdemo_sk = hd1.hd_demo_sk - AND ss_addr_sk = ad1.ca_address_sk - AND ss_item_sk = i_item_sk - AND ss_item_sk = sr_item_sk - AND ss_ticket_number = sr_ticket_number - AND ss_item_sk = cs_ui.cs_item_sk - AND c_current_cdemo_sk = cd2.cd_demo_sk - AND c_current_hdemo_sk = hd2.hd_demo_sk - AND c_current_addr_sk = ad2.ca_address_sk - AND c_first_sales_date_sk = d2.d_date_sk - AND c_first_shipto_date_sk = d3.d_date_sk - AND ss_promo_sk = p_promo_sk - AND hd1.hd_income_band_sk = ib1.ib_income_band_sk - AND hd2.hd_income_band_sk = ib2.ib_income_band_sk - AND cd1.cd_marital_status <> cd2.cd_marital_status - AND i_color IN ('purple', - 'burlywood', - 'indian', - 'spring', - 'floral', - 'medium') - AND i_current_price BETWEEN 64 AND 64 + 10 - AND i_current_price BETWEEN 64 + 1 AND 64 + 15 - GROUP BY i_product_name, - i_item_sk, - s_store_name, - s_zip, - ad1.ca_street_number, - ad1.ca_street_name, - ad1.ca_city, - ad1.ca_zip, - ad2.ca_street_number, - ad2.ca_street_name, - ad2.ca_city, - ad2.ca_zip, - d1.d_year, - d2.d_year, - d3.d_year) -SELECT cs1.product_name, - cs1.store_name, - cs1.store_zip, - cs1.b_street_number, - cs1.b_street_name, - cs1.b_city, - cs1.b_zip, - cs1.c_street_number, - cs1.c_street_name, - cs1.c_city, - cs1.c_zip, - cs1.syear cs1syear, - cs1.cnt cs1cnt, - cs1.s1 AS s11, - cs1.s2 AS s21, - cs1.s3 AS s31, - cs2.s1 AS s12, - cs2.s2 AS s22, - cs2.s3 AS s32, - cs2.syear, - cs2.cnt -FROM cross_sales cs1, - cross_sales cs2 -WHERE cs1.item_sk=cs2.item_sk - AND cs1.syear = 1999 - AND cs2.syear = 1999 + 1 - AND cs2.cnt <= cs1.cnt - AND cs1.store_name = cs2.store_name - AND cs1.store_zip = cs2.store_zip -ORDER BY cs1.product_name, - cs1.store_name, - cs2.cnt, - cs1.s1, - cs2.s1; diff --git a/benchmark/tpcds/as_queries/q65.sql b/benchmark/tpcds/as_queries/q65.sql deleted file mode 100644 index e774e277..00000000 --- a/benchmark/tpcds/as_queries/q65.sql +++ /dev/null @@ -1,23 +0,0 @@ --- Hand transcription of Q65's compiled PAC dataflow, for auditing (priv_rewrite=false). --- This intentionally retains the scalarized inner SUM and majority filter from the compiler. -SELECT s_store_name, i_item_desc, - CAST(as_noised(sc.revenue) AS DECIMAL(38,2)) AS revenue, - i_current_price, i_wholesale_cost, i_brand -FROM main.store, main.item, - (SELECT ss_store_sk, avg(revenue) AS ave - FROM (SELECT ss_store_sk, ss_item_sk, - as_noised_sum(priv_hash(hash(ss_customer_sk)),ss_sales_price) AS revenue - FROM main.store_sales, main.date_dim - WHERE ss_sold_date_sk=d_date_sk AND d_month_seq BETWEEN 1176 AND 1187 - GROUP BY ss_store_sk,ss_item_sk) sa - GROUP BY ss_store_sk) sb, - (SELECT ss_store_sk,ss_item_sk, - as_sum(priv_hash(hash(ss_customer_sk)),ss_sales_price) AS revenue - FROM main.store_sales, main.date_dim - WHERE ss_sold_date_sk=d_date_sk AND d_month_seq BETWEEN 1176 AND 1187 - GROUP BY ss_store_sk,ss_item_sk) sc -WHERE sb.ss_store_sk=sc.ss_store_sk - AND priv_filter_gte(CAST(0.1*sb.ave AS FLOAT),sc.revenue) - AND s_store_sk=sc.ss_store_sk AND i_item_sk=sc.ss_item_sk -ORDER BY s_store_name NULLS FIRST,i_item_desc NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q66.sql b/benchmark/tpcds/as_queries/q66.sql deleted file mode 100644 index 7c9e669f..00000000 --- a/benchmark/tpcds/as_queries/q66.sql +++ /dev/null @@ -1,221 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Only the two leaf UNION branches are stochastic; the outer monthly rollup remains ordinary. -select - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,ship_carriers - ,year_ - ,sum(jan_sales) as jan_sales - ,sum(feb_sales) as feb_sales - ,sum(mar_sales) as mar_sales - ,sum(apr_sales) as apr_sales - ,sum(may_sales) as may_sales - ,sum(jun_sales) as jun_sales - ,sum(jul_sales) as jul_sales - ,sum(aug_sales) as aug_sales - ,sum(sep_sales) as sep_sales - ,sum(oct_sales) as oct_sales - ,sum(nov_sales) as nov_sales - ,sum(dec_sales) as dec_sales - ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot - ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot - ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot - ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot - ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot - ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot - ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot - ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot - ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot - ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot - ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot - ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot - ,sum(jan_net) as jan_net - ,sum(feb_net) as feb_net - ,sum(mar_net) as mar_net - ,sum(apr_net) as apr_net - ,sum(may_net) as may_net - ,sum(jun_net) as jun_net - ,sum(jul_net) as jul_net - ,sum(aug_net) as aug_net - ,sum(sep_net) as sep_net - ,sum(oct_net) as oct_net - ,sum(nov_net) as nov_net - ,sum(dec_net) as dec_net - from ( - select - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,'DHL,BARIAN' as ship_carriers - ,d_year as year_ - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 1 - then ws_ext_sales_price* ws_quantity else 0 end) as jan_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 2 - then ws_ext_sales_price* ws_quantity else 0 end) as feb_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 3 - then ws_ext_sales_price* ws_quantity else 0 end) as mar_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 4 - then ws_ext_sales_price* ws_quantity else 0 end) as apr_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 5 - then ws_ext_sales_price* ws_quantity else 0 end) as may_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 6 - then ws_ext_sales_price* ws_quantity else 0 end) as jun_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 7 - then ws_ext_sales_price* ws_quantity else 0 end) as jul_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 8 - then ws_ext_sales_price* ws_quantity else 0 end) as aug_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 9 - then ws_ext_sales_price* ws_quantity else 0 end) as sep_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 10 - then ws_ext_sales_price* ws_quantity else 0 end) as oct_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 11 - then ws_ext_sales_price* ws_quantity else 0 end) as nov_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 12 - then ws_ext_sales_price* ws_quantity else 0 end) as dec_sales - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 1 - then ws_net_paid * ws_quantity else 0 end) as jan_net - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 2 - then ws_net_paid * ws_quantity else 0 end) as feb_net - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 3 - then ws_net_paid * ws_quantity else 0 end) as mar_net - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 4 - then ws_net_paid * ws_quantity else 0 end) as apr_net - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 5 - then ws_net_paid * ws_quantity else 0 end) as may_net - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 6 - then ws_net_paid * ws_quantity else 0 end) as jun_net - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 7 - then ws_net_paid * ws_quantity else 0 end) as jul_net - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 8 - then ws_net_paid * ws_quantity else 0 end) as aug_net - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 9 - then ws_net_paid * ws_quantity else 0 end) as sep_net - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 10 - then ws_net_paid * ws_quantity else 0 end) as oct_net - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 11 - then ws_net_paid * ws_quantity else 0 end) as nov_net - ,as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 12 - then ws_net_paid * ws_quantity else 0 end) as dec_net - from - web_sales - ,warehouse - ,date_dim - ,time_dim - ,ship_mode - where - ws_warehouse_sk = w_warehouse_sk - and ws_sold_date_sk = d_date_sk - and ws_sold_time_sk = t_time_sk - and ws_ship_mode_sk = sm_ship_mode_sk - and d_year = 2001 - and t_time between 30838 and 30838+28800 - and sm_carrier in ('DHL','BARIAN') - group by - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,d_year - -UNION ALL - - select - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,'DHL,BARIAN' as ship_carriers - ,d_year as year_ - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 1 - then cs_sales_price* cs_quantity else 0 end) as jan_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 2 - then cs_sales_price* cs_quantity else 0 end) as feb_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 3 - then cs_sales_price* cs_quantity else 0 end) as mar_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 4 - then cs_sales_price* cs_quantity else 0 end) as apr_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 5 - then cs_sales_price* cs_quantity else 0 end) as may_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 6 - then cs_sales_price* cs_quantity else 0 end) as jun_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 7 - then cs_sales_price* cs_quantity else 0 end) as jul_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 8 - then cs_sales_price* cs_quantity else 0 end) as aug_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 9 - then cs_sales_price* cs_quantity else 0 end) as sep_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 10 - then cs_sales_price* cs_quantity else 0 end) as oct_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 11 - then cs_sales_price* cs_quantity else 0 end) as nov_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 12 - then cs_sales_price* cs_quantity else 0 end) as dec_sales - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 1 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as jan_net - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 2 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as feb_net - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 3 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as mar_net - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 4 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as apr_net - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 5 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as may_net - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 6 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as jun_net - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 7 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as jul_net - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 8 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as aug_net - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 9 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as sep_net - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 10 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as oct_net - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 11 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as nov_net - ,as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 12 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as dec_net - from - catalog_sales - ,warehouse - ,date_dim - ,time_dim - ,ship_mode - where - cs_warehouse_sk = w_warehouse_sk - and cs_sold_date_sk = d_date_sk - and cs_sold_time_sk = t_time_sk - and cs_ship_mode_sk = sm_ship_mode_sk - and d_year = 2001 - and t_time between 30838 AND 30838+28800 - and sm_carrier in ('DHL','BARIAN') - group by - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,d_year - ) x - group by - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,ship_carriers - ,year_ - order by w_warehouse_name NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q68.sql b/benchmark/tpcds/as_queries/q68.sql deleted file mode 100644 index 7cf7f5c7..00000000 --- a/benchmark/tpcds/as_queries/q68.sql +++ /dev/null @@ -1,45 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -SELECT c_last_name, - c_first_name, - ca_city, - bought_city, - ss_ticket_number, - extended_price, - extended_tax, - list_price -FROM - (SELECT ss_ticket_number, - ss_customer_sk, - ca_city bought_city, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) extended_price, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_list_price) list_price, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_tax) extended_tax - FROM store_sales, - date_dim, - store, - household_demographics, - customer_address - WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk - AND store_sales.ss_store_sk = store.s_store_sk - AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - AND store_sales.ss_addr_sk = customer_address.ca_address_sk - AND date_dim.d_dom BETWEEN 1 AND 2 - AND (household_demographics.hd_dep_count = 4 - OR household_demographics.hd_vehicle_count= 3) - AND date_dim.d_year IN (1999, - 1999+1, - 1999+2) - AND store.s_city IN ('Fairview', - 'Midway') - GROUP BY ss_ticket_number, - ss_customer_sk, - ss_addr_sk, - ca_city) dn, - customer, - customer_address current_addr -WHERE ss_customer_sk = c_customer_sk - AND customer.c_current_addr_sk = current_addr.ca_address_sk - AND current_addr.ca_city <> bought_city -ORDER BY c_last_name NULLS FIRST, - ss_ticket_number NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q69.sql b/benchmark/tpcds/as_queries/q69.sql deleted file mode 100644 index 58d35272..00000000 --- a/benchmark/tpcds/as_queries/q69.sql +++ /dev/null @@ -1,52 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -SELECT cd_gender, - cd_marital_status, - cd_education_status, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt1, - cd_purchase_estimate, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt2, - cd_credit_rating, - as_noised_count(priv_hash(hash(c.c_customer_sk)), 1::BIGINT) cnt3 -FROM customer c, - customer_address ca, - customer_demographics -WHERE c.c_current_addr_sk = ca.ca_address_sk - AND ca_state IN ('KY', - 'GA', - 'NM') - AND cd_demo_sk = c.c_current_cdemo_sk - AND EXISTS - (SELECT * - FROM store_sales, - date_dim - WHERE c.c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy BETWEEN 4 AND 4+2) - AND (NOT EXISTS - (SELECT * - FROM web_sales, - date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy BETWEEN 4 AND 4+2) - AND NOT EXISTS - (SELECT * - FROM catalog_sales, - date_dim - WHERE c.c_customer_sk = cs_ship_customer_sk - AND cs_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy BETWEEN 4 AND 4+2)) -GROUP BY cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating -ORDER BY cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q72.sql b/benchmark/tpcds/as_queries/q72.sql deleted file mode 100644 index fb593e23..00000000 --- a/benchmark/tpcds/as_queries/q72.sql +++ /dev/null @@ -1,40 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- One catalog-sales mask drives the states after the original joins, including LEFT JOINs. -SELECT i_item_desc, - w_warehouse_name, - d1.d_week_seq, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE - WHEN p_promo_sk IS NULL THEN 1 - ELSE 0 - END) no_promo, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE - WHEN p_promo_sk IS NOT NULL THEN 1 - ELSE 0 - END) promo, - as_noised_count(priv_hash(hash(cs_bill_customer_sk)), 1::BIGINT) total_cnt -FROM catalog_sales -JOIN inventory ON (cs_item_sk = inv_item_sk) -JOIN warehouse ON (w_warehouse_sk=inv_warehouse_sk) -JOIN item ON (i_item_sk = cs_item_sk) -JOIN customer_demographics ON (cs_bill_cdemo_sk = cd_demo_sk) -JOIN household_demographics ON (cs_bill_hdemo_sk = hd_demo_sk) -JOIN date_dim d1 ON (cs_sold_date_sk = d1.d_date_sk) -JOIN date_dim d2 ON (inv_date_sk = d2.d_date_sk) -JOIN date_dim d3 ON (cs_ship_date_sk = d3.d_date_sk) -LEFT OUTER JOIN promotion ON (cs_promo_sk=p_promo_sk) -LEFT OUTER JOIN catalog_returns ON (cr_item_sk = cs_item_sk - AND cr_order_number = cs_order_number) -WHERE d1.d_week_seq = d2.d_week_seq - AND inv_quantity_on_hand < cs_quantity - AND d3.d_date > d1.d_date + 5 -- SQL Server: DATEADD(day, 5, d1.d_date) - AND hd_buy_potential = '>10000' - AND d1.d_year = 1999 - AND cd_marital_status = 'D' -GROUP BY i_item_desc, - w_warehouse_name, - d1.d_week_seq -ORDER BY total_cnt DESC NULLS FIRST, - i_item_desc NULLS FIRST, - w_warehouse_name NULLS FIRST, - d1.d_week_seq NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q73.sql b/benchmark/tpcds/as_queries/q73.sql deleted file mode 100644 index c6eb962c..00000000 --- a/benchmark/tpcds/as_queries/q73.sql +++ /dev/null @@ -1,42 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Terminal range comparisons are separate majority decisions; output cnt is finalized afterwards. - -SELECT c_last_name, - c_first_name, - c_salutation, - c_preferred_cust_flag, - ss_ticket_number, - CAST(as_noised(cnt) AS BIGINT) AS cnt -FROM - (SELECT ss_ticket_number, - ss_customer_sk, - as_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) cnt - FROM store_sales, - date_dim, - store, - household_demographics - WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk - AND store_sales.ss_store_sk = store.s_store_sk - AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - AND date_dim.d_dom BETWEEN 1 AND 2 - AND (household_demographics.hd_buy_potential = 'Unknown' - OR household_demographics.hd_buy_potential = '>10000') - AND household_demographics.hd_vehicle_count > 0 - AND CASE - WHEN household_demographics.hd_vehicle_count > 0 THEN (household_demographics.hd_dep_count*1.000)/ household_demographics.hd_vehicle_count - ELSE NULL - END > 1 - AND date_dim.d_year IN (1999, - 1999+1, - 1999+2) - AND store.s_county IN ('Orange County', - 'Bronx County', - 'Franklin Parish', - 'Williamson County') - GROUP BY ss_ticket_number, - ss_customer_sk) dj, - customer -WHERE ss_customer_sk = c_customer_sk - AND priv_filter_lte(1::FLOAT, cnt) AND priv_filter_gte(5::FLOAT, cnt) -ORDER BY cnt DESC, - c_last_name ASC; diff --git a/benchmark/tpcds/as_queries/q74.sql b/benchmark/tpcds/as_queries/q74.sql deleted file mode 100644 index 1a7ee67d..00000000 --- a/benchmark/tpcds/as_queries/q74.sql +++ /dev/null @@ -1,64 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH year_total AS (SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - d_year AS year_, - as_noised_sum(priv_hash(hash(c_customer_sk)), ss_net_paid) year_total, - 's' sale_type - FROM customer, - store_sales, - date_dim - WHERE c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - AND d_year IN (2001, - 2001+1) - GROUP BY c_customer_id, - c_first_name, - c_last_name, - d_year - UNION ALL SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - d_year AS year_, - as_noised_sum(priv_hash(hash(c_customer_sk)), ws_net_paid) year_total, - 'w' sale_type - FROM customer, - web_sales, - date_dim - WHERE c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year IN (2001, - 2001+1) - GROUP BY c_customer_id, - c_first_name, - c_last_name, - d_year) -SELECT t_s_secyear.customer_id, - t_s_secyear.customer_first_name, - t_s_secyear.customer_last_name -FROM year_total t_s_firstyear, - year_total t_s_secyear, - year_total t_w_firstyear, - year_total t_w_secyear -WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id - AND t_s_firstyear.customer_id = t_w_secyear.customer_id - AND t_s_firstyear.customer_id = t_w_firstyear.customer_id - AND t_s_firstyear.sale_type = 's' - AND t_w_firstyear.sale_type = 'w' - AND t_s_secyear.sale_type = 's' - AND t_w_secyear.sale_type = 'w' - AND t_s_firstyear.year_ = 2001 - AND t_s_secyear.year_ = 2001+1 - AND t_w_firstyear.year_ = 2001 - AND t_w_secyear.year_ = 2001+1 - AND t_s_firstyear.year_total > 0 - AND t_w_firstyear.year_total > 0 - AND CASE - WHEN t_w_firstyear.year_total > 0 THEN t_w_secyear.year_total / t_w_firstyear.year_total - ELSE NULL - END > CASE - WHEN t_s_firstyear.year_total > 0 THEN t_s_secyear.year_total / t_s_firstyear.year_total - ELSE NULL - END -ORDER BY 1 NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q77.sql b/benchmark/tpcds/as_queries/q77.sql deleted file mode 100644 index c29146cd..00000000 --- a/benchmark/tpcds/as_queries/q77.sql +++ /dev/null @@ -1,91 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Keep all six original branches, including the original catalog Cartesian product. -WITH ss AS (SELECT s_store_sk, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS sales, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) AS profit - FROM store_sales, - date_dim, - store - WHERE ss_sold_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - AND ss_store_sk = s_store_sk - GROUP BY s_store_sk), -sr AS (SELECT s_store_sk, - as_noised_sum(priv_hash(hash(sr_customer_sk)), sr_return_amt) AS returns_, - as_noised_sum(priv_hash(hash(sr_customer_sk)), sr_net_loss) AS profit_loss - FROM store_returns, - date_dim, - store - WHERE sr_returned_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - AND sr_store_sk = s_store_sk - GROUP BY s_store_sk), -cs AS (SELECT cs_call_center_sk, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) AS sales, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_net_profit) AS profit - FROM catalog_sales, - date_dim - WHERE cs_sold_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - GROUP BY cs_call_center_sk), -cr AS (SELECT cr_call_center_sk, - as_noised_sum(priv_hash(hash(cr_returning_customer_sk)), cr_return_amount) AS returns_, - as_noised_sum(priv_hash(hash(cr_returning_customer_sk)), cr_net_loss) AS profit_loss - FROM catalog_returns, - date_dim - WHERE cr_returned_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - GROUP BY cr_call_center_sk ), -ws AS (SELECT wp_web_page_sk, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) AS sales, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_net_profit) AS profit - FROM web_sales, - date_dim, - web_page - WHERE ws_sold_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - AND ws_web_page_sk = wp_web_page_sk - GROUP BY wp_web_page_sk), -wr AS (SELECT wp_web_page_sk, - as_noised_sum(priv_hash(hash(wr_returning_customer_sk)), wr_return_amt) AS returns_, - as_noised_sum(priv_hash(hash(wr_returning_customer_sk)), wr_net_loss) AS profit_loss - FROM web_returns, - date_dim, - web_page - WHERE wr_returned_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - AND wr_web_page_sk = wp_web_page_sk - GROUP BY wp_web_page_sk) -SELECT channel , - id , - sum(sales) AS sales , - sum(returns_) AS returns_ , - sum(profit) AS profit -FROM - (SELECT 'store channel' AS channel , - ss.s_store_sk AS id , - sales , - coalesce(returns_, 0) AS returns_ , - (profit - coalesce(profit_loss,0)) AS profit - FROM ss - LEFT JOIN sr ON ss.s_store_sk = sr.s_store_sk - UNION ALL SELECT 'catalog channel' AS channel , - cs_call_center_sk AS id , - sales , - returns_ , - (profit - profit_loss) AS profit - FROM cs , - cr - UNION ALL SELECT 'web channel' AS channel , - ws.wp_web_page_sk AS id , - sales , - coalesce(returns_, 0) returns_ , - (profit - coalesce(profit_loss,0)) AS profit - FROM ws - LEFT JOIN wr ON ws.wp_web_page_sk = wr.wp_web_page_sk ) x -GROUP BY ROLLUP (channel, - id) -ORDER BY channel NULLS FIRST, - id NULLS FIRST, - returns_ DESC -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q78.sql b/benchmark/tpcds/as_queries/q78.sql deleted file mode 100644 index d91eeae1..00000000 --- a/benchmark/tpcds/as_queries/q78.sql +++ /dev/null @@ -1,75 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Preserve original sales/returns outer joins; ownership comes from each sales branch. -WITH ws AS (SELECT d_year AS ws_sold_year, - ws_item_sk, - ws_bill_customer_sk ws_customer_sk, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_quantity) ws_qty, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_wholesale_cost) ws_wc, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_sales_price) ws_sp - FROM web_sales - LEFT JOIN web_returns ON wr_order_number=ws_order_number - AND ws_item_sk=wr_item_sk - JOIN date_dim ON ws_sold_date_sk = d_date_sk - WHERE wr_order_number IS NULL - GROUP BY d_year, - ws_item_sk, - ws_bill_customer_sk ), -cs AS (SELECT d_year AS cs_sold_year, - cs_item_sk, - cs_bill_customer_sk cs_customer_sk, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_quantity) cs_qty, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_wholesale_cost) cs_wc, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_sales_price) cs_sp - FROM catalog_sales - LEFT JOIN catalog_returns ON cr_order_number=cs_order_number - AND cs_item_sk=cr_item_sk - JOIN date_dim ON cs_sold_date_sk = d_date_sk - WHERE cr_order_number IS NULL - GROUP BY d_year, - cs_item_sk, - cs_bill_customer_sk ), -ss AS (SELECT d_year AS ss_sold_year, - ss_item_sk, - ss_customer_sk, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_quantity) ss_qty, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_wholesale_cost) ss_wc, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_sales_price) ss_sp - FROM store_sales - LEFT JOIN store_returns ON sr_ticket_number=ss_ticket_number - AND ss_item_sk=sr_item_sk - JOIN date_dim ON ss_sold_date_sk = d_date_sk - WHERE sr_ticket_number IS NULL - GROUP BY d_year, - ss_item_sk, - ss_customer_sk ) -SELECT ss_sold_year, - ss_item_sk, - ss_customer_sk, - round((ss_qty*1.00)/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, - ss_qty store_qty, - ss_wc store_wholesale_cost, - ss_sp store_sales_price, - coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, - coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, - coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price -FROM ss -LEFT JOIN ws ON (ws_sold_year=ss_sold_year - AND ws_item_sk=ss_item_sk - AND ws_customer_sk=ss_customer_sk) -LEFT JOIN cs ON (cs_sold_year=ss_sold_year - AND cs_item_sk=ss_item_sk - AND cs_customer_sk=ss_customer_sk) -WHERE (coalesce(ws_qty,0)>0 - OR coalesce(cs_qty, 0)>0) - AND ss_sold_year=2000 -ORDER BY ss_sold_year, - ss_item_sk, - ss_customer_sk, - ss_qty DESC, - ss_wc DESC, - ss_sp DESC, - other_chan_qty, - other_chan_wholesale_cost, - other_chan_sales_price, - ratio -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q79.sql b/benchmark/tpcds/as_queries/q79.sql deleted file mode 100644 index 670f5494..00000000 --- a/benchmark/tpcds/as_queries/q79.sql +++ /dev/null @@ -1,39 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -SELECT c_last_name, - c_first_name, - SUBSTRING(s_city,1,30), - ss_ticket_number, - amt, - profit -FROM - (SELECT ss_ticket_number , - ss_customer_sk , - store.s_city , - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) amt , - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) profit - FROM store_sales, - date_dim, - store, - household_demographics - WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk - AND store_sales.ss_store_sk = store.s_store_sk - AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - AND (household_demographics.hd_dep_count = 6 - OR household_demographics.hd_vehicle_count > 2) - AND date_dim.d_dow = 1 - AND date_dim.d_year IN (1999, - 1999+1, - 1999+2) - AND store.s_number_employees BETWEEN 200 AND 295 - GROUP BY ss_ticket_number, - ss_customer_sk, - ss_addr_sk, - store.s_city) ms, - customer -WHERE ss_customer_sk = c_customer_sk -ORDER BY c_last_name NULLS FIRST, - c_first_name NULLS FIRST, - SUBSTRING(s_city,1,30) NULLS FIRST, - profit NULLS FIRST, - ss_ticket_number -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q80.sql b/benchmark/tpcds/as_queries/q80.sql deleted file mode 100644 index ae118341..00000000 --- a/benchmark/tpcds/as_queries/q80.sql +++ /dev/null @@ -1,85 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Stochastic sales/returns expressions share the sales-side owner; outer ROLLUP stays scalar. -WITH ssr AS (SELECT s_store_id AS store_id, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS sales, - as_noised_sum(priv_hash(hash(ss_customer_sk)), coalesce(sr_return_amt, 0)) AS returns_, - as_noised_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit - coalesce(sr_net_loss, 0)) AS profit - FROM store_sales - LEFT OUTER JOIN store_returns ON (ss_item_sk = sr_item_sk - AND ss_ticket_number = sr_ticket_number), date_dim, - store, - item, - promotion - WHERE ss_sold_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - AND ss_store_sk = s_store_sk - AND ss_item_sk = i_item_sk - AND i_current_price > 50 - AND ss_promo_sk = p_promo_sk - AND p_channel_tv = 'N' - GROUP BY s_store_id), -csr AS (SELECT cp_catalog_page_id AS catalog_page_id, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) AS sales, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), coalesce(cr_return_amount, 0)) AS returns_, - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), cs_net_profit - coalesce(cr_net_loss, 0)) AS profit - FROM catalog_sales - LEFT OUTER JOIN catalog_returns ON (cs_item_sk = cr_item_sk - AND cs_order_number = cr_order_number), date_dim, - catalog_page, - item, - promotion - WHERE cs_sold_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - AND cs_catalog_page_sk = cp_catalog_page_sk - AND cs_item_sk = i_item_sk - AND i_current_price > 50 - AND cs_promo_sk = p_promo_sk - AND p_channel_tv = 'N' - GROUP BY cp_catalog_page_id), -wsr AS (SELECT web_site_id, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) AS sales, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), coalesce(wr_return_amt, 0)) AS returns_, - as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_net_profit - coalesce(wr_net_loss, 0)) AS profit - FROM web_sales - LEFT OUTER JOIN web_returns ON (ws_item_sk = wr_item_sk - AND ws_order_number = wr_order_number), date_dim, - web_site, - item, - promotion - WHERE ws_sold_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - AND ws_web_site_sk = web_site_sk - AND ws_item_sk = i_item_sk - AND i_current_price > 50 - AND ws_promo_sk = p_promo_sk - AND p_channel_tv = 'N' - GROUP BY web_site_id) -SELECT channel , - id , - sum(sales) AS sales , - sum(returns_) AS returns_ , - sum(profit) AS profit -FROM - (SELECT 'store channel' AS channel , - concat('store', store_id) AS id , - sales , - returns_ , - profit - FROM ssr - UNION ALL SELECT 'catalog channel' AS channel , - concat('catalog_page', catalog_page_id) AS id , - sales , - returns_ , - profit - FROM csr - UNION ALL SELECT 'web channel' AS channel , - concat('web_site', web_site_id) AS id , - sales , - returns_ , - profit - FROM wsr ) x -GROUP BY ROLLUP (channel, - id) -ORDER BY channel NULLS FIRST, - id NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q81.sql b/benchmark/tpcds/as_queries/q81.sql deleted file mode 100644 index 7549b3f6..00000000 --- a/benchmark/tpcds/as_queries/q81.sql +++ /dev/null @@ -1,55 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Per-customer CTE totals stay scalar; only the correlated cross-customer AVG supplies a counter list. -WITH customer_total_return AS (SELECT cr_returning_customer_sk AS ctr_customer_sk , - ca_state AS ctr_state, - sum(cr_return_amt_inc_tax) AS ctr_total_return - FROM catalog_returns , - date_dim , - customer_address - WHERE cr_returned_date_sk = d_date_sk - AND d_year = 2000 - AND cr_returning_addr_sk = ca_address_sk - GROUP BY cr_returning_customer_sk , - ca_state) -SELECT c_customer_id, - c_salutation, - c_first_name, - c_last_name, - ca_street_number, - ca_street_name , - ca_street_type, - ca_suite_number, - ca_city, - ca_county, - ca_state, - ca_zip, - ca_country, - ca_gmt_offset , - ca_location_type, - ctr_total_return -FROM customer_total_return ctr1 , - customer_address , - customer -WHERE priv_filter_gt(CAST(CAST(ctr1.ctr_total_return AS DOUBLE) * 0.8333333333333334 AS FLOAT), (SELECT priv_div(as_sum(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return), as_count(priv_hash(hash(ctr2.ctr_customer_sk)), ctr_total_return)) - FROM customer_total_return ctr2 - WHERE ctr1.ctr_state = ctr2.ctr_state)) - AND ca_address_sk = c_current_addr_sk - AND ca_state = 'GA' - AND ctr1.ctr_customer_sk = c_customer_sk -ORDER BY c_customer_id, - c_salutation, - c_first_name, - c_last_name, - ca_street_number, - ca_street_name , - ca_street_type, - ca_suite_number, - ca_city, - ca_county, - ca_state, - ca_zip, - ca_country, - ca_gmt_offset , - ca_location_type, - ctr_total_return -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q83.sql b/benchmark/tpcds/as_queries/q83.sql deleted file mode 100644 index 9f365214..00000000 --- a/benchmark/tpcds/as_queries/q83.sql +++ /dev/null @@ -1,68 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -WITH sr_items AS (SELECT i_item_id item_id, - as_noised_sum(priv_hash(hash(sr_customer_sk)), sr_return_quantity) sr_item_qty - FROM store_returns, - item, - date_dim - WHERE sr_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM date_dim - WHERE d_week_seq IN - (SELECT d_week_seq - FROM date_dim - WHERE d_date IN ('2000-06-30', - '2000-09-27', - '2000-11-17'))) - AND sr_returned_date_sk = d_date_sk - GROUP BY i_item_id), -cr_items AS (SELECT i_item_id item_id, - as_noised_sum(priv_hash(hash(cr_returning_customer_sk)), cr_return_quantity) cr_item_qty - FROM catalog_returns, - item, - date_dim - WHERE cr_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM date_dim - WHERE d_week_seq IN - (SELECT d_week_seq - FROM date_dim - WHERE d_date IN ('2000-06-30', - '2000-09-27', - '2000-11-17'))) - AND cr_returned_date_sk = d_date_sk - GROUP BY i_item_id), -wr_items AS (SELECT i_item_id item_id, - as_noised_sum(priv_hash(hash(wr_returning_customer_sk)), wr_return_quantity) wr_item_qty - FROM web_returns, - item, - date_dim - WHERE wr_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM date_dim - WHERE d_week_seq IN - (SELECT d_week_seq - FROM date_dim - WHERE d_date IN ('2000-06-30', - '2000-09-27', - '2000-11-17'))) - AND wr_returned_date_sk = d_date_sk - GROUP BY i_item_id) -SELECT sr_items.item_id , - sr_item_qty , - (sr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 sr_dev , - cr_item_qty , - (cr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 cr_dev , - wr_item_qty , - (wr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 wr_dev , - (sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average -FROM sr_items , - cr_items , - wr_items -WHERE sr_items.item_id=cr_items.item_id - AND sr_items.item_id=wr_items.item_id -ORDER BY sr_items.item_id NULLS FIRST, - sr_item_qty NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q85.sql b/benchmark/tpcds/as_queries/q85.sql deleted file mode 100644 index 47ba8f73..00000000 --- a/benchmark/tpcds/as_queries/q85.sql +++ /dev/null @@ -1,60 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. --- The compiler uses one accessible web-sales customer mask, not a second-mask intersection. - -SELECT SUBSTRING(r_reason_desc,1,20) , - (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), ws_quantity) AS DOUBLE)) avg1, - (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), wr_refunded_cash) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), wr_refunded_cash) AS DOUBLE)) avg2, - (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), wr_fee) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), wr_fee) AS DOUBLE)) -FROM web_sales, - web_returns, - web_page, - customer_demographics cd1, - customer_demographics cd2, - customer_address, - date_dim, - reason -WHERE ws_web_page_sk = wp_web_page_sk - AND ws_item_sk = wr_item_sk - AND ws_order_number = wr_order_number - AND ws_sold_date_sk = d_date_sk - AND d_year = 2000 - AND cd1.cd_demo_sk = wr_refunded_cdemo_sk - AND cd2.cd_demo_sk = wr_returning_cdemo_sk - AND ca_address_sk = wr_refunded_addr_sk - AND r_reason_sk = wr_reason_sk - AND ( ( cd1.cd_marital_status = 'M' - AND cd1.cd_marital_status = cd2.cd_marital_status - AND cd1.cd_education_status = 'Advanced Degree' - AND cd1.cd_education_status = cd2.cd_education_status - AND ws_sales_price BETWEEN 100.00 AND 150.00 ) - OR ( cd1.cd_marital_status = 'S' - AND cd1.cd_marital_status = cd2.cd_marital_status - AND cd1.cd_education_status = 'College' - AND cd1.cd_education_status = cd2.cd_education_status - AND ws_sales_price BETWEEN 50.00 AND 100.00 ) - OR ( cd1.cd_marital_status = 'W' - AND cd1.cd_marital_status = cd2.cd_marital_status - AND cd1.cd_education_status = '2 yr Degree' - AND cd1.cd_education_status = cd2.cd_education_status - AND ws_sales_price BETWEEN 150.00 AND 200.00 ) ) - AND ( ( ca_country = 'United States' - AND ca_state IN ('IN', - 'OH', - 'NJ') - AND ws_net_profit BETWEEN 100 AND 200) - OR ( ca_country = 'United States' - AND ca_state IN ('WI', - 'CT', - 'KY') - AND ws_net_profit BETWEEN 150 AND 300) - OR ( ca_country = 'United States' - AND ca_state IN ('LA', - 'IA', - 'AR') - AND ws_net_profit BETWEEN 50 AND 250) ) -GROUP BY r_reason_desc -ORDER BY SUBSTRING(r_reason_desc,1,20) , - (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), ws_quantity) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), ws_quantity) AS DOUBLE)) , - (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), wr_refunded_cash) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), wr_refunded_cash) AS DOUBLE)) , - (CAST(as_noised_sum(priv_hash(hash(ws_bill_customer_sk)), wr_fee) AS DOUBLE) / CAST(as_noised_count(priv_hash(hash(ws_bill_customer_sk)), wr_fee) AS DOUBLE)) -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q88.sql b/benchmark/tpcds/as_queries/q88.sql deleted file mode 100644 index 5d178052..00000000 --- a/benchmark/tpcds/as_queries/q88.sql +++ /dev/null @@ -1,139 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -SELECT * -FROM - (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h8_30_to_9 - FROM store_sales, - household_demographics, - time_dim, - store - WHERE ss_sold_time_sk = time_dim.t_time_sk - AND ss_hdemo_sk = household_demographics.hd_demo_sk - AND ss_store_sk = s_store_sk - AND time_dim.t_hour = 8 - AND time_dim.t_minute >= 30 - AND ((household_demographics.hd_dep_count = 4 - AND household_demographics.hd_vehicle_count<=4+2) - OR (household_demographics.hd_dep_count = 2 - AND household_demographics.hd_vehicle_count<=2+2) - OR (household_demographics.hd_dep_count = 0 - AND household_demographics.hd_vehicle_count<=0+2)) - AND store.s_store_name = 'ese') s1, - (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h9_to_9_30 - FROM store_sales, - household_demographics, - time_dim, - store - WHERE ss_sold_time_sk = time_dim.t_time_sk - AND ss_hdemo_sk = household_demographics.hd_demo_sk - AND ss_store_sk = s_store_sk - AND time_dim.t_hour = 9 - AND time_dim.t_minute < 30 - AND ((household_demographics.hd_dep_count = 4 - AND household_demographics.hd_vehicle_count<=4+2) - OR (household_demographics.hd_dep_count = 2 - AND household_demographics.hd_vehicle_count<=2+2) - OR (household_demographics.hd_dep_count = 0 - AND household_demographics.hd_vehicle_count<=0+2)) - AND store.s_store_name = 'ese') s2, - (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h9_30_to_10 - FROM store_sales, - household_demographics, - time_dim, - store - WHERE ss_sold_time_sk = time_dim.t_time_sk - AND ss_hdemo_sk = household_demographics.hd_demo_sk - AND ss_store_sk = s_store_sk - AND time_dim.t_hour = 9 - AND time_dim.t_minute >= 30 - AND ((household_demographics.hd_dep_count = 4 - AND household_demographics.hd_vehicle_count<=4+2) - OR (household_demographics.hd_dep_count = 2 - AND household_demographics.hd_vehicle_count<=2+2) - OR (household_demographics.hd_dep_count = 0 - AND household_demographics.hd_vehicle_count<=0+2)) - AND store.s_store_name = 'ese') s3, - (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h10_to_10_30 - FROM store_sales, - household_demographics, - time_dim, - store - WHERE ss_sold_time_sk = time_dim.t_time_sk - AND ss_hdemo_sk = household_demographics.hd_demo_sk - AND ss_store_sk = s_store_sk - AND time_dim.t_hour = 10 - AND time_dim.t_minute < 30 - AND ((household_demographics.hd_dep_count = 4 - AND household_demographics.hd_vehicle_count<=4+2) - OR (household_demographics.hd_dep_count = 2 - AND household_demographics.hd_vehicle_count<=2+2) - OR (household_demographics.hd_dep_count = 0 - AND household_demographics.hd_vehicle_count<=0+2)) - AND store.s_store_name = 'ese') s4, - (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h10_30_to_11 - FROM store_sales, - household_demographics, - time_dim, - store - WHERE ss_sold_time_sk = time_dim.t_time_sk - AND ss_hdemo_sk = household_demographics.hd_demo_sk - AND ss_store_sk = s_store_sk - AND time_dim.t_hour = 10 - AND time_dim.t_minute >= 30 - AND ((household_demographics.hd_dep_count = 4 - AND household_demographics.hd_vehicle_count<=4+2) - OR (household_demographics.hd_dep_count = 2 - AND household_demographics.hd_vehicle_count<=2+2) - OR (household_demographics.hd_dep_count = 0 - AND household_demographics.hd_vehicle_count<=0+2)) - AND store.s_store_name = 'ese') s5, - (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h11_to_11_30 - FROM store_sales, - household_demographics, - time_dim, - store - WHERE ss_sold_time_sk = time_dim.t_time_sk - AND ss_hdemo_sk = household_demographics.hd_demo_sk - AND ss_store_sk = s_store_sk - AND time_dim.t_hour = 11 - AND time_dim.t_minute < 30 - AND ((household_demographics.hd_dep_count = 4 - AND household_demographics.hd_vehicle_count<=4+2) - OR (household_demographics.hd_dep_count = 2 - AND household_demographics.hd_vehicle_count<=2+2) - OR (household_demographics.hd_dep_count = 0 - AND household_demographics.hd_vehicle_count<=0+2)) - AND store.s_store_name = 'ese') s6, - (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h11_30_to_12 - FROM store_sales, - household_demographics, - time_dim, - store - WHERE ss_sold_time_sk = time_dim.t_time_sk - AND ss_hdemo_sk = household_demographics.hd_demo_sk - AND ss_store_sk = s_store_sk - AND time_dim.t_hour = 11 - AND time_dim.t_minute >= 30 - AND ((household_demographics.hd_dep_count = 4 - AND household_demographics.hd_vehicle_count<=4+2) - OR (household_demographics.hd_dep_count = 2 - AND household_demographics.hd_vehicle_count<=2+2) - OR (household_demographics.hd_dep_count = 0 - AND household_demographics.hd_vehicle_count<=0+2)) - AND store.s_store_name = 'ese') s7, - (SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) h12_to_12_30 - FROM store_sales, - household_demographics, - time_dim, - store - WHERE ss_sold_time_sk = time_dim.t_time_sk - AND ss_hdemo_sk = household_demographics.hd_demo_sk - AND ss_store_sk = s_store_sk - AND time_dim.t_hour = 12 - AND time_dim.t_minute < 30 - AND ((household_demographics.hd_dep_count = 4 - AND household_demographics.hd_vehicle_count<=4+2) - OR (household_demographics.hd_dep_count = 2 - AND household_demographics.hd_vehicle_count<=2+2) - OR (household_demographics.hd_dep_count = 0 - AND household_demographics.hd_vehicle_count<=0+2)) - AND store.s_store_name = 'ese') s8 ; diff --git a/benchmark/tpcds/as_queries/q90.sql b/benchmark/tpcds/as_queries/q90.sql deleted file mode 100644 index 2de65cf8..00000000 --- a/benchmark/tpcds/as_queries/q90.sql +++ /dev/null @@ -1,26 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- Evaluate the NULL guard and DECIMAL ratio lane-wise before FLOAT-list finalization. -SELECT CAST(as_noised(list_transform(list_zip(amc,pmc), lambda x: CASE WHEN x[2]=0 THEN NULL ELSE CAST(x[1] AS DECIMAL(15,4))/CAST(x[2] AS DECIMAL(15,4)) END)::FLOAT[]) AS DOUBLE) AS am_pm_ratio -FROM (SELECT as_count(priv_hash(hash(ws_bill_customer_sk)), 1::BIGINT) amc - FROM web_sales, - household_demographics, - time_dim, - web_page - WHERE ws_sold_time_sk = time_dim.t_time_sk - AND ws_ship_hdemo_sk = household_demographics.hd_demo_sk - AND ws_web_page_sk = web_page.wp_web_page_sk - AND time_dim.t_hour BETWEEN 8 AND 8+1 - AND household_demographics.hd_dep_count = 6 - AND web_page.wp_char_count BETWEEN 5000 AND 5200) at_, (SELECT as_count(priv_hash(hash(ws_bill_customer_sk)), 1::BIGINT) pmc - FROM web_sales, - household_demographics, - time_dim, - web_page - WHERE ws_sold_time_sk = time_dim.t_time_sk - AND ws_ship_hdemo_sk = household_demographics.hd_demo_sk - AND ws_web_page_sk = web_page.wp_web_page_sk - AND time_dim.t_hour BETWEEN 19 AND 19+1 - AND household_demographics.hd_dep_count = 6 - AND web_page.wp_char_count BETWEEN 5000 AND 5200) pt -ORDER BY am_pm_ratio -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q91.sql b/benchmark/tpcds/as_queries/q91.sql deleted file mode 100644 index cc0f31e5..00000000 --- a/benchmark/tpcds/as_queries/q91.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT cc_call_center_id Call_Center, - cc_name Call_Center_Name, - cc_manager Manager, - as_noised_sum(priv_hash(hash(c_customer_sk)), cr_net_loss) Returns_Loss -FROM call_center, - catalog_returns, - date_dim, - customer, - customer_address, - customer_demographics, - household_demographics -WHERE cr_call_center_sk = cc_call_center_sk - AND cr_returned_date_sk = d_date_sk - AND cr_returning_customer_sk= c_customer_sk - AND cd_demo_sk = c_current_cdemo_sk - AND hd_demo_sk = c_current_hdemo_sk - AND ca_address_sk = c_current_addr_sk - AND d_year = 1998 - AND d_moy = 11 - AND ((cd_marital_status = 'M' - AND cd_education_status = 'Unknown') or(cd_marital_status = 'W' - AND cd_education_status = 'Advanced Degree')) - AND hd_buy_potential LIKE 'Unknown%' - AND ca_gmt_offset = -7 -GROUP BY cc_call_center_id, - cc_name, - cc_manager, - cd_marital_status, - cd_education_status -ORDER BY as_noised_sum(priv_hash(hash(c_customer_sk)), cr_net_loss) DESC; diff --git a/benchmark/tpcds/as_queries/q92.sql b/benchmark/tpcds/as_queries/q92.sql deleted file mode 100644 index 6f041b12..00000000 --- a/benchmark/tpcds/as_queries/q92.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -SELECT as_noised_sum(priv_select_gt(priv_hash(hash(ws_bill_customer_sk)), CAST(ws_ext_discount_amt AS FLOAT), (SELECT list_transform(priv_div(as_sum(priv_hash(hash(ws_bill_customer_sk)), CAST(ws_ext_discount_amt AS DOUBLE)), as_count(priv_hash(hash(ws_bill_customer_sk)), ws_ext_discount_amt)), lambda x: CAST(1.3 * x AS FLOAT)) - FROM web_sales, - date_dim - WHERE ws_item_sk = i_item_sk - AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) - AND d_date_sk = ws_sold_date_sk )), ws_ext_discount_amt) AS "Excess Discount Amount" -FROM web_sales, - item, - date_dim -WHERE i_manufact_id = 350 - AND i_item_sk = ws_item_sk - AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) - AND d_date_sk = ws_sold_date_sk - AND priv_select_gt(priv_hash(hash(ws_bill_customer_sk)), CAST(ws_ext_discount_amt AS FLOAT), (SELECT list_transform(priv_div(as_sum(priv_hash(hash(ws_bill_customer_sk)), CAST(ws_ext_discount_amt AS DOUBLE)), as_count(priv_hash(hash(ws_bill_customer_sk)), ws_ext_discount_amt)), lambda x: CAST(1.3 * x AS FLOAT)) - FROM web_sales, - date_dim - WHERE ws_item_sk = i_item_sk - AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) - AND d_date_sk = ws_sold_date_sk )) <> 0 -ORDER BY "Excess Discount Amount" -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q93.sql b/benchmark/tpcds/as_queries/q93.sql deleted file mode 100644 index 8be9d191..00000000 --- a/benchmark/tpcds/as_queries/q93.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. -SELECT ss_customer_sk, - as_noised_sum(priv_hash(hash(ss_customer_sk)), act_sales) sumsales -FROM - (SELECT ss_item_sk, - ss_ticket_number, - ss_customer_sk, - CASE - WHEN sr_return_quantity IS NOT NULL THEN (ss_quantity-sr_return_quantity)*ss_sales_price - ELSE (ss_quantity*ss_sales_price) - END act_sales - FROM store_sales - LEFT OUTER JOIN store_returns ON (sr_item_sk = ss_item_sk - AND sr_ticket_number = ss_ticket_number) ,reason - WHERE sr_reason_sk = r_reason_sk - AND r_reason_desc = 'reason 28') t -GROUP BY ss_customer_sk -ORDER BY sumsales NULLS FIRST, - ss_customer_sk NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q94.sql b/benchmark/tpcds/as_queries/q94.sql deleted file mode 100644 index 3b2858b8..00000000 --- a/benchmark/tpcds/as_queries/q94.sql +++ /dev/null @@ -1,24 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- COUNT(DISTINCT) unions membership by order; EXISTS/NOT EXISTS retain their original SQL semantics. -WITH asa_input AS NOT MATERIALIZED (SELECT ws_order_number, ws_ext_ship_cost, ws_net_profit, priv_hash(hash(ws1.ws_bill_customer_sk)) AS membership FROM web_sales ws1 , - date_dim , - customer_address , - web_site -WHERE d_date BETWEEN '1999-02-01' AND cast('1999-04-02' AS date) - AND ws1.ws_ship_date_sk = d_date_sk - AND ws1.ws_ship_addr_sk = ca_address_sk - AND ca_state = 'IL' - AND ws1.ws_web_site_sk = web_site_sk - AND web_company_name = 'pri' - AND EXISTS - (SELECT * - FROM web_sales ws2 - WHERE ws1.ws_order_number = ws2.ws_order_number - AND ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) - AND NOT exists - (SELECT * - FROM web_returns wr1 - WHERE ws1.ws_order_number = wr1.wr_order_number) -), -asa_distinct AS NOT MATERIALIZED (SELECT ws_order_number, bit_or(membership) FILTER (WHERE ws_order_number IS NOT NULL) AS membership FROM asa_input GROUP BY ws_order_number) -SELECT d.n AS "order count", s.shipping AS "total shipping cost", s.profit AS "total net profit" FROM (SELECT as_noised_count(membership, 1::BIGINT) AS n FROM asa_distinct) d CROSS JOIN (SELECT as_noised_sum(membership, ws_ext_ship_cost) AS shipping, as_noised_sum(membership, ws_net_profit) AS profit FROM asa_input) s ORDER BY "order count" LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q95.sql b/benchmark/tpcds/as_queries/q95.sql deleted file mode 100644 index 32c7aedd..00000000 --- a/benchmark/tpcds/as_queries/q95.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Handwritten PAC SQL, following the compiled aggregate scopes and scalar output semantics. --- The ws_wh CTE stays ordinary; order membership unions and SUMs use the outer web-sales owner. -WITH ws_wh AS NOT MATERIALIZED (SELECT ws1.ws_order_number, - ws1.ws_warehouse_sk wh1, - ws2.ws_warehouse_sk wh2 - FROM web_sales ws1, - web_sales ws2 - WHERE ws1.ws_order_number = ws2.ws_order_number - AND ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk), -asa_input AS NOT MATERIALIZED (SELECT ws_order_number, ws_ext_ship_cost, ws_net_profit, priv_hash(hash(ws1.ws_bill_customer_sk)) AS membership FROM web_sales ws1 , - date_dim , - customer_address , - web_site -WHERE d_date BETWEEN '1999-02-01' AND cast('1999-04-02' AS date) - AND ws1.ws_ship_date_sk = d_date_sk - AND ws1.ws_ship_addr_sk = ca_address_sk - AND ca_state = 'IL' - AND ws1.ws_web_site_sk = web_site_sk - AND web_company_name = 'pri' - AND ws1.ws_order_number IN - (SELECT ws_order_number - FROM ws_wh) - AND ws1.ws_order_number IN - (SELECT wr_order_number - FROM web_returns, - ws_wh - WHERE wr_order_number = ws_wh.ws_order_number) -), -asa_distinct AS NOT MATERIALIZED (SELECT ws_order_number, bit_or(membership) FILTER (WHERE ws_order_number IS NOT NULL) AS membership FROM asa_input GROUP BY ws_order_number) -SELECT d.n AS "order count", s.shipping AS "total shipping cost", s.profit AS "total net profit" FROM (SELECT as_noised_count(membership, 1::BIGINT) AS n FROM asa_distinct) d CROSS JOIN (SELECT as_noised_sum(membership, ws_ext_ship_cost) AS shipping, as_noised_sum(membership, ws_net_profit) AS profit FROM asa_input) s ORDER BY "order count" LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q96.sql b/benchmark/tpcds/as_queries/q96.sql deleted file mode 100644 index 32da38b4..00000000 --- a/benchmark/tpcds/as_queries/q96.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) -FROM store_sales , - household_demographics, - time_dim, - store -WHERE ss_sold_time_sk = time_dim.t_time_sk - AND ss_hdemo_sk = household_demographics.hd_demo_sk - AND ss_store_sk = s_store_sk - AND time_dim.t_hour = 20 - AND time_dim.t_minute >= 30 - AND household_demographics.hd_dep_count = 7 - AND store.s_store_name = 'ese' -ORDER BY as_noised_count(priv_hash(hash(ss_customer_sk)), 1::BIGINT) -LIMIT 100; diff --git a/benchmark/tpcds/as_queries/q99.sql b/benchmark/tpcds/as_queries/q99.sql deleted file mode 100644 index 4b680ee8..00000000 --- a/benchmark/tpcds/as_queries/q99.sql +++ /dev/null @@ -1,45 +0,0 @@ --- Handwritten PAC aggregates; automatic query rewriting is disabled by the benchmark. -SELECT w_substr , - sm_type , - LOWER(cc_name) cc_name_lower , - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE - WHEN (cs_ship_date_sk - cs_sold_date_sk <= 30) THEN 1 - ELSE 0 - END) AS "30 days", - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE - WHEN (cs_ship_date_sk - cs_sold_date_sk > 30) - AND (cs_ship_date_sk - cs_sold_date_sk <= 60) THEN 1 - ELSE 0 - END) AS "31-60 days", - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE - WHEN (cs_ship_date_sk - cs_sold_date_sk > 60) - AND (cs_ship_date_sk - cs_sold_date_sk <= 90) THEN 1 - ELSE 0 - END) AS "61-90 days", - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE - WHEN (cs_ship_date_sk - cs_sold_date_sk > 90) - AND (cs_ship_date_sk - cs_sold_date_sk <= 120) THEN 1 - ELSE 0 - END) AS "91-120 days", - as_noised_sum(priv_hash(hash(cs_bill_customer_sk)), CASE - WHEN (cs_ship_date_sk - cs_sold_date_sk > 120) THEN 1 - ELSE 0 - END) AS ">120 days" -FROM catalog_sales , - (SELECT SUBSTRING(w_warehouse_name,1,20) w_substr, * - FROM warehouse) AS sq1 , - ship_mode , - call_center , - date_dim -WHERE d_month_seq BETWEEN 1200 AND 1200 + 11 - AND cs_ship_date_sk = d_date_sk - AND cs_warehouse_sk = w_warehouse_sk - AND cs_ship_mode_sk = sm_ship_mode_sk - AND cs_call_center_sk = cc_call_center_sk -GROUP BY w_substr , - sm_type , - cc_name -ORDER BY w_substr NULLS FIRST, - sm_type NULLS FIRST, - cc_name_lower NULLS FIRST -LIMIT 100; diff --git a/benchmark/tpcds/as_tpcds_benchmark.cpp b/benchmark/tpcds/as_tpcds_benchmark.cpp index d622a741..5fe7a50a 100644 --- a/benchmark/tpcds/as_tpcds_benchmark.cpp +++ b/benchmark/tpcds/as_tpcds_benchmark.cpp @@ -1,14 +1,10 @@ -// Performance/coverage follows TPC-H's PAC output semantics: 64 internal worlds and scalar/categorical outputs. -// pac_mi=0 disables additive noise; priv_filter still uses PAC's deterministic majority decision. -// The timed AS path uses handwritten PAC SQL, with automatic compilation recorded separately for coverage. -// Per-world AS references are separate diagnostics, not compiler-equivalence or timing oracles. +// TPC-DS coverage and performance use the original SQL and the automatic rewriter. +// Execution success is not an independent validation of per-world correctness. #include "duckdb.hpp" #include "benchmark_json.hpp" #include "benchmark_paths.hpp" -#include #include -#include #include #include #include @@ -57,363 +53,10 @@ static string Name(int query) { return name.str(); } -// Separate per-world reference queries expand reduced aggregate results, never base tuples. -using WorldRows = vector>>; - -static void SetHandwrittenAggregates(Connection &con, bool conditional, idx_t first_world, idx_t world_count) { - for (auto &operation : - {string("sum"), string("count"), string("avg"), string("stddev_samp"), string("min"), string("max")}) { - string expression; - if (conditional) { - expression = "["; - for (idx_t world = 0; world < 64; world++) { - if (world) { - expression += ","; - } - if (world < first_world || world >= first_world + world_count) { - expression += "NULL"; - } else { - expression += operation + "(v) FILTER (WHERE (m & " + std::to_string(uint64_t(1) << world) + - "::UBIGINT) <> 0)"; - } - } - expression += "]"; - } else if (operation == "stddev_samp") { - expression = "as_stddev_samp(m, v)"; - } else if (operation == "count") { - expression = "list_transform(as_count(m, v), lambda x: x / 2.0)"; - } else { - auto aggregate = operation == "avg" ? "sum" : operation; - auto value = operation == "avg" ? "x[1] / x[2]" : operation == "sum" ? "x[1] / 2.0" : "x[1]"; - // MIN/MAX dispatch on their SQL input type during binding, including macro definition. - auto argument = operation == "min" || operation == "max" ? "CAST(v AS DOUBLE)" : "v"; - expression = "list_transform(list_zip(as_" + string(aggregate) + "(m, " + argument + - "), as_count(m, v)), lambda x: CASE WHEN x[2] = 0 THEN NULL ELSE " + value + " END)"; - } - Query(con, "CREATE OR REPLACE TEMP MACRO tpcds_" + operation + "(m, v) AS " + expression); - } -} - -static WorldRows CollectHandwritten(QueryResult &result) { - WorldRows rows(64); - for (idx_t r = 0; r < result.RowCount(); r++) { - auto world = result.Collection().GetValue(0, r).GetValue(); - if (world < 0 || world >= 64) { - throw std::runtime_error("Invalid handwritten world index"); - } - vector row; - for (idx_t col = 1; col < result.ColumnCount(); col++) { - row.push_back(result.Collection().GetValue(col, r)); - } - rows[world].push_back(std::move(row)); - } - return rows; -} - -static void InstallSampledViews(Connection &con) { - Query(con, "SET VARIABLE tpcds_world=0"); - Query(con, "CREATE SCHEMA IF NOT EXISTS tpcds_sampled"); - Query(con, "CREATE OR REPLACE VIEW tpcds_sampled.customer AS SELECT * FROM main.customer " - "WHERE ((priv_hash(hash(c_customer_sk)) >> getvariable('tpcds_world')) & 1) <> 0"); - for (const auto &owner : vector> {{"store_sales", "ss_customer_sk"}, - {"store_returns", "sr_customer_sk"}, - {"catalog_sales", "cs_bill_customer_sk"}, - {"catalog_returns", "cr_returning_customer_sk"}, - {"web_sales", "ws_bill_customer_sk"}, - {"web_returns", "wr_returning_customer_sk"}}) { - Query(con, "CREATE OR REPLACE VIEW tpcds_sampled." + owner.first + " AS SELECT * FROM main." + owner.first + - " WHERE " + owner.second + " IN (SELECT c_customer_sk FROM tpcds_sampled.customer)"); - } -} - -struct ValidationComparison { - idx_t rows = 0; - idx_t values = 0; - idx_t exact_mismatches = 0; - idx_t float_mismatches = 0; - idx_t empty_worlds = 0; - double max_absolute_error = 0; - double max_relative_error = 0; -}; - -// SQL ORDER BY may leave ties unresolved. Compare result multisets, retaining the SQL's per-world LIMIT. -static vector CanonicalRowOrder(const vector> &rows, const vector &keys) { - vector order; - vector labels; - for (idx_t row = 0; row < rows.size(); row++) { - order.push_back(row); - string key; - for (idx_t col = 0; col < keys.size(); col++) { - if (keys[col]) { - auto value = rows[row][col].IsNull() ? string("NULL") : rows[row][col].ToString(); - key += std::to_string(value.size()) + ":" + value; - } - } - labels.push_back(std::move(key)); - } - std::sort(order.begin(), order.end(), [&](idx_t a, idx_t b) { - if (labels[a] != labels[b]) { - return labels[a] < labels[b]; - } - for (idx_t col = 0; col < keys.size(); col++) { - if (keys[col]) { - continue; - } - auto &x = rows[a][col]; - auto &y = rows[b][col]; - if (x.IsNull() != y.IsNull()) { - return x.IsNull(); - } - if (!x.IsNull()) { - auto xv = x.GetValue(), yv = y.GetValue(); - if (std::isnan(xv) != std::isnan(yv)) { - return std::isnan(xv); - } - if (xv != yv && !(std::isnan(xv) && std::isnan(yv))) { - return xv < yv; - } - } - } - return false; - }); - return order; -} - -static ValidationComparison CompareWorldRows(const WorldRows &expected, const WorldRows &actual, idx_t world_count = 64, - bool integral_keys = true) { - ValidationComparison comparison; - for (idx_t world = 0; world < world_count; world++) { - if (expected[world].size() != actual[world].size()) { - throw std::runtime_error("World " + std::to_string(world) + " row count: expected " + - std::to_string(expected[world].size()) + ", got " + - std::to_string(actual[world].size())); - } - comparison.empty_worlds += expected[world].empty(); - comparison.rows += expected[world].size(); - if (expected[world].empty()) { - continue; - } - if (expected[world][0].size() != actual[world][0].size()) { - throw std::runtime_error("Handwritten output column count differs"); - } - vector keys; - for (idx_t col = 0; col < expected[world][0].size(); col++) { - auto &e = expected[world][0][col].type(); - auto &a = actual[world][0][col].type(); - keys.push_back( - !e.IsNumeric() || !a.IsNumeric() || - (integral_keys && e == a && e.id() != LogicalTypeId::DOUBLE && e.id() != LogicalTypeId::FLOAT)); - } - auto eo = CanonicalRowOrder(expected[world], keys); - auto ao = CanonicalRowOrder(actual[world], keys); - for (idx_t row = 0; row < expected[world].size(); row++) { - if (expected[world][eo[row]].size() != actual[world][ao[row]].size()) { - throw std::runtime_error("Handwritten output column count differs"); - } - for (idx_t col = 0; col < expected[world][row].size(); col++) { - auto &e = expected[world][eo[row]][col]; - auto &a = actual[world][ao[row]][col]; - auto location = "world=" + std::to_string(world) + ", row=" + std::to_string(row) + - ", column=" + std::to_string(col); - if (e.IsNull() != a.IsNull()) { - throw std::runtime_error("NULL mismatch: " + location); - } - if (e.IsNull()) { - continue; - } - if (!e.type().IsNumeric() || !a.type().IsNumeric()) { - if (e.ToString() != a.ToString()) { - throw std::runtime_error("Value/order mismatch: " + location); - } - continue; - } - double x = e.GetValue(), y = a.GetValue(); - if (x == y || (std::isnan(x) && std::isnan(y))) { - comparison.values++; - continue; - } - if (!std::isfinite(x) || !std::isfinite(y)) { - throw std::runtime_error("Non-finite value mismatch: " + location); - } - comparison.exact_mismatches++; - comparison.float_mismatches += static_cast(x) != static_cast(y); - comparison.values++; - auto error = std::abs(x - y); - comparison.max_absolute_error = std::max(comparison.max_absolute_error, error); - comparison.max_relative_error = - std::max(comparison.max_relative_error, error / std::max(1.0, std::abs(x))); - } - } - } - return comparison; -} - -static void ValidateHandwritten(Connection &con, const JsonValue &config, int id) { - auto name = Name(id); - auto output = JsonString(config, "out_dir"); - auto directory = JsonString(config, "reference_queries_dir", "benchmark/tpcds/reference_queries"); - auto handwritten = Read(directory + "/" + name + ".sql"); - auto original = Read(JsonString(config, "queries_dir") + "/" + name + ".sql"); - Query(con, "SET priv_rewrite=false"); - InstallSampledViews(con); - Query(con, "SET search_path='tpcds_sampled,main'"); - WorldRows expected(64); - for (idx_t world = 0; world < 64; world++) { - Query(con, "SET VARIABLE tpcds_world=" + std::to_string(world)); - auto result = Query(con, original); - for (idx_t row = 0; row < result->RowCount(); row++) { - vector values; - for (idx_t col = 0; col < result->ColumnCount(); col++) { - values.push_back(result->Collection().GetValue(col, row)); - } - expected[world].push_back(std::move(values)); - } - } - std::ofstream csv(output + "/" + name + "_validation.csv"); - csv << "query,method,status,worlds,rows,numeric_values,exact_mismatches,float_rounded_mismatches," - "max_absolute_error,max_relative_error,empty_worlds,error\n"; - for (bool conditional : {true, false}) { - auto method = conditional ? "conditional_sql" : "simd_asa_handwritten"; - try { - WorldRows actual(64); - // Bound the ordinary conditional oracle's state size; this is validation, not timing. - auto block_size = conditional ? idx_t(8) : idx_t(64); - for (idx_t first = 0; first < 64; first += block_size) { - SetHandwrittenAggregates(con, conditional, first, block_size); - auto result = Query(con, handwritten); - auto block = CollectHandwritten(*result); - for (idx_t world = first; world < first + block_size; world++) { - actual[world] = std::move(block[world]); - } - } - auto comparison = CompareWorldRows(expected, actual); - auto status = comparison.exact_mismatches ? "numeric_difference" - : comparison.empty_worlds == 64 ? "empty_only" - : "exact_match"; - csv << name << ',' << method << ',' << status << ",64," << comparison.rows << ',' << comparison.values - << ',' << comparison.exact_mismatches << ',' << comparison.float_mismatches << ',' - << std::setprecision(17) << comparison.max_absolute_error << ',' << comparison.max_relative_error << ',' - << comparison.empty_worlds << ",\"\"\n"; - } catch (const std::exception &error) { - csv << name << ',' << method << ",error,64,,,,,,,," << Csv(error.what()) << '\n'; - } - csv.flush(); - } - Query(con, "SET search_path='main'"); - Query(con, "DROP SCHEMA tpcds_sampled CASCADE"); -} - -// TPC-H-style equivalence check: original SQL through the compiler versus explicit PAC SQL. -static void ValidatePacQuery(Connection &con, const JsonValue &config, int id) { - auto name = Name(id); - auto output = JsonString(config, "out_dir"); - auto original = Read(JsonString(config, "queries_dir") + "/" + name + ".sql"); - auto handwritten = Read(JsonString(config, "as_queries_dir", "benchmark/tpcds/as_queries") + "/" + name + ".sql"); - std::ofstream csv(output + "/" + name + "_pac_validation.csv"); - csv << "query,status,compiled_rows,handwritten_rows,numeric_values,exact_mismatches,float_rounded_mismatches," - "max_absolute_error,max_relative_error,error\n"; - idx_t compiled_rows = 0, handwritten_rows = 0; - try { - WorldRows expected(64), actual(64); - for (bool automatic : {true, false}) { - Query(con, "SET priv_rewrite=" + string(automatic ? "true" : "false")); - auto &sql = automatic ? original : handwritten; - auto plan = Query(con, "EXPLAIN " + sql); - std::ofstream plan_file(output + "/" + name + (automatic ? "_compiled.plan.txt" : "_handwritten.plan.txt")); - for (idx_t row = 0; row < plan->RowCount(); row++) { - plan_file << plan->Collection().GetValue(1, row).ToString() << '\n'; - } - auto result = Query(con, sql); - if (automatic) { - compiled_rows = result->RowCount(); - } else { - handwritten_rows = result->RowCount(); - } - auto &rows = automatic ? expected[0] : actual[0]; - for (idx_t row = 0; row < result->RowCount(); row++) { - vector values; - for (idx_t col = 0; col < result->ColumnCount(); col++) { - values.push_back(result->Collection().GetValue(col, row)); - } - rows.push_back(std::move(values)); - } - } - auto comparison = CompareWorldRows(expected, actual, 1, false); - auto status = comparison.exact_mismatches ? "numeric_difference" - : comparison.empty_worlds ? "empty_only" - : "exact_match"; - csv << name << ',' << status << ',' << compiled_rows << ',' << handwritten_rows << ',' << comparison.values - << ',' << comparison.exact_mismatches << ',' << comparison.float_mismatches << ',' << std::setprecision(17) - << comparison.max_absolute_error << ',' << comparison.max_relative_error << ",\"\"\n"; - } catch (const std::exception &error) { - csv << name << ",error," << compiled_rows << ',' << handwritten_rows << ",,,,,," << Csv(error.what()) << '\n'; - } - Query(con, "SET priv_rewrite=false"); -} - -// Output-semantics diagnostic only: ordinary world-zero SQL is not an equivalence oracle for PAC output. -// With mi=0, scalar terminals select lane 0, but PAC terminal filters may use majority decisions and SUM/COUNT -// retain their scaling. Differences recorded here must not be interpreted as compiler-correctness failures. -static void ValidateCompiled(Connection &con, const JsonValue &config, int id) { - auto name = Name(id); - auto output = JsonString(config, "out_dir"); - auto original = Read(JsonString(config, "queries_dir") + "/" + name + ".sql"); - Query(con, "SET priv_rewrite=false"); - InstallSampledViews(con); - Query(con, "SET search_path='tpcds_sampled,main'"); - auto reference = Query(con, original); - WorldRows expected(64), actual(64); - for (idx_t row = 0; row < reference->RowCount(); row++) { - vector values; - for (idx_t col = 0; col < reference->ColumnCount(); col++) { - values.push_back(reference->Collection().GetValue(col, row)); - } - expected[0].push_back(std::move(values)); - } - Query(con, "SET search_path='main'; SET priv_rewrite=true"); - std::ofstream csv(output + "/" + name + "_compiled_validation.csv"); - csv << "query,status,reference_world,reference_rows,compiled_rows,numeric_values,exact_mismatches," - "float_rounded_mismatches,max_absolute_error,max_relative_error,error\n"; - idx_t compiled_rows = 0; - bool executed = false; - try { - auto explain = Query(con, "EXPLAIN " + original); - std::ofstream plan(output + "/" + name + "_compiled_validation.plan.txt"); - for (idx_t row = 0; row < explain->RowCount(); row++) { - plan << explain->Collection().GetValue(1, row).ToString() << '\n'; - } - auto compiled = Query(con, original); - compiled_rows = compiled->RowCount(); - executed = true; - for (idx_t row = 0; row < compiled_rows; row++) { - vector values; - for (idx_t col = 0; col < compiled->ColumnCount(); col++) { - values.push_back(compiled->Collection().GetValue(col, row)); - } - actual[0].push_back(std::move(values)); - } - auto comparison = CompareWorldRows(expected, actual, 1, false); - auto status = comparison.exact_mismatches ? "numeric_difference" - : comparison.empty_worlds ? "empty_only" - : "exact_match"; - csv << name << ',' << status << ",0," << reference->RowCount() << ',' << compiled_rows << ',' - << comparison.values << ',' << comparison.exact_mismatches << ',' << comparison.float_mismatches << ',' - << std::setprecision(17) << comparison.max_absolute_error << ',' << comparison.max_relative_error - << ",\"\"\n"; - } catch (const std::exception &error) { - csv << name << (executed ? ",result_mismatch,0," : ",execution_error,0,") << reference->RowCount() << ',' - << compiled_rows << ",,,,,," << Csv(error.what()) << '\n'; - } - Query(con, "SET priv_rewrite=false; SET search_path='main'"); - Query(con, "DROP SCHEMA tpcds_sampled CASCADE"); -} - int main(int argc, char **argv) { try { - if (argc != 3 && !(argc == 4 && (string(argv[3]) == "--validate" || string(argv[3]) == "--validate-worlds" || - string(argv[3]) == "--validate-compiled"))) { - throw std::runtime_error("Usage: as_tpcds_benchmark CONFIG --prepare|QUERY_NUMBER " - "[--validate|--validate-worlds|--validate-compiled]"); + if (argc != 3) { + throw std::runtime_error("Usage: as_tpcds_benchmark CONFIG --prepare|QUERY_NUMBER"); } auto config = JsonParser(Read(argv[1])).Parse(); auto db_path = JsonString(config, "db"); @@ -434,7 +77,7 @@ int main(int argc, char **argv) { Query(con, "SET memory_limit=" + Quote(JsonString(config, "memory_limit", "4GB"))); // privacy_noise=false bypasses PAC compilation; mi=0 disables additive noise without bypassing it. Query(con, "SET priv_rewrite=false; SET pac_mi=0; SET privacy_noise=true; SET privacy_seed=42; " - "SET pac_sample_diversity_check=false; SET pac_conservative_mode=true; SET priv_check=false;"); + "SET pac_sample_diversity_check=false; SET pac_conservative_mode=true;"); if (string(argv[2]) == "--prepare") { auto extension_dir = JsonString(config, "extension_directory"); if (!extension_dir.empty()) { @@ -481,7 +124,7 @@ int main(int argc, char **argv) { metadata << "SUM: approximate\n"; #endif metadata << "Settings: m=64, privacy_seed=42, pac_mi=0, privacy_noise=true, " - "pac_sample_diversity_check=false, pac_conservative_mode=true, priv_check=false\n"; + "pac_sample_diversity_check=false, pac_conservative_mode=true\n"; Query(con, "CHECKPOINT"); return 0; } @@ -501,49 +144,17 @@ int main(int argc, char **argv) { Query(con, declaration); } } - if (argc == 4) { - if (string(argv[3]) == "--validate-compiled") { - ValidateCompiled(con, config, id); - return 0; - } - if (string(argv[3]) == "--validate-worlds") { - ValidateHandwritten(con, config, id); - } else { - ValidatePacQuery(con, config, id); - } - return 0; - } std::ofstream csv(output + "/" + name + ".csv"); if (!csv) { throw std::runtime_error("Cannot create result CSV"); } - csv << "query,mode,phase,status,rewritten,repetition,runtime_ms,rows,world_multiplier,error\n"; - for (auto mode : - {string("duckdb"), string("simd_asa_handwritten"), string("simd_asa_auto"), string("naive_as")}) { - Query(con, "SET priv_rewrite=" + string(mode == "simd_asa_auto" ? "true" : "false")); - string query_sql = sql; - if (mode == "simd_asa_handwritten") { - auto path = JsonString(config, "as_queries_dir", "benchmark/tpcds/as_queries") + "/" + name + ".sql"; - std::ifstream handwritten(path); - if (!handwritten) { - csv << name << ',' << mode << ",setup,not_available,0,0,,,,\"\"\n"; - continue; - } - query_sql = Read(path); - } - auto multiplier = mode == "naive_as" ? 64 : 1; + csv << "query,mode,phase,status,rewritten,repetition,runtime_ms,rows,error\n"; + for (auto mode : {string("duckdb"), string("simd_asa_auto")}) { + Query(con, "SET priv_rewrite=" + string(mode == "duckdb" ? "false" : "true")); bool rewritten = false; string phase = "explain"; try { - if (mode == "naive_as") { - // Match TPC-H: prepare once outside timing, time one sampled-world execution, - // and multiply by 64. Views apply customer sampling at query time to every fact scan. - InstallSampledViews(con); - Query(con, "SET search_path='tpcds_sampled,main'"); - Query(con, "PREPARE tpcds_naive AS " + sql); - query_sql = "EXECUTE tpcds_naive"; - } - auto explain = Query(con, "EXPLAIN " + query_sql); + auto explain = Query(con, "EXPLAIN " + sql); string plan; for (idx_t row = 0; row < explain->RowCount(); row++) { plan += explain->Collection().GetValue(1, row).ToString() + "\n"; @@ -552,32 +163,27 @@ int main(int argc, char **argv) { plan_file << plan; rewritten = plan.find("as_noised_") != string::npos || plan.find("as_sum(") != string::npos || plan.find("as_count(") != string::npos || plan.find("priv_select") != string::npos; - if ((mode == "simd_asa_auto" || mode == "simd_asa_handwritten") && !rewritten) { - csv << name << ',' << mode << ",explain,no_stochastic_operator,0,0,,,,\"\"\n"; + if (mode != "duckdb" && !rewritten) { + csv << name << ',' << mode << ",explain,no_stochastic_operator,0,0,,,\"\"\n"; csv.flush(); continue; } phase = "warmup"; - Query(con, query_sql); + Query(con, sql); phase = "measure"; for (int run = 1; run <= repetitions; run++) { auto start = std::chrono::steady_clock::now(); - auto result = Query(con, query_sql); + auto result = Query(con, sql); auto elapsed = std::chrono::duration(std::chrono::steady_clock::now() - start); csv << name << ',' << mode << ",measure,ok," << rewritten << ',' << run << ',' - << std::setprecision(12) << elapsed.count() * multiplier << ',' << result->RowCount() << ',' - << multiplier << ",\"\"\n"; + << std::setprecision(12) << elapsed.count() << ',' << result->RowCount() << ",\"\"\n"; csv.flush(); } } catch (const std::exception &error) { - csv << name << ',' << mode << ',' << phase << ",error," << rewritten << ",0,,,," << Csv(error.what()) + csv << name << ',' << mode << ',' << phase << ",error," << rewritten << ",0,,," << Csv(error.what()) << '\n'; csv.flush(); } - if (mode == "naive_as") { - Query(con, "SET search_path='main'"); - Query(con, "DROP SCHEMA IF EXISTS tpcds_sampled CASCADE"); - } } return 0; } catch (const std::exception &error) { diff --git a/benchmark/tpcds/q17_fixture.sql b/benchmark/tpcds/q17_fixture.sql deleted file mode 100644 index 2beeac68..00000000 --- a/benchmark/tpcds/q17_fixture.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Synthetic nonempty correctness fixture for the unchanged TPC-DS Q17 SQL, not a performance dataset. --- Run against a fresh database. Empty owned tables allow the standard ownership setup to be reused. -SET priv_rewrite=false; -CREATE TABLE customer AS SELECT i::INTEGER AS c_customer_sk FROM range(128) r(i); -CREATE TABLE store_sales AS -SELECT c_customer_sk AS ss_customer_sk, 1 AS ss_item_sk, c_customer_sk AS ss_ticket_number, - 1 AS ss_sold_date_sk, 1 AS ss_store_sk, c_customer_sk % 10 + 1 AS ss_quantity -FROM customer; -CREATE TABLE store_returns AS -SELECT c_customer_sk AS sr_customer_sk, 1 AS sr_item_sk, c_customer_sk AS sr_ticket_number, - 1 AS sr_returned_date_sk, c_customer_sk % 4 + 1 AS sr_return_quantity -FROM customer; -CREATE TABLE catalog_sales AS -SELECT c_customer_sk AS cs_bill_customer_sk, 1 AS cs_item_sk, 1 AS cs_sold_date_sk, - c_customer_sk % 13 + 1 AS cs_quantity -FROM customer; -CREATE TABLE catalog_returns(cr_returning_customer_sk INTEGER); -CREATE TABLE web_sales(ws_bill_customer_sk INTEGER); -CREATE TABLE web_returns(wr_returning_customer_sk INTEGER); -CREATE TABLE date_dim AS SELECT 1 AS d_date_sk, '2001Q1' AS d_quarter_name; -CREATE TABLE store AS SELECT 1 AS s_store_sk, 'CA' AS s_state; -CREATE TABLE item AS SELECT 1 AS i_item_sk, 'ITEM1' AS i_item_id, 'test item' AS i_item_desc; diff --git a/benchmark/tpcds/reference_queries/q01.sql b/benchmark/tpcds/reference_queries/q01.sql deleted file mode 100644 index 5509f88e..00000000 --- a/benchmark/tpcds/reference_queries/q01.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Correlated thresholds use only the matching world of the reduced return aggregates. -WITH return_stats AS ( -SELECT sr_customer_sk AS ctr_customer_sk, - sr_store_sk AS ctr_store_sk, - tpcds_sum(priv_hash(hash(sr_customer_sk)), sr_return_amt) AS ctr_total_return, - tpcds_count(priv_hash(hash(sr_customer_sk)),1) AS presence -FROM main.store_returns, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE sr_customer_sk=asa_owner.asa_key AND sr_returned_date_sk = d_date_sk - AND d_year = 2000 - GROUP BY sr_customer_sk, - sr_store_sk -), customer_total_return AS ( -SELECT world, ctr_customer_sk, ctr_store_sk, ctr_total_return[world+1] AS ctr_total_return -FROM return_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT ctr1.world AS world, c_customer_id -FROM customer_total_return ctr1, - main.store, - main.customer -WHERE ctr1.ctr_total_return > - (SELECT avg(ctr_total_return)*1.2 - FROM customer_total_return ctr2 - WHERE ctr1.world=ctr2.world AND ctr1.ctr_store_sk = ctr2.ctr_store_sk) - AND s_store_sk = ctr1.ctr_store_sk - AND s_state = 'TN' - AND ctr1.ctr_customer_sk = c_customer_sk -QUALIFY row_number() OVER (PARTITION BY ctr1.world ORDER BY c_customer_id) <= 100 -ORDER BY ctr1.world, c_customer_id; diff --git a/benchmark/tpcds/reference_queries/q03.sql b/benchmark/tpcds/reference_queries/q03.sql deleted file mode 100644 index 06c8dada..00000000 --- a/benchmark/tpcds/reference_queries/q03.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT dt.d_year AS d_year, - item.i_brand_id AS brand_id, - item.i_brand AS brand, - tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS sum_agg, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.date_dim dt, - main.store_sales, - main.item, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND dt.d_date_sk = store_sales.ss_sold_date_sk - AND store_sales.ss_item_sk = item.i_item_sk - AND item.i_manufact_id = 128 - AND dt.d_moy=11 -GROUP BY dt.d_year, - main.item.i_brand, - main.item.i_brand_id -), answers AS ( - SELECT world, d_year, - brand_id, - brand, - sum_agg[world+1] AS sum_agg - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY d_year, sum_agg DESC, brand_id) <= 100 -ORDER BY world, d_year, sum_agg DESC, brand_id; diff --git a/benchmark/tpcds/reference_queries/q04.sql b/benchmark/tpcds/reference_queries/q04.sql deleted file mode 100644 index 68905a47..00000000 --- a/benchmark/tpcds/reference_queries/q04.sql +++ /dev/null @@ -1,135 +0,0 @@ --- Shared AS states per channel/customer/year; joins align the world index. -WITH year_total_stats AS ( -SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - c_preferred_cust_flag customer_preferred_cust_flag, - c_birth_country customer_birth_country, - c_login customer_login, - c_email_address customer_email_address, - d_year dyear, - tpcds_sum(priv_hash(hash(c_customer_sk)), ((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total, - 's' sale_type, - tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence -FROM main.customer, - main.store_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - GROUP BY c_customer_id, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_country, - c_login, - c_email_address, - d_year - -UNION ALL - SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - c_preferred_cust_flag customer_preferred_cust_flag, - c_birth_country customer_birth_country, - c_login customer_login, - c_email_address customer_email_address, - d_year dyear, - tpcds_sum(priv_hash(hash(c_customer_sk)), (((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2)) year_total, - 'c' sale_type, - tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence -FROM main.customer, - main.catalog_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = cs_bill_customer_sk - AND cs_sold_date_sk = d_date_sk - GROUP BY c_customer_id, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_country, - c_login, - c_email_address, - d_year - -UNION ALL - SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - c_preferred_cust_flag customer_preferred_cust_flag, - c_birth_country customer_birth_country, - c_login customer_login, - c_email_address customer_email_address, - d_year dyear, - tpcds_sum(priv_hash(hash(c_customer_sk)), (((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2)) year_total, - 'w' sale_type, - tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence -FROM main.customer, - main.web_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - GROUP BY c_customer_id, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_country, - c_login, - c_email_address, - d_year -), year_total AS ( -SELECT world, customer_id, customer_first_name, customer_last_name, customer_preferred_cust_flag, customer_birth_country, customer_login, customer_email_address, dyear, year_total[world+1] AS year_total, sale_type -FROM year_total_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT t_s_firstyear.world AS world, t_s_secyear.customer_id, - t_s_secyear.customer_first_name, - t_s_secyear.customer_last_name, - t_s_secyear.customer_preferred_cust_flag -FROM year_total t_s_firstyear, - year_total t_s_secyear, - year_total t_c_firstyear, - year_total t_c_secyear, - year_total t_w_firstyear, - year_total t_w_secyear -WHERE t_s_firstyear.world=t_s_secyear.world AND t_s_firstyear.world=t_c_firstyear.world AND t_s_firstyear.world=t_c_secyear.world AND t_s_firstyear.world=t_w_firstyear.world AND t_s_firstyear.world=t_w_secyear.world AND t_s_secyear.customer_id = t_s_firstyear.customer_id - AND t_s_firstyear.customer_id = t_c_secyear.customer_id - AND t_s_firstyear.customer_id = t_c_firstyear.customer_id - AND t_s_firstyear.customer_id = t_w_firstyear.customer_id - AND t_s_firstyear.customer_id = t_w_secyear.customer_id - AND t_s_firstyear.sale_type = 's' - AND t_c_firstyear.sale_type = 'c' - AND t_w_firstyear.sale_type = 'w' - AND t_s_secyear.sale_type = 's' - AND t_c_secyear.sale_type = 'c' - AND t_w_secyear.sale_type = 'w' - AND t_s_firstyear.dyear = 2001 - AND t_s_secyear.dyear = 2001+1 - AND t_c_firstyear.dyear = 2001 - AND t_c_secyear.dyear = 2001+1 - AND t_w_firstyear.dyear = 2001 - AND t_w_secyear.dyear = 2001+1 - AND t_s_firstyear.year_total > 0 - AND t_c_firstyear.year_total > 0 - AND t_w_firstyear.year_total > 0 - AND CASE - WHEN t_c_firstyear.year_total > 0 THEN t_c_secyear.year_total / t_c_firstyear.year_total - ELSE NULL - END > CASE - WHEN t_s_firstyear.year_total > 0 THEN t_s_secyear.year_total / t_s_firstyear.year_total - ELSE NULL - END - AND CASE - WHEN t_c_firstyear.year_total > 0 THEN t_c_secyear.year_total / t_c_firstyear.year_total - ELSE NULL - END > CASE - WHEN t_w_firstyear.year_total > 0 THEN t_w_secyear.year_total / t_w_firstyear.year_total - ELSE NULL - END -QUALIFY row_number() OVER (PARTITION BY t_s_firstyear.world ORDER BY t_s_secyear.customer_id NULLS FIRST, - t_s_secyear.customer_first_name NULLS FIRST, - t_s_secyear.customer_last_name NULLS FIRST, - t_s_secyear.customer_preferred_cust_flag NULLS FIRST) <= 100 -ORDER BY t_s_firstyear.world, t_s_secyear.customer_id NULLS FIRST, - t_s_secyear.customer_first_name NULLS FIRST, - t_s_secyear.customer_last_name NULLS FIRST, - t_s_secyear.customer_preferred_cust_flag NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q06.sql b/benchmark/tpcds/reference_queries/q06.sql deleted file mode 100644 index 25dfcac0..00000000 --- a/benchmark/tpcds/reference_queries/q06.sql +++ /dev/null @@ -1,36 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT a.ca_state AS state, - tpcds_count((priv_hash(hash(s.ss_customer_sk))), 1) AS cnt, - tpcds_count((priv_hash(hash(s.ss_customer_sk))), 1) AS presence - FROM main.customer_address a , - main.customer c , - main.store_sales s , - main.date_dim d , - main.item i, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE s.ss_customer_sk = asa_owner0.asa_key0 - AND a.ca_address_sk = c.c_current_addr_sk - AND c.c_customer_sk = s.ss_customer_sk - AND s.ss_sold_date_sk = d.d_date_sk - AND s.ss_item_sk = i.i_item_sk - AND d.d_month_seq = - (SELECT DISTINCT (d_month_seq) - FROM main.date_dim - WHERE d_year = 2001 - AND d_moy = 1 ) - AND i.i_current_price > 1.2 * - (SELECT avg(j.i_current_price) - FROM main.item j - WHERE j.i_category = i.i_category) -GROUP BY a.ca_state -), answers AS ( - SELECT world, state, - cnt[world+1] AS cnt - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -WHERE cnt >= 10 -QUALIFY row_number() OVER (PARTITION BY world ORDER BY cnt NULLS FIRST, state NULLS FIRST) <= 100 -ORDER BY world, cnt NULLS FIRST, state NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q07.sql b/benchmark/tpcds/reference_queries/q07.sql deleted file mode 100644 index 43f73f96..00000000 --- a/benchmark/tpcds/reference_queries/q07.sql +++ /dev/null @@ -1,38 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT i_item_id, - tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_quantity) AS agg1, - tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_list_price) AS agg2, - tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_coupon_amt) AS agg3, - tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_sales_price) AS agg4, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.store_sales, - main.customer_demographics, - main.date_dim, - main.item, - main.promotion, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND ss_sold_date_sk = d_date_sk - AND ss_item_sk = i_item_sk - AND ss_cdemo_sk = cd_demo_sk - AND ss_promo_sk = p_promo_sk - AND cd_gender = 'M' - AND cd_marital_status = 'S' - AND cd_education_status = 'College' - AND (p_channel_email = 'N' - OR p_channel_event = 'N') - AND d_year = 2000 -GROUP BY i_item_id -), answers AS ( - SELECT world, i_item_id, - agg1[world+1] AS agg1, - agg2[world+1] AS agg2, - agg3[world+1] AS agg3, - agg4[world+1] AS agg4 - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id) <= 100 -ORDER BY world, i_item_id; diff --git a/benchmark/tpcds/reference_queries/q09.sql b/benchmark/tpcds/reference_queries/q09.sql deleted file mode 100644 index 3cb2df1b..00000000 --- a/benchmark/tpcds/reference_queries/q09.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Each CASE selects between its world-specific aggregate results. -WITH stats AS (SELECT tpcds_count(CASE WHEN ss_quantity BETWEEN 1 AND 20 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS n1, -tpcds_avg(CASE WHEN ss_quantity BETWEEN 1 AND 20 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_ext_discount_amt) AS discount1, -tpcds_avg(CASE WHEN ss_quantity BETWEEN 1 AND 20 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_net_paid) AS paid1, -tpcds_count(CASE WHEN ss_quantity BETWEEN 21 AND 40 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS n2, -tpcds_avg(CASE WHEN ss_quantity BETWEEN 21 AND 40 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_ext_discount_amt) AS discount2, -tpcds_avg(CASE WHEN ss_quantity BETWEEN 21 AND 40 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_net_paid) AS paid2, -tpcds_count(CASE WHEN ss_quantity BETWEEN 41 AND 60 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS n3, -tpcds_avg(CASE WHEN ss_quantity BETWEEN 41 AND 60 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_ext_discount_amt) AS discount3, -tpcds_avg(CASE WHEN ss_quantity BETWEEN 41 AND 60 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_net_paid) AS paid3, -tpcds_count(CASE WHEN ss_quantity BETWEEN 61 AND 80 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS n4, -tpcds_avg(CASE WHEN ss_quantity BETWEEN 61 AND 80 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_ext_discount_amt) AS discount4, -tpcds_avg(CASE WHEN ss_quantity BETWEEN 61 AND 80 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_net_paid) AS paid4, -tpcds_count(CASE WHEN ss_quantity BETWEEN 81 AND 100 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS n5, -tpcds_avg(CASE WHEN ss_quantity BETWEEN 81 AND 100 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_ext_discount_amt) AS discount5, -tpcds_avg(CASE WHEN ss_quantity BETWEEN 81 AND 100 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,ss_net_paid) AS paid5 FROM main.store_sales JOIN main.customer ON ss_customer_sk=c_customer_sk) -SELECT world, CASE WHEN n1[world+1]>74129 THEN discount1[world+1] ELSE paid1[world+1] END AS bucket1, CASE WHEN n2[world+1]>122840 THEN discount2[world+1] ELSE paid2[world+1] END AS bucket2, CASE WHEN n3[world+1]>56580 THEN discount3[world+1] ELSE paid3[world+1] END AS bucket3, CASE WHEN n4[world+1]>10097 THEN discount4[world+1] ELSE paid4[world+1] END AS bucket4, CASE WHEN n5[world+1]>165306 THEN discount5[world+1] ELSE paid5[world+1] END AS bucket5 FROM stats CROSS JOIN range(64) worlds(world) CROSS JOIN main.reason WHERE r_reason_sk=1 ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q10.sql b/benchmark/tpcds/reference_queries/q10.sql deleted file mode 100644 index 590bc867..00000000 --- a/benchmark/tpcds/reference_queries/q10.sql +++ /dev/null @@ -1,59 +0,0 @@ --- Same-owner EXISTS is static after entity inclusion; shipping-customer EXISTS needs the bill-owner mask. -WITH catalog_presence AS ( -SELECT cs_ship_customer_sk,bit_or(priv_hash(hash(cs_bill_customer_sk))) AS membership - FROM main.catalog_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_moy BETWEEN 1 AND 1+3 -GROUP BY cs_ship_customer_sk -), stats AS (SELECT cd_gender, -cd_marital_status, -cd_education_status, -cd_purchase_estimate, -cd_credit_rating, -cd_dep_count, -cd_dep_employed_count, -cd_dep_college_count, -tpcds_count((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_moy BETWEEN 1 AND 1+3) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),1) AS n -FROM main.customer c CROSS JOIN - main.customer_address ca CROSS JOIN - main.customer_demographics LEFT JOIN catalog_presence ON c.c_customer_sk=cs_ship_customer_sk -WHERE c.c_current_addr_sk = ca.ca_address_sk - AND ca_county IN ('Rush County', - 'Toole County', - 'Jefferson County', - 'Dona Ana County', - 'La Porte County') - AND cd_demo_sk = c.c_current_cdemo_sk - -AND EXISTS(SELECT * - FROM main.store_sales, - main.date_dim - WHERE c.c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_moy BETWEEN 1 AND 1+3) -GROUP BY cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count -), answers AS (SELECT world, cd_gender, cd_marital_status, cd_education_status, n[world+1] AS cnt1, cd_purchase_estimate, n[world+1] AS cnt2, cd_credit_rating, n[world+1] AS cnt3, cd_dep_count, n[world+1] AS cnt4, cd_dep_employed_count, n[world+1] AS cnt5, cd_dep_college_count, n[world+1] AS cnt6 FROM stats CROSS JOIN range(64) worlds(world) WHERE n[world+1]>0) -SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating, - cd_dep_count, - cd_dep_employed_count, - cd_dep_college_count)<=100 ORDER BY world,cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating, - cd_dep_count, - cd_dep_employed_count, - cd_dep_college_count; diff --git a/benchmark/tpcds/reference_queries/q11.sql b/benchmark/tpcds/reference_queries/q11.sql deleted file mode 100644 index 07e84dc3..00000000 --- a/benchmark/tpcds/reference_queries/q11.sql +++ /dev/null @@ -1,93 +0,0 @@ --- Shared AS states per channel/customer/year; joins align the world index. -WITH year_total_stats AS ( -SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - c_preferred_cust_flag customer_preferred_cust_flag, - c_birth_country customer_birth_country, - c_login customer_login, - c_email_address customer_email_address, - d_year dyear, - tpcds_sum(priv_hash(hash(c_customer_sk)), ss_ext_list_price-ss_ext_discount_amt) year_total, - 's' sale_type, - tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence -FROM main.customer, - main.store_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - GROUP BY c_customer_id, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_country, - c_login, - c_email_address, - d_year - -UNION ALL - SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - c_preferred_cust_flag customer_preferred_cust_flag, - c_birth_country customer_birth_country, - c_login customer_login, - c_email_address customer_email_address, - d_year dyear, - tpcds_sum(priv_hash(hash(c_customer_sk)), ws_ext_list_price-ws_ext_discount_amt) year_total, - 'w' sale_type, - tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence -FROM main.customer, - main.web_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - GROUP BY c_customer_id, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_country, - c_login, - c_email_address, - d_year -), year_total AS ( -SELECT world, customer_id, customer_first_name, customer_last_name, customer_preferred_cust_flag, customer_birth_country, customer_login, customer_email_address, dyear, year_total[world+1] AS year_total, sale_type -FROM year_total_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT t_s_firstyear.world AS world, t_s_secyear.customer_id, - t_s_secyear.customer_first_name, - t_s_secyear.customer_last_name, - t_s_secyear.customer_preferred_cust_flag -FROM year_total t_s_firstyear, - year_total t_s_secyear, - year_total t_w_firstyear, - year_total t_w_secyear -WHERE t_s_firstyear.world=t_s_secyear.world AND t_s_firstyear.world=t_w_firstyear.world AND t_s_firstyear.world=t_w_secyear.world AND t_s_secyear.customer_id = t_s_firstyear.customer_id - AND t_s_firstyear.customer_id = t_w_secyear.customer_id - AND t_s_firstyear.customer_id = t_w_firstyear.customer_id - AND t_s_firstyear.sale_type = 's' - AND t_w_firstyear.sale_type = 'w' - AND t_s_secyear.sale_type = 's' - AND t_w_secyear.sale_type = 'w' - AND t_s_firstyear.dyear = 2001 - AND t_s_secyear.dyear = 2001+1 - AND t_w_firstyear.dyear = 2001 - AND t_w_secyear.dyear = 2001+1 - AND t_s_firstyear.year_total > 0 - AND t_w_firstyear.year_total > 0 - AND CASE - WHEN t_w_firstyear.year_total > 0 THEN (t_w_secyear.year_total*1.0000) / t_w_firstyear.year_total - ELSE 0.0 - END > CASE - WHEN t_s_firstyear.year_total > 0 THEN (t_s_secyear.year_total*1.0000) / t_s_firstyear.year_total - ELSE 0.0 - END -QUALIFY row_number() OVER (PARTITION BY t_s_firstyear.world ORDER BY t_s_secyear.customer_id NULLS FIRST, - t_s_secyear.customer_first_name NULLS FIRST, - t_s_secyear.customer_last_name NULLS FIRST, - t_s_secyear.customer_preferred_cust_flag NULLS FIRST) <= 100 -ORDER BY t_s_firstyear.world, t_s_secyear.customer_id NULLS FIRST, - t_s_secyear.customer_first_name NULLS FIRST, - t_s_secyear.customer_last_name NULLS FIRST, - t_s_secyear.customer_preferred_cust_flag NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q13.sql b/benchmark/tpcds/reference_queries/q13.sql deleted file mode 100644 index 35325b68..00000000 --- a/benchmark/tpcds/reference_queries/q13.sql +++ /dev/null @@ -1,55 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_quantity) AS avg1, - tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS avg2, - tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_ext_wholesale_cost) AS avg3, - tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_wholesale_cost) AS total, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.store_sales , - main.store , - main.customer_demographics , - main.household_demographics , - main.customer_address , - main.date_dim, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND s_store_sk = ss_store_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2001 and((ss_hdemo_sk=hd_demo_sk - AND cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'M' - AND cd_education_status = 'Advanced Degree' - AND ss_sales_price BETWEEN 100.00 AND 150.00 - AND hd_dep_count = 3) - OR (ss_hdemo_sk=hd_demo_sk - AND cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'S' - AND cd_education_status = 'College' - AND ss_sales_price BETWEEN 50.00 AND 100.00 - AND hd_dep_count = 1 ) - OR (ss_hdemo_sk=hd_demo_sk - AND cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'W' - AND cd_education_status = '2 yr Degree' - AND ss_sales_price BETWEEN 150.00 AND 200.00 - AND hd_dep_count = 1)) and((ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('TX', 'OH', 'TX') - AND ss_net_profit BETWEEN 100 AND 200) - OR (ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('OR', 'NM', 'KY') - AND ss_net_profit BETWEEN 150 AND 300) - OR (ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('VA', 'TX', 'MS') - AND ss_net_profit BETWEEN 50 AND 250)) -), answers AS ( - SELECT world, avg1[world+1] AS avg1, - avg2[world+1] AS avg2, - avg3[world+1] AS avg3, - total[world+1] AS total - FROM stats CROSS JOIN range(64) worlds(world) -) -SELECT * FROM answers -ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q15.sql b/benchmark/tpcds/reference_queries/q15.sql deleted file mode 100644 index 07aa9526..00000000 --- a/benchmark/tpcds/reference_queries/q15.sql +++ /dev/null @@ -1,39 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT ca_zip, - tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), cs_sales_price) AS total, - tpcds_count((priv_hash(hash(cs_bill_customer_sk))), 1) AS presence - FROM main.catalog_sales, - main.customer, - main.customer_address, - main.date_dim, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE cs_bill_customer_sk = asa_owner0.asa_key0 - AND cs_bill_customer_sk = c_customer_sk - AND c_current_addr_sk = ca_address_sk - AND (SUBSTRING(ca_zip, 1, 5) IN ('85669', - '86197', - '88274', - '83405', - '86475', - '85392', - '85460', - '80348', - '81792') - OR ca_state IN ('CA', - 'WA', - 'GA') - OR cs_sales_price > 500) - AND cs_sold_date_sk = d_date_sk - AND d_qoy = 2 - AND d_year = 2001 -GROUP BY ca_zip -), answers AS ( - SELECT world, ca_zip, - total[world+1] AS total - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY ca_zip NULLS FIRST) <= 100 -ORDER BY world, ca_zip NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q16.sql b/benchmark/tpcds/reference_queries/q16.sql deleted file mode 100644 index 9d291fd6..00000000 --- a/benchmark/tpcds/reference_queries/q16.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Order-level existence masks are combined before AS; DISTINCT unions memberships by order. -WITH warehouses AS (SELECT cs_order_number AS order_number,cs_warehouse_sk AS warehouse,bit_or(priv_hash(hash(cs_bill_customer_sk))) AS membership FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk GROUP BY cs_order_number,cs_warehouse_sk), -other_warehouses AS (SELECT a.order_number,a.warehouse,bit_or(b.membership) AS membership FROM warehouses a JOIN warehouses b ON a.order_number=b.order_number AND a.warehouse<>b.warehouse GROUP BY a.order_number,a.warehouse), -return_orders AS (SELECT cr_order_number AS order_number,bit_or(priv_hash(hash(cr_returning_customer_sk))) AS membership FROM main.catalog_returns JOIN main.customer ON cr_returning_customer_sk=c_customer_sk GROUP BY cr_order_number), -input AS (SELECT cs_order_number AS order_number,cs_ext_ship_cost AS shipping,cs_net_profit AS profit,(priv_hash(hash(cs_bill_customer_sk)) & other.membership & ~coalesce(ret.membership,0::UBIGINT)) AS membership -FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk JOIN main.date_dim ON cs_ship_date_sk=d_date_sk JOIN main.customer_address ON cs_ship_addr_sk=ca_address_sk JOIN main.call_center ON cs_call_center_sk=cc_call_center_sk JOIN other_warehouses other ON cs_order_number=other.order_number AND cs_warehouse_sk=other.warehouse LEFT JOIN return_orders ret ON cs_order_number=ret.order_number WHERE d_date BETWEEN DATE '2002-02-01' AND DATE '2002-04-02' AND ca_state='GA' AND cc_county='Williamson County'), -totals AS (SELECT tpcds_sum(membership,shipping) AS shipping,tpcds_sum(membership,profit) AS profit FROM input), -unique_orders AS (SELECT order_number,bit_or(membership) AS membership FROM input GROUP BY order_number), -counts AS (SELECT tpcds_count(membership,1) AS n FROM unique_orders) -SELECT world,n[world+1] AS order_count,shipping[world+1] AS total_shipping_cost,profit[world+1] AS total_net_profit FROM totals CROSS JOIN counts CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q17.sql b/benchmark/tpcds/reference_queries/q17.sql deleted file mode 100644 index 6910c700..00000000 --- a/benchmark/tpcds/reference_queries/q17.sql +++ /dev/null @@ -1,67 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT i_item_id, - i_item_desc, - s_state, - tpcds_count((priv_hash(hash(ss_customer_sk))), ss_quantity) AS v0_count, - tpcds_avg((priv_hash(hash(ss_customer_sk))), ss_quantity) AS v0_avg, - tpcds_stddev_samp((priv_hash(hash(ss_customer_sk))), ss_quantity) AS v0_stddev_samp, - tpcds_count((priv_hash(hash(ss_customer_sk))), sr_return_quantity) AS v1_count, - tpcds_avg((priv_hash(hash(ss_customer_sk))), sr_return_quantity) AS v1_avg, - tpcds_stddev_samp((priv_hash(hash(ss_customer_sk))), sr_return_quantity) AS v1_stddev_samp, - tpcds_count((priv_hash(hash(ss_customer_sk))), cs_quantity) AS v2_count, - tpcds_avg((priv_hash(hash(ss_customer_sk))), cs_quantity) AS v2_avg, - tpcds_stddev_samp((priv_hash(hash(ss_customer_sk))), cs_quantity) AS v2_stddev_samp, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.store_sales, - main.store_returns, - main.catalog_sales, - main.date_dim d1, - main.date_dim d2, - main.date_dim d3, - main.store, - main.item, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND d1.d_quarter_name = '2001Q1' - AND d1.d_date_sk = ss_sold_date_sk - AND i_item_sk = ss_item_sk - AND s_store_sk = ss_store_sk - AND ss_customer_sk = sr_customer_sk - AND ss_item_sk = sr_item_sk - AND ss_ticket_number = sr_ticket_number - AND sr_returned_date_sk = d2.d_date_sk - AND d2.d_quarter_name IN ('2001Q1', - '2001Q2', - '2001Q3') - AND sr_customer_sk = cs_bill_customer_sk - AND sr_item_sk = cs_item_sk - AND cs_sold_date_sk = d3.d_date_sk - AND d3.d_quarter_name IN ('2001Q1', - '2001Q2', - '2001Q3') -GROUP BY i_item_id, - i_item_desc, - s_state -), answers AS ( - SELECT world, i_item_id, - i_item_desc, - s_state, - v0_count[world+1] AS store_sales_quantitycount, - v0_avg[world+1] AS store_sales_quantityave, - v0_stddev_samp[world+1] AS store_sales_quantitystdev, - v0_stddev_samp[world+1]/v0_avg[world+1] AS store_sales_quantitycov, - v1_count[world+1] AS store_returns_quantitycount, - v1_avg[world+1] AS store_returns_quantityave, - v1_stddev_samp[world+1] AS store_returns_quantitystdev, - v1_stddev_samp[world+1]/v1_avg[world+1] AS store_returns_quantitycov, - v2_count[world+1] AS catalog_sales_quantitycount, - v2_avg[world+1] AS catalog_sales_quantityave, - v2_stddev_samp[world+1] AS catalog_sales_quantitystdev, - v2_stddev_samp[world+1]/v2_avg[world+1] AS catalog_sales_quantitycov - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id NULLS FIRST, i_item_desc NULLS FIRST, s_state NULLS FIRST) <= 100 -ORDER BY world, i_item_id NULLS FIRST, i_item_desc NULLS FIRST, s_state NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q18.sql b/benchmark/tpcds/reference_queries/q18.sql deleted file mode 100644 index d00e2c97..00000000 --- a/benchmark/tpcds/reference_queries/q18.sql +++ /dev/null @@ -1,68 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT i_item_id, - ca_country, - ca_state, - ca_county, - tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cs_quantity AS DECIMAL(12,2))) AS agg1, - tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cs_list_price AS DECIMAL(12,2))) AS agg2, - tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cs_coupon_amt AS DECIMAL(12,2))) AS agg3, - tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cs_sales_price AS DECIMAL(12,2))) AS agg4, - tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cs_net_profit AS DECIMAL(12,2))) AS agg5, - tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(c_birth_year AS DECIMAL(12,2))) AS agg6, - tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), CAST(cd1.cd_dep_count AS DECIMAL(12,2))) AS agg7, - tpcds_count((priv_hash(hash(cs_bill_customer_sk))), 1) AS presence, - GROUPING(i_item_id, ca_country, ca_state, ca_county) AS grouping_level - FROM main.catalog_sales, - main.customer_demographics cd1, - main.customer_demographics cd2, - main.customer, - main.customer_address, - main.date_dim, - main.item, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE cs_bill_customer_sk = asa_owner0.asa_key0 - AND cs_sold_date_sk = d_date_sk - AND cs_item_sk = i_item_sk - AND cs_bill_cdemo_sk = cd1.cd_demo_sk - AND cs_bill_customer_sk = c_customer_sk - AND cd1.cd_gender = 'F' - AND cd1.cd_education_status = 'Unknown' - AND c_current_cdemo_sk = cd2.cd_demo_sk - AND c_current_addr_sk = ca_address_sk - AND c_birth_month IN (1, - 6, - 8, - 9, - 12, - 2) - AND d_year = 1998 - AND ca_state IN ('MS', - 'IN', - 'ND', - 'OK', - 'NM', - 'VA', - 'MS') -GROUP BY ROLLUP (i_item_id, - ca_country, - ca_state, - ca_county) -), answers AS ( - SELECT world, i_item_id, - ca_country, - ca_state, - ca_county, - agg1[world+1] AS agg1, - agg2[world+1] AS agg2, - agg3[world+1] AS agg3, - agg4[world+1] AS agg4, - agg5[world+1] AS agg5, - agg6[world+1] AS agg6, - agg7[world+1] AS agg7 - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 OR grouping_level = 15 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY ca_country NULLS FIRST, ca_state NULLS FIRST, ca_county NULLS FIRST, i_item_id NULLS FIRST) <= 100 -ORDER BY world, ca_country NULLS FIRST, ca_state NULLS FIRST, ca_county NULLS FIRST, i_item_id NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q19.sql b/benchmark/tpcds/reference_queries/q19.sql deleted file mode 100644 index beecad17..00000000 --- a/benchmark/tpcds/reference_queries/q19.sql +++ /dev/null @@ -1,41 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT i_brand_id AS brand_id, - i_brand AS brand, - i_manufact_id, - i_manufact, - tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS ext_price, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.date_dim, - main.store_sales, - main.item, - main.customer, - main.customer_address, - main.store, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND d_date_sk = ss_sold_date_sk - AND ss_item_sk = i_item_sk - AND i_manager_id=8 - AND d_moy=11 - AND d_year=1998 - AND ss_customer_sk = c_customer_sk - AND c_current_addr_sk = ca_address_sk - AND SUBSTRING(ca_zip, 1, 5) <> SUBSTRING(s_zip, 1, 5) - AND ss_store_sk = s_store_sk -GROUP BY i_brand, - i_brand_id, - i_manufact_id, - i_manufact -), answers AS ( - SELECT world, brand_id, - brand, - i_manufact_id, - i_manufact, - ext_price[world+1] AS ext_price - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY ext_price DESC, brand, brand_id, i_manufact_id, i_manufact) <= 100 -ORDER BY world, ext_price DESC, brand, brand_id, i_manufact_id, i_manufact; diff --git a/benchmark/tpcds/reference_queries/q23.sql b/benchmark/tpcds/reference_queries/q23.sql deleted file mode 100644 index c863cb8f..00000000 --- a/benchmark/tpcds/reference_queries/q23.sql +++ /dev/null @@ -1,96 +0,0 @@ --- Item-date multiplicities are preserved; global maxima and customer cutoffs remain world-specific. -WITH frequent_stats AS ( -SELECT itemdesc, - i_item_sk item_sk, - d_date solddate, - tpcds_count(priv_hash(hash(ss_customer_sk)),1) cnt, tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence -FROM main.store_sales, - main.date_dim, - (SELECT SUBSTRING(i_item_desc, 1, 30) itemdesc, - * - FROM main.item) sq1, (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk=asa_owner0.asa_key0 AND ss_sold_date_sk = d_date_sk - AND ss_item_sk = i_item_sk - AND d_year IN (2000, - 2000+1, - 2000+2, - 2000+3) - GROUP BY itemdesc, - i_item_sk, - d_date - -), -frequent_ss_items AS (SELECT itemdesc,item_sk,solddate,priv_select(18446744073709551615::UBIGINT,list_transform(cnt,lambda x: x>4)) AS membership FROM frequent_stats), -customer_period_stats AS ( -SELECT c_customer_sk, - tpcds_sum(priv_hash(hash(c_customer_sk)), ss_quantity*ss_sales_price) csales, - tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence -FROM main.store_sales, - main.customer, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE c_customer_sk=asa_owner.asa_key AND ss_customer_sk = c_customer_sk - AND ss_sold_date_sk = d_date_sk - AND d_year IN (2000, - 2000+1, - 2000+2, - 2000+3) - GROUP BY c_customer_sk -), -customer_period AS ( -SELECT world, c_customer_sk, csales[world+1] AS csales -FROM customer_period_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -max_sales AS (SELECT world,max(csales) AS maximum FROM customer_period GROUP BY world), -max_array AS (SELECT list(maximum ORDER BY w.world) AS thresholds FROM range(64) w(world) LEFT JOIN max_sales USING(world)), -best_stats AS (SELECT c_customer_sk,tpcds_sum(priv_hash(hash(c_customer_sk)),ss_quantity*ss_sales_price) AS ssales FROM main.store_sales JOIN main.customer ON ss_customer_sk=c_customer_sk GROUP BY c_customer_sk), -best_ss_customer AS (SELECT c_customer_sk,priv_select(priv_hash(hash(c_customer_sk)),list_transform(list_zip(ssales,thresholds),lambda x: x[1]>0.5*x[2])) AS membership FROM best_stats CROSS JOIN max_array), -channel0_stats AS ( -SELECT c_last_name, - c_first_name, - tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & frequent_ss_items.membership & best_ss_customer.membership), cs_quantity*cs_list_price) sales, tpcds_count((priv_hash(hash(cs_bill_customer_sk)) & frequent_ss_items.membership & best_ss_customer.membership),1) AS presence -FROM main.catalog_sales, - main.customer, - main.date_dim, - frequent_ss_items, - best_ss_customer -WHERE d_year = 2000 - AND d_moy = 2 - AND cs_sold_date_sk = d_date_sk - AND cs_item_sk = item_sk - AND cs_bill_customer_sk = best_ss_customer.c_customer_sk - AND cs_bill_customer_sk = customer.c_customer_sk - GROUP BY c_last_name, - c_first_name - -), -channel0 AS ( -SELECT world, c_last_name, c_first_name, sales[world+1] AS sales -FROM channel0_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -channel1_stats AS ( - SELECT c_last_name, - c_first_name, - tpcds_sum((priv_hash(hash(ws_bill_customer_sk)) & frequent_ss_items.membership & best_ss_customer.membership), ws_quantity*ws_list_price) sales, tpcds_count((priv_hash(hash(ws_bill_customer_sk)) & frequent_ss_items.membership & best_ss_customer.membership),1) AS presence -FROM main.web_sales, - main.customer, - main.date_dim, - frequent_ss_items, - best_ss_customer -WHERE d_year = 2000 - AND d_moy = 2 - AND ws_sold_date_sk = d_date_sk - AND ws_item_sk = item_sk - AND ws_bill_customer_sk = best_ss_customer.c_customer_sk - AND ws_bill_customer_sk = customer.c_customer_sk - GROUP BY c_last_name, - c_first_name -), -channel1 AS ( -SELECT world, c_last_name, c_first_name, sales[world+1] AS sales -FROM channel1_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -answers AS (SELECT * FROM channel0 UNION ALL SELECT * FROM channel1) -SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY c_last_name NULLS FIRST,c_first_name NULLS FIRST,sales NULLS FIRST)<=100 ORDER BY world,c_last_name NULLS FIRST,c_first_name NULLS FIRST,sales NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q24.sql b/benchmark/tpcds/reference_queries/q24.sql deleted file mode 100644 index d58fe980..00000000 --- a/benchmark/tpcds/reference_queries/q24.sql +++ /dev/null @@ -1,58 +0,0 @@ --- Sale and return ownership masks are intersected before the shared inner aggregation. -WITH ssales_stats AS ( -SELECT c_last_name, - c_first_name, - s_store_name, - ca_state, - s_state, - i_color, - i_current_price, - i_manager_id, - i_units, - i_size, - tpcds_sum((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk))), ss_net_paid) netpaid, tpcds_count((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk))),1) AS presence -FROM main.store_sales, - main.store_returns, - main.store, - main.item, - main.customer, - main.customer_address, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_return_owner -WHERE sr_customer_sk=asa_return_owner.asa_key AND ss_ticket_number = sr_ticket_number - AND ss_item_sk = sr_item_sk - AND ss_customer_sk = c_customer_sk - AND ss_item_sk = i_item_sk - AND ss_store_sk = s_store_sk - AND c_current_addr_sk = ca_address_sk - AND c_birth_country <> upper(ca_country) - AND s_zip = ca_zip - AND s_market_id=8 - GROUP BY c_last_name, - c_first_name, - s_store_name, - ca_state, - s_state, - i_color, - i_current_price, - i_manager_id, - i_units, - i_size -), ssales AS ( -SELECT world, c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size, netpaid[world+1] AS netpaid -FROM ssales_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT outer_ss.world AS world, c_last_name, - c_first_name, - s_store_name, - sum(netpaid) paid -FROM ssales outer_ss -WHERE i_color = 'peach' -GROUP BY outer_ss.world, c_last_name, - c_first_name, - s_store_name -HAVING sum(netpaid) > - (SELECT 0.05*avg(netpaid) - FROM ssales inner_ss WHERE inner_ss.world=outer_ss.world) -ORDER BY outer_ss.world, c_last_name, - c_first_name, - s_store_name; diff --git a/benchmark/tpcds/reference_queries/q25.sql b/benchmark/tpcds/reference_queries/q25.sql deleted file mode 100644 index 963bf844..00000000 --- a/benchmark/tpcds/reference_queries/q25.sql +++ /dev/null @@ -1,54 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT i_item_id, - i_item_desc, - s_store_id, - s_store_name, - tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_net_profit) AS store_sales_profit, - tpcds_sum((priv_hash(hash(ss_customer_sk))), sr_net_loss) AS store_returns_loss, - tpcds_sum((priv_hash(hash(ss_customer_sk))), cs_net_profit) AS catalog_sales_profit, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.store_sales , - main.store_returns , - main.catalog_sales , - main.date_dim d1 , - main.date_dim d2 , - main.date_dim d3 , - main.store , - main.item, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND d1.d_moy = 4 - AND d1.d_year = 2001 - AND d1.d_date_sk = ss_sold_date_sk - AND i_item_sk = ss_item_sk - AND s_store_sk = ss_store_sk - AND ss_customer_sk = sr_customer_sk - AND ss_item_sk = sr_item_sk - AND ss_ticket_number = sr_ticket_number - AND sr_returned_date_sk = d2.d_date_sk - AND d2.d_moy BETWEEN 4 AND 10 - AND d2.d_year = 2001 - AND sr_customer_sk = cs_bill_customer_sk - AND sr_item_sk = cs_item_sk - AND cs_sold_date_sk = d3.d_date_sk - AND d3.d_moy BETWEEN 4 AND 10 - AND d3.d_year = 2001 -GROUP BY i_item_id , - i_item_desc , - s_store_id , - s_store_name -), answers AS ( - SELECT world, i_item_id, - i_item_desc, - s_store_id, - s_store_name, - store_sales_profit[world+1] AS store_sales_profit, - store_returns_loss[world+1] AS store_returns_loss, - catalog_sales_profit[world+1] AS catalog_sales_profit - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id, i_item_desc, s_store_id, s_store_name) <= 100 -ORDER BY world, i_item_id, i_item_desc, s_store_id, s_store_name; diff --git a/benchmark/tpcds/reference_queries/q26.sql b/benchmark/tpcds/reference_queries/q26.sql deleted file mode 100644 index 5c8055f1..00000000 --- a/benchmark/tpcds/reference_queries/q26.sql +++ /dev/null @@ -1,38 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT i_item_id, - tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), cs_quantity) AS agg1, - tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), cs_list_price) AS agg2, - tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), cs_coupon_amt) AS agg3, - tpcds_avg((priv_hash(hash(cs_bill_customer_sk))), cs_sales_price) AS agg4, - tpcds_count((priv_hash(hash(cs_bill_customer_sk))), 1) AS presence - FROM main.catalog_sales, - main.customer_demographics, - main.date_dim, - main.item, - main.promotion, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE cs_bill_customer_sk = asa_owner0.asa_key0 - AND cs_sold_date_sk = d_date_sk - AND cs_item_sk = i_item_sk - AND cs_bill_cdemo_sk = cd_demo_sk - AND cs_promo_sk = p_promo_sk - AND cd_gender = 'M' - AND cd_marital_status = 'S' - AND cd_education_status = 'College' - AND (p_channel_email = 'N' - OR p_channel_event = 'N') - AND d_year = 2000 -GROUP BY i_item_id -), answers AS ( - SELECT world, i_item_id, - agg1[world+1] AS agg1, - agg2[world+1] AS agg2, - agg3[world+1] AS agg3, - agg4[world+1] AS agg4 - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id) <= 100 -ORDER BY world, i_item_id; diff --git a/benchmark/tpcds/reference_queries/q27.sql b/benchmark/tpcds/reference_queries/q27.sql deleted file mode 100644 index 45c51b9a..00000000 --- a/benchmark/tpcds/reference_queries/q27.sql +++ /dev/null @@ -1,25 +0,0 @@ --- Explicit grouping sets reproduce all three original UNION ALL branches, including empty grand totals. -WITH input AS ( -SELECT priv_hash(hash(ss_customer_sk)) AS membership, i_item_id, - s_state, - 0 AS g_state, - ss_quantity agg1, - ss_list_price agg2, - ss_coupon_amt agg3, - ss_sales_price agg4 - FROM main.store_sales, - main.customer_demographics, - main.date_dim, - main.store, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk = d_date_sk - AND ss_item_sk = i_item_sk - AND ss_store_sk = s_store_sk - AND ss_cdemo_sk = cd_demo_sk - AND cd_gender = 'M' - AND cd_marital_status = 'S' - AND cd_education_status = 'College' - AND d_year = 2002 - AND s_state = 'TN' -), stats AS (SELECT i_item_id, s_state, grouping(s_state) AS g_state, grouping(i_item_id) AS grand_total, tpcds_avg(membership,agg1) AS agg1, tpcds_avg(membership,agg2) AS agg2, tpcds_avg(membership,agg3) AS agg3, tpcds_avg(membership,agg4) AS agg4, tpcds_count(membership,1) AS presence FROM input GROUP BY GROUPING SETS ((i_item_id,s_state),(i_item_id),())), answers AS (SELECT world, i_item_id, s_state, g_state, agg1[world+1] AS agg1, agg2[world+1] AS agg2, agg3[world+1] AS agg3, agg4[world+1] AS agg4 FROM stats CROSS JOIN range(64) worlds(world) WHERE presence[world+1]>0 OR grand_total=1) -SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id NULLS FIRST,s_state NULLS FIRST)<=100 ORDER BY world,i_item_id NULLS FIRST,s_state NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q28.sql b/benchmark/tpcds/reference_queries/q28.sql deleted file mode 100644 index 6c7dd994..00000000 --- a/benchmark/tpcds/reference_queries/q28.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Per-value membership union implements COUNT(DISTINCT) without duplicating base rows by world. -WITH input AS (SELECT CASE WHEN (ss_quantity BETWEEN 0 AND 5 AND (ss_list_price BETWEEN 8 AND 18 OR ss_coupon_amt BETWEEN 459 AND 1459 OR ss_wholesale_cost BETWEEN 57 AND 77)) THEN 1 WHEN (ss_quantity BETWEEN 6 AND 10 AND (ss_list_price BETWEEN 90 AND 100 OR ss_coupon_amt BETWEEN 2323 AND 3323 OR ss_wholesale_cost BETWEEN 31 AND 51)) THEN 2 WHEN (ss_quantity BETWEEN 11 AND 15 AND (ss_list_price BETWEEN 142 AND 152 OR ss_coupon_amt BETWEEN 12214 AND 13214 OR ss_wholesale_cost BETWEEN 79 AND 99)) THEN 3 WHEN (ss_quantity BETWEEN 16 AND 20 AND (ss_list_price BETWEEN 135 AND 145 OR ss_coupon_amt BETWEEN 6071 AND 7071 OR ss_wholesale_cost BETWEEN 38 AND 58)) THEN 4 WHEN (ss_quantity BETWEEN 21 AND 25 AND (ss_list_price BETWEEN 122 AND 132 OR ss_coupon_amt BETWEEN 836 AND 1836 OR ss_wholesale_cost BETWEEN 17 AND 37)) THEN 5 WHEN (ss_quantity BETWEEN 26 AND 30 AND (ss_list_price BETWEEN 154 AND 164 OR ss_coupon_amt BETWEEN 7326 AND 8326 OR ss_wholesale_cost BETWEEN 7 AND 27)) THEN 6 END AS bucket, ss_list_price AS price, priv_hash(hash(ss_customer_sk)) AS membership -FROM main.store_sales JOIN main.customer ON ss_customer_sk=c_customer_sk WHERE (ss_quantity BETWEEN 0 AND 5 AND (ss_list_price BETWEEN 8 AND 18 OR ss_coupon_amt BETWEEN 459 AND 1459 OR ss_wholesale_cost BETWEEN 57 AND 77)) OR (ss_quantity BETWEEN 6 AND 10 AND (ss_list_price BETWEEN 90 AND 100 OR ss_coupon_amt BETWEEN 2323 AND 3323 OR ss_wholesale_cost BETWEEN 31 AND 51)) OR (ss_quantity BETWEEN 11 AND 15 AND (ss_list_price BETWEEN 142 AND 152 OR ss_coupon_amt BETWEEN 12214 AND 13214 OR ss_wholesale_cost BETWEEN 79 AND 99)) OR (ss_quantity BETWEEN 16 AND 20 AND (ss_list_price BETWEEN 135 AND 145 OR ss_coupon_amt BETWEEN 6071 AND 7071 OR ss_wholesale_cost BETWEEN 38 AND 58)) OR (ss_quantity BETWEEN 21 AND 25 AND (ss_list_price BETWEEN 122 AND 132 OR ss_coupon_amt BETWEEN 836 AND 1836 OR ss_wholesale_cost BETWEEN 17 AND 37)) OR (ss_quantity BETWEEN 26 AND 30 AND (ss_list_price BETWEEN 154 AND 164 OR ss_coupon_amt BETWEEN 7326 AND 8326 OR ss_wholesale_cost BETWEEN 7 AND 27))), -stats AS (SELECT bucket,tpcds_avg(membership,price) AS average,tpcds_count(membership,price) AS n FROM input GROUP BY bucket), -unique_prices AS (SELECT bucket,price,bit_or(membership) AS membership FROM input WHERE price IS NOT NULL GROUP BY bucket,price), -unique_counts AS (SELECT bucket,tpcds_count(membership,1) AS n FROM unique_prices GROUP BY bucket), -expanded AS (SELECT world,bucket,average[world+1] AS average,coalesce(stats.n[world+1],0) AS n,coalesce(unique_counts.n[world+1],0) AS nd -FROM (VALUES(1),(2),(3),(4),(5),(6)) b(bucket) CROSS JOIN range(64) worlds(world) LEFT JOIN stats USING(bucket) LEFT JOIN unique_counts USING(bucket)) -SELECT world,max(average) FILTER (WHERE bucket=1) AS b1_lp, max(n) FILTER (WHERE bucket=1) AS b1_cnt, max(nd) FILTER (WHERE bucket=1) AS b1_cntd, max(average) FILTER (WHERE bucket=2) AS b2_lp, max(n) FILTER (WHERE bucket=2) AS b2_cnt, max(nd) FILTER (WHERE bucket=2) AS b2_cntd, max(average) FILTER (WHERE bucket=3) AS b3_lp, max(n) FILTER (WHERE bucket=3) AS b3_cnt, max(nd) FILTER (WHERE bucket=3) AS b3_cntd, max(average) FILTER (WHERE bucket=4) AS b4_lp, max(n) FILTER (WHERE bucket=4) AS b4_cnt, max(nd) FILTER (WHERE bucket=4) AS b4_cntd, max(average) FILTER (WHERE bucket=5) AS b5_lp, max(n) FILTER (WHERE bucket=5) AS b5_cnt, max(nd) FILTER (WHERE bucket=5) AS b5_cntd, max(average) FILTER (WHERE bucket=6) AS b6_lp, max(n) FILTER (WHERE bucket=6) AS b6_cnt, max(nd) FILTER (WHERE bucket=6) AS b6_cntd FROM expanded GROUP BY world ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q29.sql b/benchmark/tpcds/reference_queries/q29.sql deleted file mode 100644 index 17f70177..00000000 --- a/benchmark/tpcds/reference_queries/q29.sql +++ /dev/null @@ -1,55 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT i_item_id, - i_item_desc, - s_store_id, - s_store_name, - tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_quantity) AS store_sales_quantity, - tpcds_sum((priv_hash(hash(ss_customer_sk))), sr_return_quantity) AS store_returns_quantity, - tpcds_sum((priv_hash(hash(ss_customer_sk))), cs_quantity) AS catalog_sales_quantity, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.store_sales, - main.store_returns, - main.catalog_sales, - main.date_dim d1, - main.date_dim d2, - main.date_dim d3, - main.store, - main.item, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND d1.d_moy = 9 - AND d1.d_year = 1999 - AND d1.d_date_sk = ss_sold_date_sk - AND i_item_sk = ss_item_sk - AND s_store_sk = ss_store_sk - AND ss_customer_sk = sr_customer_sk - AND ss_item_sk = sr_item_sk - AND ss_ticket_number = sr_ticket_number - AND sr_returned_date_sk = d2.d_date_sk - AND d2.d_moy BETWEEN 9 AND 9 + 3 - AND d2.d_year = 1999 - AND sr_customer_sk = cs_bill_customer_sk - AND sr_item_sk = cs_item_sk - AND cs_sold_date_sk = d3.d_date_sk - AND d3.d_year IN (1999, - 1999+1, - 1999+2) -GROUP BY i_item_id, - i_item_desc, - s_store_id, - s_store_name -), answers AS ( - SELECT world, i_item_id, - i_item_desc, - s_store_id, - s_store_name, - store_sales_quantity[world+1] AS store_sales_quantity, - store_returns_quantity[world+1] AS store_returns_quantity, - catalog_sales_quantity[world+1] AS catalog_sales_quantity - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id, i_item_desc, s_store_id, s_store_name) <= 100 -ORDER BY world, i_item_id, i_item_desc, s_store_id, s_store_name; diff --git a/benchmark/tpcds/reference_queries/q30.sql b/benchmark/tpcds/reference_queries/q30.sql deleted file mode 100644 index 625a01f6..00000000 --- a/benchmark/tpcds/reference_queries/q30.sql +++ /dev/null @@ -1,68 +0,0 @@ --- Correlated thresholds use only the matching world of the reduced return aggregates. -WITH return_stats AS ( -SELECT wr_returning_customer_sk AS ctr_customer_sk, - ca_state AS ctr_state, - tpcds_sum(priv_hash(hash(wr_returning_customer_sk)), wr_return_amt) AS ctr_total_return, - tpcds_count(priv_hash(hash(wr_returning_customer_sk)),1) AS presence -FROM main.web_returns, - main.date_dim, - main.customer_address, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE wr_returning_customer_sk=asa_owner.asa_key AND wr_returned_date_sk = d_date_sk - AND d_year = 2002 - AND wr_returning_addr_sk = ca_address_sk - GROUP BY wr_returning_customer_sk, - ca_state -), customer_total_return AS ( -SELECT world, ctr_customer_sk, ctr_state, ctr_total_return[world+1] AS ctr_total_return -FROM return_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT ctr1.world AS world, c_customer_id, - c_salutation, - c_first_name, - c_last_name, - c_preferred_cust_flag, - c_birth_day, - c_birth_month, - c_birth_year, - c_birth_country, - c_login, - c_email_address, - c_last_review_date_sk, - ctr_total_return -FROM customer_total_return ctr1, - main.customer_address, - main.customer -WHERE ctr1.ctr_total_return > - (SELECT avg(ctr_total_return)*1.2 - FROM customer_total_return ctr2 - WHERE ctr1.world=ctr2.world AND ctr1.ctr_state = ctr2.ctr_state) - AND ca_address_sk = c_current_addr_sk - AND ca_state = 'GA' - AND ctr1.ctr_customer_sk = c_customer_sk -QUALIFY row_number() OVER (PARTITION BY ctr1.world ORDER BY c_customer_id NULLS FIRST, - c_salutation NULLS FIRST, - c_first_name NULLS FIRST, - c_last_name NULLS FIRST, - c_preferred_cust_flag NULLS FIRST, - c_birth_day NULLS FIRST, - c_birth_month NULLS FIRST, - c_birth_year NULLS FIRST, - c_birth_country NULLS FIRST, - c_login NULLS FIRST, - c_email_address NULLS FIRST, - c_last_review_date_sk NULLS FIRST, - ctr_total_return NULLS FIRST) <= 100 -ORDER BY ctr1.world, c_customer_id NULLS FIRST, - c_salutation NULLS FIRST, - c_first_name NULLS FIRST, - c_last_name NULLS FIRST, - c_preferred_cust_flag NULLS FIRST, - c_birth_day NULLS FIRST, - c_birth_month NULLS FIRST, - c_birth_year NULLS FIRST, - c_birth_country NULLS FIRST, - c_login NULLS FIRST, - c_email_address NULLS FIRST, - c_last_review_date_sk NULLS FIRST, - ctr_total_return NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q31.sql b/benchmark/tpcds/reference_queries/q31.sql deleted file mode 100644 index cdaf8c82..00000000 --- a/benchmark/tpcds/reference_queries/q31.sql +++ /dev/null @@ -1,85 +0,0 @@ --- Fact scans feed AS once; subsequent processing is over reduced aggregate results. -WITH ss_stats AS ( -SELECT ca_county, - d_qoy, - d_year, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS store_sales, - tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence -FROM main.store_sales, - main.date_dim, - main.customer_address, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk = d_date_sk - AND ss_addr_sk=ca_address_sk - GROUP BY ca_county, - d_qoy, - d_year -), -ss AS ( -SELECT world, ca_county, d_qoy, d_year, store_sales[world+1] AS store_sales -FROM ss_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -ws_stats AS ( -SELECT ca_county, - d_qoy, - d_year, - tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) AS web_sales, - tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence -FROM main.web_sales, - main.date_dim, - main.customer_address, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_sold_date_sk = d_date_sk - AND ws_bill_addr_sk=ca_address_sk - GROUP BY ca_county, - d_qoy, - d_year -), -ws AS ( -SELECT world, ca_county, d_qoy, d_year, web_sales[world+1] AS web_sales -FROM ws_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT ss1.world AS world, ss1.ca_county , - ss1.d_year , - (ws2.web_sales*1.0000)/ws1.web_sales web_q1_q2_increase , - (ss2.store_sales*1.0000)/ss1.store_sales store_q1_q2_increase , - (ws3.web_sales*1.0000)/ws2.web_sales web_q2_q3_increase , - (ss3.store_sales*1.0000)/ss2.store_sales store_q2_q3_increase -FROM ss ss1 , - ss ss2 , - ss ss3 , - ws ws1 , - ws ws2 , - ws ws3 -WHERE ss1.world=ss2.world AND ss1.world=ss3.world AND ss1.world=ws1.world AND ss1.world=ws2.world AND ss1.world=ws3.world AND ss1.d_qoy = 1 - AND ss1.d_year = 2000 - AND ss1.ca_county = ss2.ca_county - AND ss2.d_qoy = 2 - AND ss2.d_year = 2000 - AND ss2.ca_county = ss3.ca_county - AND ss3.d_qoy = 3 - AND ss3.d_year = 2000 - AND ss1.ca_county = ws1.ca_county - AND ws1.d_qoy = 1 - AND ws1.d_year = 2000 - AND ws1.ca_county = ws2.ca_county - AND ws2.d_qoy = 2 - AND ws2.d_year = 2000 - AND ws1.ca_county = ws3.ca_county - AND ws3.d_qoy = 3 - AND ws3.d_year = 2000 - AND CASE - WHEN ws1.web_sales > 0 THEN (ws2.web_sales*1.0000)/ws1.web_sales - ELSE NULL - END > CASE - WHEN ss1.store_sales > 0 THEN (ss2.store_sales*1.0000)/ss1.store_sales - ELSE NULL - END - AND CASE - WHEN ws2.web_sales > 0 THEN (ws3.web_sales*1.0000)/ws2.web_sales - ELSE NULL - END > CASE - WHEN ss2.store_sales > 0 THEN (ss3.store_sales*1.0000)/ss2.store_sales - ELSE NULL - END -ORDER BY ss1.world, ss1.ca_county; diff --git a/benchmark/tpcds/reference_queries/q32.sql b/benchmark/tpcds/reference_queries/q32.sql deleted file mode 100644 index 46f64028..00000000 --- a/benchmark/tpcds/reference_queries/q32.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Correlated aggregate predicates update the mask before the outer AS SUM. -WITH thresholds AS ( - SELECT cs_item_sk, tpcds_avg(priv_hash(hash(cs_bill_customer_sk)),cs_ext_discount_amt) AS averages - FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk JOIN main.date_dim ON cs_sold_date_sk=d_date_sk - WHERE d_date BETWEEN DATE '2000-01-27' AND DATE '2000-04-26' GROUP BY cs_item_sk -), filtered AS ( - SELECT s.cs_ext_discount_amt AS discount, - priv_select(priv_hash(hash(cs_bill_customer_sk)),list_transform(averages, lambda average: s.cs_ext_discount_amt>1.3*average)) AS membership - FROM main.catalog_sales s JOIN main.customer ON cs_bill_customer_sk=c_customer_sk - JOIN main.item ON s.cs_item_sk=i_item_sk JOIN main.date_dim ON s.cs_sold_date_sk=d_date_sk - JOIN thresholds t ON s.cs_item_sk=t.cs_item_sk - WHERE i_manufact_id=977 AND d_date BETWEEN DATE '2000-01-27' AND DATE '2000-04-26' -), stats AS (SELECT tpcds_sum(membership,discount) AS total FROM filtered) -SELECT world,total[world+1] AS excess_discount_amount FROM stats CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q33.sql b/benchmark/tpcds/reference_queries/q33.sql deleted file mode 100644 index b260140b..00000000 --- a/benchmark/tpcds/reference_queries/q33.sql +++ /dev/null @@ -1,90 +0,0 @@ --- Fact scans feed AS once; subsequent processing is over reduced aggregate results. -WITH ss_stats AS ( -SELECT i_manufact_id, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales, - tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence -FROM main.store_sales, - main.date_dim, - main.customer_address, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND i_manufact_id IN - (SELECT i_manufact_id - FROM main.item - WHERE i_category IN ('Electronics')) - AND ss_item_sk = i_item_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 5 - AND ss_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_manufact_id -), -ss AS ( -SELECT world, i_manufact_id, total_sales[world+1] AS total_sales -FROM ss_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -cs_stats AS ( -SELECT i_manufact_id, - tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales, - tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence -FROM main.catalog_sales, - main.date_dim, - main.customer_address, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cs_bill_customer_sk=asa_owner.asa_key AND i_manufact_id IN - (SELECT i_manufact_id - FROM main.item - WHERE i_category IN ('Electronics')) - AND cs_item_sk = i_item_sk - AND cs_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 5 - AND cs_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_manufact_id -), -cs AS ( -SELECT world, i_manufact_id, total_sales[world+1] AS total_sales -FROM cs_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -ws_stats AS ( -SELECT i_manufact_id, - tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales, - tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence -FROM main.web_sales, - main.date_dim, - main.customer_address, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ws_bill_customer_sk=asa_owner.asa_key AND i_manufact_id IN - (SELECT i_manufact_id - FROM main.item - WHERE i_category IN ('Electronics')) - AND ws_item_sk = i_item_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 5 - AND ws_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_manufact_id -), -ws AS ( -SELECT world, i_manufact_id, total_sales[world+1] AS total_sales -FROM ws_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -combined AS ( -SELECT world, i_manufact_id, - sum(total_sales) total_sales -FROM - (SELECT * - FROM ss - UNION ALL SELECT * - FROM cs - UNION ALL SELECT * - FROM ws) tmp1 -GROUP BY world, i_manufact_id -) -SELECT * FROM combined QUALIFY row_number() OVER (PARTITION BY world ORDER BY total_sales) <= 100 -ORDER BY world, total_sales; diff --git a/benchmark/tpcds/reference_queries/q34.sql b/benchmark/tpcds/reference_queries/q34.sql deleted file mode 100644 index 97109eb0..00000000 --- a/benchmark/tpcds/reference_queries/q34.sql +++ /dev/null @@ -1,51 +0,0 @@ --- AS aggregation is shared; only the reduced per-ticket results are expanded by world. -WITH stats AS ( - SELECT ss_ticket_number, - ss_customer_sk, - tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS cnt, - tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS presence - FROM main.store_sales, - main.date_dim, - main.store, - main.household_demographics, - (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key - AND store_sales.ss_sold_date_sk = date_dim.d_date_sk - AND store_sales.ss_store_sk = store.s_store_sk - AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - AND (date_dim.d_dom BETWEEN 1 AND 3 - OR date_dim.d_dom BETWEEN 25 AND 28) - AND (household_demographics.hd_buy_potential = '>10000' - OR household_demographics.hd_buy_potential = 'Unknown') - AND household_demographics.hd_vehicle_count > 0 - AND (CASE - WHEN household_demographics.hd_vehicle_count > 0 THEN (household_demographics.hd_dep_count*1.000)/ household_demographics.hd_vehicle_count - ELSE NULL - END) > 1.2 - AND date_dim.d_year IN (1999, - 1999+1, - 1999+2) - AND store.s_county = 'Williamson County' - GROUP BY ss_ticket_number, - ss_customer_sk -), expanded AS ( - SELECT world, ss_ticket_number, ss_customer_sk, cnt[world+1] AS cnt - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1]>0 -) -SELECT world, c_last_name , - c_first_name , - c_salutation , - c_preferred_cust_flag , - ss_ticket_number , - cnt -FROM - expanded dn, - main.customer -WHERE ss_customer_sk = c_customer_sk - AND cnt BETWEEN 15 AND 20 -ORDER BY world, c_last_name NULLS FIRST, - c_first_name NULLS FIRST, - c_salutation NULLS FIRST, - c_preferred_cust_flag DESC NULLS FIRST, - ss_ticket_number NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q35.sql b/benchmark/tpcds/reference_queries/q35.sql deleted file mode 100644 index 4a421997..00000000 --- a/benchmark/tpcds/reference_queries/q35.sql +++ /dev/null @@ -1,111 +0,0 @@ --- Same-owner EXISTS is static after entity inclusion; shipping-customer EXISTS needs the bill-owner mask. -WITH catalog_presence AS ( -SELECT cs_ship_customer_sk,bit_or(priv_hash(hash(cs_bill_customer_sk))) AS membership - FROM main.catalog_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4 -GROUP BY cs_ship_customer_sk -), stats AS (SELECT ca_state, -cd_gender, -cd_marital_status, -cd_dep_count, -cd_dep_employed_count, -cd_dep_college_count, -tpcds_count((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),1) AS n, -tpcds_min((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_count) AS min1, -tpcds_max((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_count) AS max1, -tpcds_avg((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_count) AS avg1, -tpcds_min((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_employed_count) AS min2, -tpcds_max((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_employed_count) AS max2, -tpcds_avg((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_employed_count) AS avg2, -tpcds_min((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_college_count) AS min3, -tpcds_max((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_college_count) AS max3, -tpcds_avg((priv_hash(hash(c.c_customer_sk)) & ((CASE WHEN EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) THEN 18446744073709551615::UBIGINT ELSE 0::UBIGINT END) | coalesce(catalog_presence.membership,0::UBIGINT))),cd_dep_college_count) AS avg3 -FROM main.customer c CROSS JOIN - main.customer_address ca CROSS JOIN - main.customer_demographics LEFT JOIN catalog_presence ON c.c_customer_sk=cs_ship_customer_sk -WHERE c.c_current_addr_sk = ca.ca_address_sk - AND cd_demo_sk = c.c_current_cdemo_sk - -AND EXISTS(SELECT * - FROM main.store_sales, - main.date_dim - WHERE c.c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2002 - AND d_qoy < 4) -GROUP BY ca_state, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count -), answers AS (SELECT world, ca_state, cd_gender, cd_marital_status, cd_dep_count, n[world+1] AS cnt1, min1[world+1] AS min1, max1[world+1] AS max1, avg1[world+1] AS avg1, cd_dep_employed_count, n[world+1] AS cnt2, min2[world+1] AS min2, max2[world+1] AS max2, avg2[world+1] AS avg2, cd_dep_college_count, n[world+1] AS cnt3, min3[world+1] AS min3, max3[world+1] AS max3, avg3[world+1] AS avg3 FROM stats CROSS JOIN range(64) worlds(world) WHERE n[world+1]>0) -SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY ca_state NULLS FIRST, - cd_gender NULLS FIRST, - cd_marital_status NULLS FIRST, - cd_dep_count NULLS FIRST, - cd_dep_employed_count NULLS FIRST, - cd_dep_college_count NULLS FIRST)<=100 ORDER BY world,ca_state NULLS FIRST, - cd_gender NULLS FIRST, - cd_marital_status NULLS FIRST, - cd_dep_count NULLS FIRST, - cd_dep_employed_count NULLS FIRST, - cd_dep_college_count NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q40.sql b/benchmark/tpcds/reference_queries/q40.sql deleted file mode 100644 index 1533f6af..00000000 --- a/benchmark/tpcds/reference_queries/q40.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Matched and NULL-extended memberships are disjoint; NULL extension is evaluated separately in each world. -WITH left_input AS ( -SELECT w_state,i_item_id,d_date,cs_sales_price,cs_order_number,cs_item_sk,priv_hash(hash(cs_bill_customer_sk)) AS membership -FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk -JOIN main.warehouse ON cs_warehouse_sk=w_warehouse_sk JOIN main.item ON i_item_sk=cs_item_sk -JOIN main.date_dim ON cs_sold_date_sk=d_date_sk -WHERE i_current_price BETWEEN 0.99 AND 1.49 AND d_date BETWEEN DATE '2000-02-10' AND DATE '2000-04-10' -), valid_returns AS (SELECT r.*,priv_hash(hash(cr_returning_customer_sk)) AS membership FROM main.catalog_returns r JOIN main.customer ON cr_returning_customer_sk=c_customer_sk), -return_presence AS (SELECT cr_order_number,cr_item_sk,bit_or(membership) AS membership FROM valid_returns GROUP BY cr_order_number,cr_item_sk), -joined AS ( - SELECT l.w_state,l.i_item_id,l.d_date,l.cs_sales_price,coalesce(r.cr_refunded_cash,0) AS refund,(l.membership & r.membership) AS membership FROM left_input l JOIN valid_returns r ON l.cs_order_number=r.cr_order_number AND l.cs_item_sk=r.cr_item_sk - UNION ALL - SELECT l.w_state,l.i_item_id,l.d_date,l.cs_sales_price,0 AS refund,(l.membership & ~coalesce(r.membership,0::UBIGINT)) AS membership FROM left_input l LEFT JOIN return_presence r ON l.cs_order_number=r.cr_order_number AND l.cs_item_sk=r.cr_item_sk -), stats AS (SELECT w_state,i_item_id,tpcds_sum(membership,CASE WHEN d_date=DATE '2000-03-11' THEN cs_sales_price-refund ELSE 0 END) AS sales_after,tpcds_count(membership,1) AS presence FROM joined GROUP BY w_state,i_item_id), -answers AS (SELECT world,w_state,i_item_id,sales_before[world+1] AS sales_before,sales_after[world+1] AS sales_after FROM stats CROSS JOIN range(64) worlds(world) WHERE presence[world+1]>0) -SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY w_state,i_item_id)<=100 ORDER BY world,w_state,i_item_id; diff --git a/benchmark/tpcds/reference_queries/q42.sql b/benchmark/tpcds/reference_queries/q42.sql deleted file mode 100644 index 3b28dcd2..00000000 --- a/benchmark/tpcds/reference_queries/q42.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT dt.d_year AS d_year, - item.i_category_id AS i_category_id, - item.i_category AS i_category, - tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS total, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.date_dim dt, - main.store_sales, - main.item, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND dt.d_date_sk = store_sales.ss_sold_date_sk - AND store_sales.ss_item_sk = item.i_item_sk - AND item.i_manager_id = 1 - AND dt.d_moy=11 - AND dt.d_year=2000 -GROUP BY dt.d_year, - main.item.i_category_id, - main.item.i_category -), answers AS ( - SELECT world, d_year, - i_category_id, - i_category, - total[world+1] AS total - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY total DESC, d_year, i_category_id, i_category) <= 100 -ORDER BY world, total DESC, d_year, i_category_id, i_category; diff --git a/benchmark/tpcds/reference_queries/q43.sql b/benchmark/tpcds/reference_queries/q43.sql deleted file mode 100644 index 63cc41c9..00000000 --- a/benchmark/tpcds/reference_queries/q43.sql +++ /dev/null @@ -1,39 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT s_store_name, - s_store_id, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Sunday' THEN ss_sales_price ELSE NULL END) AS sun_sales, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Monday' THEN ss_sales_price ELSE NULL END) AS mon_sales, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Tuesday' THEN ss_sales_price ELSE NULL END) AS tue_sales, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Wednesday' THEN ss_sales_price ELSE NULL END) AS wed_sales, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Thursday' THEN ss_sales_price ELSE NULL END) AS thu_sales, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Friday' THEN ss_sales_price ELSE NULL END) AS fri_sales, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN d_day_name='Saturday' THEN ss_sales_price ELSE NULL END) AS sat_sales, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.date_dim, - main.store_sales, - main.store, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND d_date_sk = ss_sold_date_sk - AND s_store_sk = ss_store_sk - AND s_gmt_offset = -5 - AND d_year = 2000 -GROUP BY s_store_name, - s_store_id -), answers AS ( - SELECT world, s_store_name, - s_store_id, - sun_sales[world+1] AS sun_sales, - mon_sales[world+1] AS mon_sales, - tue_sales[world+1] AS tue_sales, - wed_sales[world+1] AS wed_sales, - thu_sales[world+1] AS thu_sales, - fri_sales[world+1] AS fri_sales, - sat_sales[world+1] AS sat_sales - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY s_store_name, s_store_id, sun_sales, mon_sales, tue_sales, wed_sales, thu_sales, fri_sales, sat_sales) <= 100 -ORDER BY world, s_store_name, s_store_id, sun_sales, mon_sales, tue_sales, wed_sales, thu_sales, fri_sales, sat_sales; diff --git a/benchmark/tpcds/reference_queries/q45.sql b/benchmark/tpcds/reference_queries/q45.sql deleted file mode 100644 index f6f4d573..00000000 --- a/benchmark/tpcds/reference_queries/q45.sql +++ /dev/null @@ -1,53 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT ca_zip, - ca_city, - tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), ws_sales_price) AS total, - tpcds_count((priv_hash(hash(ws_bill_customer_sk))), 1) AS presence - FROM main.web_sales, - main.customer, - main.customer_address, - main.date_dim, - main.item, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ws_bill_customer_sk = asa_owner0.asa_key0 - AND ws_bill_customer_sk = c_customer_sk - AND c_current_addr_sk = ca_address_sk - AND ws_item_sk = i_item_sk - AND (SUBSTRING(ca_zip,1,5) IN ('85669', - '86197', - '88274', - '83405', - '86475', - '85392', - '85460', - '80348', - '81792') - OR i_item_id IN - (SELECT i_item_id - FROM main.item - WHERE i_item_sk IN (2, - 3, - 5, - 7, - 11, - 13, - 17, - 19, - 23, - 29) )) - AND ws_sold_date_sk = d_date_sk - AND d_qoy = 2 - AND d_year = 2001 -GROUP BY ca_zip, - ca_city -), answers AS ( - SELECT world, ca_zip, - ca_city, - total[world+1] AS total - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY ca_zip, ca_city) <= 100 -ORDER BY world, ca_zip, ca_city; diff --git a/benchmark/tpcds/reference_queries/q46.sql b/benchmark/tpcds/reference_queries/q46.sql deleted file mode 100644 index 3d95ef46..00000000 --- a/benchmark/tpcds/reference_queries/q46.sql +++ /dev/null @@ -1,61 +0,0 @@ --- AS aggregation is shared; only the reduced per-ticket results are expanded by world. -WITH stats AS ( - SELECT ss_ticket_number, - ss_customer_sk, - ca_city AS bought_city, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) AS amt, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) AS profit, - tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS presence - FROM main.store_sales, - main.date_dim, - main.store, - main.household_demographics, - main.customer_address, - (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key - AND store_sales.ss_sold_date_sk = date_dim.d_date_sk - AND store_sales.ss_store_sk = store.s_store_sk - AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - AND store_sales.ss_addr_sk = customer_address.ca_address_sk - AND (household_demographics.hd_dep_count = 4 - OR household_demographics.hd_vehicle_count= 3) - AND date_dim.d_dow IN (6, - 0) - AND date_dim.d_year IN (1999, - 1999+1, - 1999+2) - AND store.s_city IN ('Fairview', - 'Midway') - GROUP BY ss_ticket_number, - ss_customer_sk, - ss_addr_sk, - ca_city -), expanded AS ( - SELECT world, ss_ticket_number, ss_customer_sk, bought_city, amt[world+1] AS amt, profit[world+1] AS profit - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1]>0 -) -SELECT world, c_last_name, - c_first_name, - ca_city, - bought_city, - ss_ticket_number, - amt, - profit -FROM - expanded dn, - main.customer, - main.customer_address current_addr -WHERE ss_customer_sk = c_customer_sk - AND customer.c_current_addr_sk = current_addr.ca_address_sk - AND current_addr.ca_city <> bought_city -QUALIFY row_number() OVER (PARTITION BY world ORDER BY c_last_name NULLS FIRST, - c_first_name NULLS FIRST, - ca_city NULLS FIRST, - bought_city NULLS FIRST, - ss_ticket_number NULLS FIRST) <= 100 -ORDER BY world, c_last_name NULLS FIRST, - c_first_name NULLS FIRST, - ca_city NULLS FIRST, - bought_city NULLS FIRST, - ss_ticket_number NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q48.sql b/benchmark/tpcds/reference_queries/q48.sql deleted file mode 100644 index ebf74011..00000000 --- a/benchmark/tpcds/reference_queries/q48.sql +++ /dev/null @@ -1,50 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_quantity) AS total, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.store_sales, - main.store, - main.customer_demographics, - main.customer_address, - main.date_dim, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND s_store_sk = ss_store_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2000 - AND ((cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'M' - AND cd_education_status = '4 yr Degree' - AND ss_sales_price BETWEEN 100.00 AND 150.00) - OR (cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'D' - AND cd_education_status = '2 yr Degree' - AND ss_sales_price BETWEEN 50.00 AND 100.00) - OR (cd_demo_sk = ss_cdemo_sk - AND cd_marital_status = 'S' - AND cd_education_status = 'College' - AND ss_sales_price BETWEEN 150.00 AND 200.00)) - AND ((ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('CO', - 'OH', - 'TX') - AND ss_net_profit BETWEEN 0 AND 2000) - OR (ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('OR', - 'MN', - 'KY') - AND ss_net_profit BETWEEN 150 AND 3000) - OR (ss_addr_sk = ca_address_sk - AND ca_country = 'United States' - AND ca_state IN ('VA', - 'CA', - 'MS') - AND ss_net_profit BETWEEN 50 AND 25000)) -), answers AS ( - SELECT world, total[world+1] AS total - FROM stats CROSS JOIN range(64) worlds(world) -) -SELECT * FROM answers -ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q50.sql b/benchmark/tpcds/reference_queries/q50.sql deleted file mode 100644 index 4a699354..00000000 --- a/benchmark/tpcds/reference_queries/q50.sql +++ /dev/null @@ -1,65 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT s_store_name, - s_company_id, - s_street_number, - s_street_name, - s_street_type, - s_suite_number, - s_city, - s_county, - s_state, - s_zip, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN sr_returned_date_sk - ss_sold_date_sk <= 30 THEN 1 ELSE 0 END) AS bucket0, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN sr_returned_date_sk - ss_sold_date_sk > 30 AND sr_returned_date_sk - ss_sold_date_sk <= 60 THEN 1 ELSE 0 END) AS bucket1, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN sr_returned_date_sk - ss_sold_date_sk > 60 AND sr_returned_date_sk - ss_sold_date_sk <= 90 THEN 1 ELSE 0 END) AS bucket2, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN sr_returned_date_sk - ss_sold_date_sk > 90 AND sr_returned_date_sk - ss_sold_date_sk <= 120 THEN 1 ELSE 0 END) AS bucket3, - tpcds_sum((priv_hash(hash(ss_customer_sk))), CASE WHEN sr_returned_date_sk - ss_sold_date_sk > 120 THEN 1 ELSE 0 END) AS bucket4, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.store_sales, - main.store_returns, - main.store, - main.date_dim d1, - main.date_dim d2, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND d2.d_year = 2001 - AND d2.d_moy = 8 - AND ss_ticket_number = sr_ticket_number - AND ss_item_sk = sr_item_sk - AND ss_sold_date_sk = d1.d_date_sk - AND sr_returned_date_sk = d2.d_date_sk - AND ss_customer_sk = sr_customer_sk - AND ss_store_sk = s_store_sk -GROUP BY s_store_name, - s_company_id, - s_street_number, - s_street_name, - s_street_type, - s_suite_number, - s_city, - s_county, - s_state, - s_zip -), answers AS ( - SELECT world, s_store_name, - s_company_id, - s_street_number, - s_street_name, - s_street_type, - s_suite_number, - s_city, - s_county, - s_state, - s_zip, - bucket0[world+1] AS bucket0, - bucket1[world+1] AS bucket1, - bucket2[world+1] AS bucket2, - bucket3[world+1] AS bucket3, - bucket4[world+1] AS bucket4 - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip) <= 100 -ORDER BY world, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip; diff --git a/benchmark/tpcds/reference_queries/q52.sql b/benchmark/tpcds/reference_queries/q52.sql deleted file mode 100644 index dbbc5c6c..00000000 --- a/benchmark/tpcds/reference_queries/q52.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT dt.d_year AS d_year, - item.i_brand_id AS brand_id, - item.i_brand AS brand, - tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS ext_price, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.date_dim dt, - main.store_sales, - main.item, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND dt.d_date_sk = store_sales.ss_sold_date_sk - AND store_sales.ss_item_sk = item.i_item_sk - AND item.i_manager_id = 1 - AND dt.d_moy=11 - AND dt.d_year=2000 -GROUP BY dt.d_year, - main.item.i_brand, - main.item.i_brand_id -), answers AS ( - SELECT world, d_year, - brand_id, - brand, - ext_price[world+1] AS ext_price - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY d_year, ext_price DESC, brand_id) <= 100 -ORDER BY world, d_year, ext_price DESC, brand_id; diff --git a/benchmark/tpcds/reference_queries/q55.sql b/benchmark/tpcds/reference_queries/q55.sql deleted file mode 100644 index 14f28825..00000000 --- a/benchmark/tpcds/reference_queries/q55.sql +++ /dev/null @@ -1,28 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT i_brand_id AS brand_id, - i_brand AS brand, - tpcds_sum((priv_hash(hash(ss_customer_sk))), ss_ext_sales_price) AS ext_price, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.date_dim, - main.store_sales, - main.item, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND d_date_sk = ss_sold_date_sk - AND ss_item_sk = i_item_sk - AND i_manager_id=28 - AND d_moy=11 - AND d_year=1999 -GROUP BY i_brand, - i_brand_id -), answers AS ( - SELECT world, brand_id, - brand, - ext_price[world+1] AS ext_price - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY ext_price DESC, brand_id) <= 100 -ORDER BY world, ext_price DESC, brand_id; diff --git a/benchmark/tpcds/reference_queries/q56.sql b/benchmark/tpcds/reference_queries/q56.sql deleted file mode 100644 index 267dc3fd..00000000 --- a/benchmark/tpcds/reference_queries/q56.sql +++ /dev/null @@ -1,98 +0,0 @@ --- Fact scans feed AS once; subsequent processing is over reduced aggregate results. -WITH ss_stats AS ( -SELECT i_item_id, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales, - tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence -FROM main.store_sales, - main.date_dim, - main.customer_address, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND i_item_id IN - (SELECT i_item_id - FROM main.item - WHERE i_color IN ('slate', - 'blanched', - 'burnished')) - AND ss_item_sk = i_item_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy = 2 - AND ss_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id -), -ss AS ( -SELECT world, i_item_id, total_sales[world+1] AS total_sales -FROM ss_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -cs_stats AS ( -SELECT i_item_id, - tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales, - tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence -FROM main.catalog_sales, - main.date_dim, - main.customer_address, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cs_bill_customer_sk=asa_owner.asa_key AND i_item_id IN - (SELECT i_item_id - FROM main.item - WHERE i_color IN ('slate', - 'blanched', - 'burnished')) - AND cs_item_sk = i_item_sk - AND cs_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy = 2 - AND cs_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id -), -cs AS ( -SELECT world, i_item_id, total_sales[world+1] AS total_sales -FROM cs_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -ws_stats AS ( -SELECT i_item_id, - tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales, - tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence -FROM main.web_sales, - main.date_dim, - main.customer_address, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ws_bill_customer_sk=asa_owner.asa_key AND i_item_id IN - (SELECT i_item_id - FROM main.item - WHERE i_color IN ('slate', - 'blanched', - 'burnished')) - AND ws_item_sk = i_item_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy = 2 - AND ws_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id -), -ws AS ( -SELECT world, i_item_id, total_sales[world+1] AS total_sales -FROM ws_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -combined AS ( -SELECT world, i_item_id, - sum(total_sales) total_sales -FROM - (SELECT * - FROM ss - UNION ALL SELECT * - FROM cs - UNION ALL SELECT * - FROM ws) tmp1 -GROUP BY world, i_item_id -) -SELECT * FROM combined QUALIFY row_number() OVER (PARTITION BY world ORDER BY total_sales NULLS FIRST, - i_item_id NULLS FIRST) <= 100 -ORDER BY world, total_sales NULLS FIRST, - i_item_id NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q58.sql b/benchmark/tpcds/reference_queries/q58.sql deleted file mode 100644 index 3396ad99..00000000 --- a/benchmark/tpcds/reference_queries/q58.sql +++ /dev/null @@ -1,93 +0,0 @@ --- Fact scans feed AS once; subsequent processing is over reduced aggregate results. -WITH ss_items_stats AS ( -SELECT i_item_id item_id, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) ss_item_rev, - tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence -FROM main.store_sales, - main.item, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND ss_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM main.date_dim - WHERE d_week_seq = - (SELECT d_week_seq - FROM main.date_dim - WHERE d_date = '2000-01-03')) - AND ss_sold_date_sk = d_date_sk - GROUP BY i_item_id -), -ss_items AS ( -SELECT world, item_id, ss_item_rev[world+1] AS ss_item_rev -FROM ss_items_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -cs_items_stats AS ( -SELECT i_item_id item_id, - tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) cs_item_rev, - tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence -FROM main.catalog_sales, - main.item, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM main.date_dim - WHERE d_week_seq = - (SELECT d_week_seq - FROM main.date_dim - WHERE d_date = '2000-01-03')) - AND cs_sold_date_sk = d_date_sk - GROUP BY i_item_id -), -cs_items AS ( -SELECT world, item_id, cs_item_rev[world+1] AS cs_item_rev -FROM cs_items_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -ws_items_stats AS ( -SELECT i_item_id item_id, - tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) ws_item_rev, - tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence -FROM main.web_sales, - main.item, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM main.date_dim - WHERE d_week_seq = - (SELECT d_week_seq - FROM main.date_dim - WHERE d_date = '2000-01-03')) - AND ws_sold_date_sk = d_date_sk - GROUP BY i_item_id -), -ws_items AS ( -SELECT world, item_id, ws_item_rev[world+1] AS ws_item_rev -FROM ws_items_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT ss_items.world AS world, ss_items.item_id, - ss_item_rev, - ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev, - cs_item_rev, - cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev, - ws_item_rev, - ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev, - (ss_item_rev+cs_item_rev+ws_item_rev)/3 average -FROM ss_items, - cs_items, - ws_items -WHERE ss_items.world=cs_items.world AND ss_items.world=ws_items.world AND ss_items.item_id=cs_items.item_id - AND ss_items.item_id=ws_items.item_id - AND ss_item_rev BETWEEN 0.9 * cs_item_rev AND 1.1 * cs_item_rev - AND ss_item_rev BETWEEN 0.9 * ws_item_rev AND 1.1 * ws_item_rev - AND cs_item_rev BETWEEN 0.9 * ss_item_rev AND 1.1 * ss_item_rev - AND cs_item_rev BETWEEN 0.9 * ws_item_rev AND 1.1 * ws_item_rev - AND ws_item_rev BETWEEN 0.9 * ss_item_rev AND 1.1 * ss_item_rev - AND ws_item_rev BETWEEN 0.9 * cs_item_rev AND 1.1 * cs_item_rev -QUALIFY row_number() OVER (PARTITION BY ss_items.world ORDER BY ss_items.item_id NULLS FIRST, - ss_item_rev NULLS FIRST) <= 100 -ORDER BY ss_items.world, ss_items.item_id NULLS FIRST, - ss_item_rev NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q59.sql b/benchmark/tpcds/reference_queries/q59.sql deleted file mode 100644 index e2695b7c..00000000 --- a/benchmark/tpcds/reference_queries/q59.sql +++ /dev/null @@ -1,94 +0,0 @@ --- A shared per-store/week AS result is reused across both years. -WITH wss_stats AS ( -SELECT d_week_seq, - ss_store_sk, - tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Sunday') THEN ss_sales_price - ELSE NULL - END) sun_sales, - tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Monday') THEN ss_sales_price - ELSE NULL - END) mon_sales, - tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Tuesday') THEN ss_sales_price - ELSE NULL - END) tue_sales, - tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Wednesday') THEN ss_sales_price - ELSE NULL - END) wed_sales, - tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Thursday') THEN ss_sales_price - ELSE NULL - END) thu_sales, - tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Friday') THEN ss_sales_price - ELSE NULL - END) fri_sales, - tpcds_sum(priv_hash(hash(ss_customer_sk)), CASE - WHEN (d_day_name='Saturday') THEN ss_sales_price - ELSE NULL - END) sat_sales, - tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence -FROM main.store_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND d_date_sk = ss_sold_date_sk - GROUP BY d_week_seq, - ss_store_sk -), wss AS ( -SELECT world, d_week_seq, ss_store_sk, sun_sales[world+1] AS sun_sales, mon_sales[world+1] AS mon_sales, tue_sales[world+1] AS tue_sales, wed_sales[world+1] AS wed_sales, thu_sales[world+1] AS thu_sales, fri_sales[world+1] AS fri_sales, sat_sales[world+1] AS sat_sales -FROM wss_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT y.world AS world, s_store_name1, - s_store_id1, - d_week_seq1, - sun_sales1/sun_sales2 AS sun_sales_ratio, - mon_sales1/mon_sales2 AS mon_sales_ratio, - tue_sales1/tue_sales2 AS tue_sales_ratio, - wed_sales1/wed_sales2 AS wed_sales_ratio, - thu_sales1/thu_sales2 AS thu_sales_ratio, - fri_sales1/fri_sales2 AS fri_sales_ratio, - sat_sales1/sat_sales2 AS sat_sales_ratio -FROM - (SELECT wss.world, s_store_name s_store_name1, - wss.d_week_seq d_week_seq1, - s_store_id s_store_id1, - sun_sales sun_sales1, - mon_sales mon_sales1, - tue_sales tue_sales1, - wed_sales wed_sales1, - thu_sales thu_sales1, - fri_sales fri_sales1, - sat_sales sat_sales1 - FROM wss, - main.store, - main.date_dim d - WHERE d.d_week_seq = wss.d_week_seq - AND ss_store_sk = s_store_sk - AND d_month_seq BETWEEN 1212 AND 1212 + 11) y, - (SELECT wss.world, s_store_name s_store_name2, - wss.d_week_seq d_week_seq2, - s_store_id s_store_id2, - sun_sales sun_sales2, - mon_sales mon_sales2, - tue_sales tue_sales2, - wed_sales wed_sales2, - thu_sales thu_sales2, - fri_sales fri_sales2, - sat_sales sat_sales2 - FROM wss, - main.store, - main.date_dim d - WHERE d.d_week_seq = wss.d_week_seq - AND ss_store_sk = s_store_sk - AND d_month_seq BETWEEN 1212 + 12 AND 1212 + 23) x -WHERE y.world=x.world AND s_store_id1=s_store_id2 - AND d_week_seq1=d_week_seq2-52 -QUALIFY row_number() OVER (PARTITION BY y.world ORDER BY s_store_name1 NULLS FIRST, - s_store_id1 NULLS FIRST, - d_week_seq1 NULLS FIRST) <= 100 -ORDER BY y.world, s_store_name1 NULLS FIRST, - s_store_id1 NULLS FIRST, - d_week_seq1 NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q60.sql b/benchmark/tpcds/reference_queries/q60.sql deleted file mode 100644 index 86f4a209..00000000 --- a/benchmark/tpcds/reference_queries/q60.sql +++ /dev/null @@ -1,92 +0,0 @@ --- Fact scans feed AS once; subsequent processing is over reduced aggregate results. -WITH ss_stats AS ( -SELECT i_item_id, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total_sales, - tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence -FROM main.store_sales, - main.date_dim, - main.customer_address, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND i_item_id IN - (SELECT i_item_id - FROM main.item - WHERE i_category ='Music') - AND ss_item_sk = i_item_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 9 - AND ss_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id -), -ss AS ( -SELECT world, i_item_id, total_sales[world+1] AS total_sales -FROM ss_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -cs_stats AS ( -SELECT i_item_id, - tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) total_sales, - tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence -FROM main.catalog_sales, - main.date_dim, - main.customer_address, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cs_bill_customer_sk=asa_owner.asa_key AND i_item_id IN - (SELECT i_item_id - FROM main.item - WHERE i_category ='Music') - AND cs_item_sk = i_item_sk - AND cs_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 9 - AND cs_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id -), -cs AS ( -SELECT world, i_item_id, total_sales[world+1] AS total_sales -FROM cs_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -ws_stats AS ( -SELECT i_item_id, - tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) total_sales, - tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence -FROM main.web_sales, - main.date_dim, - main.customer_address, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ws_bill_customer_sk=asa_owner.asa_key AND i_item_id IN - (SELECT i_item_id - FROM main.item - WHERE i_category = 'Music') - AND ws_item_sk = i_item_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 1998 - AND d_moy = 9 - AND ws_bill_addr_sk = ca_address_sk - AND ca_gmt_offset = -5 - GROUP BY i_item_id -), -ws AS ( -SELECT world, i_item_id, total_sales[world+1] AS total_sales -FROM ws_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -combined AS ( -SELECT world, i_item_id, - sum(total_sales) total_sales -FROM - (SELECT * - FROM ss - UNION ALL SELECT * - FROM cs - UNION ALL SELECT * - FROM ws) tmp1 -GROUP BY world, i_item_id -) -SELECT * FROM combined QUALIFY row_number() OVER (PARTITION BY world ORDER BY i_item_id, - total_sales) <= 100 -ORDER BY world, i_item_id, - total_sales; diff --git a/benchmark/tpcds/reference_queries/q61.sql b/benchmark/tpcds/reference_queries/q61.sql deleted file mode 100644 index 737d3390..00000000 --- a/benchmark/tpcds/reference_queries/q61.sql +++ /dev/null @@ -1,49 +0,0 @@ --- Scalar AS arrays are combined only after their independent shared scans. -WITH raw0 AS ( -SELECT tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) promotions - FROM main.store_sales, - main.store, - main.promotion, - main.date_dim, - main.customer, - main.customer_address, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk = d_date_sk - AND ss_store_sk = s_store_sk - AND ss_promo_sk = p_promo_sk - AND ss_customer_sk= c_customer_sk - AND ca_address_sk = c_current_addr_sk - AND ss_item_sk = i_item_sk - AND ca_gmt_offset = -5 - AND i_category = 'Jewelry' - AND (p_channel_dmail = 'Y' - OR p_channel_email = 'Y' - OR p_channel_tv = 'Y') - AND s_gmt_offset = -5 - AND d_year = 1998 - AND d_moy = 11 -), -raw1 AS ( -SELECT tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) total - FROM main.store_sales, - main.store, - main.date_dim, - main.customer, - main.customer_address, - main.item, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk = d_date_sk - AND ss_store_sk = s_store_sk - AND ss_customer_sk= c_customer_sk - AND ca_address_sk = c_current_addr_sk - AND ss_item_sk = i_item_sk - AND ca_gmt_offset = -5 - AND i_category = 'Jewelry' - AND s_gmt_offset = -5 - AND d_year = 1998 - AND d_moy = 11 -), -p AS (SELECT world, raw0.promotions[world+1] AS promotions, raw1.total[world+1] AS total FROM raw0 CROSS JOIN raw1 CROSS JOIN range(64) worlds(world)) -SELECT world, p.promotions, - p.total, - cast(p.promotions AS decimal(15,4))/cast(p.total AS decimal(15,4))*100 - FROM p ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q62.sql b/benchmark/tpcds/reference_queries/q62.sql deleted file mode 100644 index b247bb7f..00000000 --- a/benchmark/tpcds/reference_queries/q62.sql +++ /dev/null @@ -1,43 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT w_substr, - sm_type, - web_name, - tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), CASE WHEN ws_ship_date_sk - ws_sold_date_sk <= 30 THEN 1 ELSE 0 END) AS bucket0, - tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), CASE WHEN ws_ship_date_sk - ws_sold_date_sk > 30 AND ws_ship_date_sk - ws_sold_date_sk <= 60 THEN 1 ELSE 0 END) AS bucket1, - tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), CASE WHEN ws_ship_date_sk - ws_sold_date_sk > 60 AND ws_ship_date_sk - ws_sold_date_sk <= 90 THEN 1 ELSE 0 END) AS bucket2, - tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), CASE WHEN ws_ship_date_sk - ws_sold_date_sk > 90 AND ws_ship_date_sk - ws_sold_date_sk <= 120 THEN 1 ELSE 0 END) AS bucket3, - tpcds_sum((priv_hash(hash(ws_bill_customer_sk))), CASE WHEN ws_ship_date_sk - ws_sold_date_sk > 120 THEN 1 ELSE 0 END) AS bucket4, - tpcds_count((priv_hash(hash(ws_bill_customer_sk))), 1) AS presence - FROM main.web_sales, - (SELECT SUBSTRING(w_warehouse_name,1,20) w_substr, - * - FROM main.warehouse) sq1, - main.ship_mode, - main.web_site, - main.date_dim, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ws_bill_customer_sk = asa_owner0.asa_key0 - AND d_month_seq BETWEEN 1200 AND 1200 + 11 - AND ws_ship_date_sk = d_date_sk - AND ws_warehouse_sk = w_warehouse_sk - AND ws_ship_mode_sk = sm_ship_mode_sk - AND ws_web_site_sk = web_site_sk -GROUP BY w_substr, - sm_type, - web_name -), answers AS ( - SELECT world, w_substr, - sm_type, - web_name, - bucket0[world+1] AS bucket0, - bucket1[world+1] AS bucket1, - bucket2[world+1] AS bucket2, - bucket3[world+1] AS bucket3, - bucket4[world+1] AS bucket4 - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY w_substr NULLS FIRST, sm_type NULLS FIRST, web_name NULLS FIRST) <= 100 -ORDER BY world, w_substr NULLS FIRST, sm_type NULLS FIRST, web_name NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q64.sql b/benchmark/tpcds/reference_queries/q64.sql deleted file mode 100644 index bb5a2b4d..00000000 --- a/benchmark/tpcds/reference_queries/q64.sql +++ /dev/null @@ -1,133 +0,0 @@ --- The first stochastic HAVING is folded into membership before the cross-sales aggregate. -WITH cs_ui_stats AS ( -SELECT cs_item_sk, - tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & priv_hash(hash(cr_returning_customer_sk))), cs_ext_list_price) AS sale, - tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & priv_hash(hash(cr_returning_customer_sk))), cr_refunded_cash+cr_reversed_charge+cr_store_credit) AS refund, tpcds_count((priv_hash(hash(cs_bill_customer_sk)) & priv_hash(hash(cr_returning_customer_sk))),1) AS presence -FROM main.catalog_sales, - main.catalog_returns, (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0, (SELECT c_customer_sk AS asa_key1 FROM main.customer) asa_owner1 -WHERE cs_bill_customer_sk=asa_owner0.asa_key0 AND cr_returning_customer_sk=asa_owner1.asa_key1 AND cs_item_sk = cr_item_sk - AND cs_order_number = cr_order_number - GROUP BY cs_item_sk - -), -cs_ui AS (SELECT cs_item_sk,priv_select(18446744073709551615::UBIGINT,list_transform(list_zip(sale,refund),lambda x: x[1]>2*x[2])) AS membership FROM cs_ui_stats), -cross_sales_stats AS ( -SELECT i_product_name product_name, - i_item_sk item_sk, - s_store_name store_name, - s_zip store_zip, - ad1.ca_street_number b_street_number, - ad1.ca_street_name b_street_name, - ad1.ca_city b_city, - ad1.ca_zip b_zip, - ad2.ca_street_number c_street_number, - ad2.ca_street_name c_street_name, - ad2.ca_city c_city, - ad2.ca_zip c_zip, - d1.d_year AS syear, - d2.d_year AS fsyear, - d3.d_year s2year, - tpcds_count((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk)) & cs_ui.membership),1) cnt, - tpcds_sum((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk)) & cs_ui.membership), ss_wholesale_cost) s1, - tpcds_sum((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk)) & cs_ui.membership), ss_list_price) s2, - tpcds_sum((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk)) & cs_ui.membership), ss_coupon_amt) s3, tpcds_count((priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk)) & cs_ui.membership),1) AS presence -FROM main.store_sales, - main.store_returns, - cs_ui, - main.date_dim d1, - main.date_dim d2, - main.date_dim d3, - main.store, - main.customer, - main.customer_demographics cd1, - main.customer_demographics cd2, - main.promotion, - main.household_demographics hd1, - main.household_demographics hd2, - main.customer_address ad1, - main.customer_address ad2, - main.income_band ib1, - main.income_band ib2, - main.item, (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE sr_customer_sk=asa_owner0.asa_key0 AND ss_store_sk = s_store_sk - AND ss_sold_date_sk = d1.d_date_sk - AND ss_customer_sk = c_customer_sk - AND ss_cdemo_sk= cd1.cd_demo_sk - AND ss_hdemo_sk = hd1.hd_demo_sk - AND ss_addr_sk = ad1.ca_address_sk - AND ss_item_sk = i_item_sk - AND ss_item_sk = sr_item_sk - AND ss_ticket_number = sr_ticket_number - AND ss_item_sk = cs_ui.cs_item_sk - AND c_current_cdemo_sk = cd2.cd_demo_sk - AND c_current_hdemo_sk = hd2.hd_demo_sk - AND c_current_addr_sk = ad2.ca_address_sk - AND c_first_sales_date_sk = d2.d_date_sk - AND c_first_shipto_date_sk = d3.d_date_sk - AND ss_promo_sk = p_promo_sk - AND hd1.hd_income_band_sk = ib1.ib_income_band_sk - AND hd2.hd_income_band_sk = ib2.ib_income_band_sk - AND cd1.cd_marital_status <> cd2.cd_marital_status - AND i_color IN ('purple', - 'burlywood', - 'indian', - 'spring', - 'floral', - 'medium') - AND i_current_price BETWEEN 64 AND 64 + 10 - AND i_current_price BETWEEN 64 + 1 AND 64 + 15 - GROUP BY i_product_name, - i_item_sk, - s_store_name, - s_zip, - ad1.ca_street_number, - ad1.ca_street_name, - ad1.ca_city, - ad1.ca_zip, - ad2.ca_street_number, - ad2.ca_street_name, - ad2.ca_city, - ad2.ca_zip, - d1.d_year, - d2.d_year, - d3.d_year -), -cross_sales AS ( -SELECT world, product_name, item_sk, store_name, store_zip, b_street_number, b_street_name, b_city, b_zip, c_street_number, c_street_name, c_city, c_zip, syear, fsyear, s2year, cnt[world+1] AS cnt, s1[world+1] AS s1, s2[world+1] AS s2, s3[world+1] AS s3 -FROM cross_sales_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT cs1.world AS world, cs1.product_name, - cs1.store_name, - cs1.store_zip, - cs1.b_street_number, - cs1.b_street_name, - cs1.b_city, - cs1.b_zip, - cs1.c_street_number, - cs1.c_street_name, - cs1.c_city, - cs1.c_zip, - cs1.syear cs1syear, - cs1.cnt cs1cnt, - cs1.s1 AS s11, - cs1.s2 AS s21, - cs1.s3 AS s31, - cs2.s1 AS s12, - cs2.s2 AS s22, - cs2.s3 AS s32, - cs2.syear, - cs2.cnt -FROM cross_sales cs1, - cross_sales cs2 -WHERE cs1.world=cs2.world AND cs1.item_sk=cs2.item_sk - AND cs1.syear = 1999 - AND cs2.syear = 1999 + 1 - AND cs2.cnt <= cs1.cnt - AND cs1.store_name = cs2.store_name - AND cs1.store_zip = cs2.store_zip -ORDER BY cs1.world, cs1.product_name, - cs1.store_name, - cs2.cnt, - cs1.s1, - cs2.s1; diff --git a/benchmark/tpcds/reference_queries/q65.sql b/benchmark/tpcds/reference_queries/q65.sql deleted file mode 100644 index f3d32ffe..00000000 --- a/benchmark/tpcds/reference_queries/q65.sql +++ /dev/null @@ -1,17 +0,0 @@ --- Inner item revenues are shared across both outer uses; AVG remains world-specific. -WITH revenue_stats AS ( -SELECT ss_store_sk, ss_item_sk, tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_sales_price) AS revenue, - tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence -FROM main.store_sales, main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk=d_date_sk AND d_month_seq BETWEEN 1176 AND 1176+11 -GROUP BY ss_store_sk, ss_item_sk -), sc AS ( -SELECT world, ss_store_sk, ss_item_sk, revenue[world+1] AS revenue -FROM revenue_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), sb AS (SELECT world, ss_store_sk, avg(revenue) AS ave FROM sc GROUP BY world, ss_store_sk) -SELECT sc.world AS world, s_store_name, i_item_desc, sc.revenue, i_current_price, i_wholesale_cost, i_brand -FROM main.store, main.item, sb, sc -WHERE sc.world=sb.world AND sb.ss_store_sk=sc.ss_store_sk AND sc.revenue<=0.1*sb.ave AND s_store_sk=sc.ss_store_sk AND i_item_sk=sc.ss_item_sk -QUALIFY row_number() OVER (PARTITION BY sc.world ORDER BY s_store_name NULLS FIRST, i_item_desc NULLS FIRST) <= 100 -ORDER BY sc.world, s_store_name NULLS FIRST, i_item_desc NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q66.sql b/benchmark/tpcds/reference_queries/q66.sql deleted file mode 100644 index b33e2bdf..00000000 --- a/benchmark/tpcds/reference_queries/q66.sql +++ /dev/null @@ -1,227 +0,0 @@ --- Shared monthly AS aggregates retain each channel's original price/profit expression. -WITH stats AS ( - - select - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,'DHL,BARIAN' as ship_carriers - ,d_year as year_ - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 1 - then ws_ext_sales_price* ws_quantity else 0 end) as jan_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 2 - then ws_ext_sales_price* ws_quantity else 0 end) as feb_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 3 - then ws_ext_sales_price* ws_quantity else 0 end) as mar_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 4 - then ws_ext_sales_price* ws_quantity else 0 end) as apr_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 5 - then ws_ext_sales_price* ws_quantity else 0 end) as may_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 6 - then ws_ext_sales_price* ws_quantity else 0 end) as jun_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 7 - then ws_ext_sales_price* ws_quantity else 0 end) as jul_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 8 - then ws_ext_sales_price* ws_quantity else 0 end) as aug_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 9 - then ws_ext_sales_price* ws_quantity else 0 end) as sep_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 10 - then ws_ext_sales_price* ws_quantity else 0 end) as oct_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 11 - then ws_ext_sales_price* ws_quantity else 0 end) as nov_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 12 - then ws_ext_sales_price* ws_quantity else 0 end) as dec_sales - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 1 - then ws_net_paid * ws_quantity else 0 end) as jan_net - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 2 - then ws_net_paid * ws_quantity else 0 end) as feb_net - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 3 - then ws_net_paid * ws_quantity else 0 end) as mar_net - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 4 - then ws_net_paid * ws_quantity else 0 end) as apr_net - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 5 - then ws_net_paid * ws_quantity else 0 end) as may_net - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 6 - then ws_net_paid * ws_quantity else 0 end) as jun_net - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 7 - then ws_net_paid * ws_quantity else 0 end) as jul_net - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 8 - then ws_net_paid * ws_quantity else 0 end) as aug_net - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 9 - then ws_net_paid * ws_quantity else 0 end) as sep_net - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 10 - then ws_net_paid * ws_quantity else 0 end) as oct_net - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 11 - then ws_net_paid * ws_quantity else 0 end) as nov_net - ,tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), case when d_moy = 12 - then ws_net_paid * ws_quantity else 0 end) as dec_net, - tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence -from - main.web_sales - ,main.warehouse - ,main.date_dim - ,main.time_dim - ,main.ship_mode, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_warehouse_sk = w_warehouse_sk - and ws_sold_date_sk = d_date_sk - and ws_sold_time_sk = t_time_sk - and ws_ship_mode_sk = sm_ship_mode_sk - and d_year = 2001 - and t_time between 30838 and 30838+28800 - and sm_carrier in ('DHL','BARIAN') - group by - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,d_year - -UNION ALL - - select - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,'DHL,BARIAN' as ship_carriers - ,d_year as year_ - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 1 - then cs_sales_price* cs_quantity else 0 end) as jan_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 2 - then cs_sales_price* cs_quantity else 0 end) as feb_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 3 - then cs_sales_price* cs_quantity else 0 end) as mar_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 4 - then cs_sales_price* cs_quantity else 0 end) as apr_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 5 - then cs_sales_price* cs_quantity else 0 end) as may_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 6 - then cs_sales_price* cs_quantity else 0 end) as jun_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 7 - then cs_sales_price* cs_quantity else 0 end) as jul_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 8 - then cs_sales_price* cs_quantity else 0 end) as aug_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 9 - then cs_sales_price* cs_quantity else 0 end) as sep_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 10 - then cs_sales_price* cs_quantity else 0 end) as oct_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 11 - then cs_sales_price* cs_quantity else 0 end) as nov_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 12 - then cs_sales_price* cs_quantity else 0 end) as dec_sales - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 1 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as jan_net - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 2 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as feb_net - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 3 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as mar_net - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 4 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as apr_net - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 5 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as may_net - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 6 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as jun_net - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 7 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as jul_net - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 8 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as aug_net - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 9 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as sep_net - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 10 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as oct_net - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 11 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as nov_net - ,tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), case when d_moy = 12 - then cs_net_paid_inc_tax * cs_quantity else 0 end) as dec_net, - tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence -from - main.catalog_sales - ,main.warehouse - ,main.date_dim - ,main.time_dim - ,main.ship_mode, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_warehouse_sk = w_warehouse_sk - and cs_sold_date_sk = d_date_sk - and cs_sold_time_sk = t_time_sk - and cs_ship_mode_sk = sm_ship_mode_sk - and d_year = 2001 - and t_time between 30838 AND 30838+28800 - and sm_carrier in ('DHL','BARIAN') - group by - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,d_year - -), expanded AS ( -SELECT world, w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, ship_carriers, year_, jan_sales[world+1] AS jan_sales, feb_sales[world+1] AS feb_sales, mar_sales[world+1] AS mar_sales, apr_sales[world+1] AS apr_sales, may_sales[world+1] AS may_sales, jun_sales[world+1] AS jun_sales, jul_sales[world+1] AS jul_sales, aug_sales[world+1] AS aug_sales, sep_sales[world+1] AS sep_sales, oct_sales[world+1] AS oct_sales, nov_sales[world+1] AS nov_sales, dec_sales[world+1] AS dec_sales, jan_net[world+1] AS jan_net, feb_net[world+1] AS feb_net, mar_net[world+1] AS mar_net, apr_net[world+1] AS apr_net, may_net[world+1] AS may_net, jun_net[world+1] AS jun_net, jul_net[world+1] AS jul_net, aug_net[world+1] AS aug_net, sep_net[world+1] AS sep_net, oct_net[world+1] AS oct_net, nov_net[world+1] AS nov_net, dec_net[world+1] AS dec_net -FROM stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), answers AS ( -select world, - w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,ship_carriers - ,year_ - ,sum(jan_sales) as jan_sales - ,sum(feb_sales) as feb_sales - ,sum(mar_sales) as mar_sales - ,sum(apr_sales) as apr_sales - ,sum(may_sales) as may_sales - ,sum(jun_sales) as jun_sales - ,sum(jul_sales) as jul_sales - ,sum(aug_sales) as aug_sales - ,sum(sep_sales) as sep_sales - ,sum(oct_sales) as oct_sales - ,sum(nov_sales) as nov_sales - ,sum(dec_sales) as dec_sales - ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot - ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot - ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot - ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot - ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot - ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot - ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot - ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot - ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot - ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot - ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot - ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot - ,sum(jan_net) as jan_net - ,sum(feb_net) as feb_net - ,sum(mar_net) as mar_net - ,sum(apr_net) as apr_net - ,sum(may_net) as may_net - ,sum(jun_net) as jun_net - ,sum(jul_net) as jul_net - ,sum(aug_net) as aug_net - ,sum(sep_net) as sep_net - ,sum(oct_net) as oct_net - ,sum(nov_net) as nov_net - ,sum(dec_net) as dec_net - from expanded x - GROUP BY world, w_warehouse_name - ,w_warehouse_sq_ft - ,w_city - ,w_county - ,w_state - ,w_country - ,ship_carriers - ,year_ - -) -SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY w_warehouse_name NULLS FIRST)<=100 ORDER BY world,w_warehouse_name NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q68.sql b/benchmark/tpcds/reference_queries/q68.sql deleted file mode 100644 index cd722de8..00000000 --- a/benchmark/tpcds/reference_queries/q68.sql +++ /dev/null @@ -1,56 +0,0 @@ --- AS aggregation is shared; only the reduced per-ticket results are expanded by world. -WITH stats AS ( - SELECT ss_ticket_number, - ss_customer_sk, - ca_city AS bought_city, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS extended_price, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_list_price) AS list_price, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_tax) AS extended_tax, - tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS presence - FROM main.store_sales, - main.date_dim, - main.store, - main.household_demographics, - main.customer_address, - (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key - AND store_sales.ss_sold_date_sk = date_dim.d_date_sk - AND store_sales.ss_store_sk = store.s_store_sk - AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - AND store_sales.ss_addr_sk = customer_address.ca_address_sk - AND date_dim.d_dom BETWEEN 1 AND 2 - AND (household_demographics.hd_dep_count = 4 - OR household_demographics.hd_vehicle_count= 3) - AND date_dim.d_year IN (1999, - 1999+1, - 1999+2) - AND store.s_city IN ('Fairview', - 'Midway') - GROUP BY ss_ticket_number, - ss_customer_sk, - ss_addr_sk, - ca_city -), expanded AS ( - SELECT world, ss_ticket_number, ss_customer_sk, bought_city, extended_price[world+1] AS extended_price, list_price[world+1] AS list_price, extended_tax[world+1] AS extended_tax - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1]>0 -) -SELECT world, c_last_name, - c_first_name, - ca_city, - bought_city, - ss_ticket_number, - extended_price, - extended_tax, - list_price -FROM - expanded dn, - main.customer, - main.customer_address current_addr -WHERE ss_customer_sk = c_customer_sk - AND customer.c_current_addr_sk = current_addr.ca_address_sk - AND current_addr.ca_city <> bought_city -QUALIFY row_number() OVER (PARTITION BY world ORDER BY c_last_name NULLS FIRST, - ss_ticket_number NULLS FIRST) <= 100 -ORDER BY world, c_last_name NULLS FIRST, - ss_ticket_number NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q69.sql b/benchmark/tpcds/reference_queries/q69.sql deleted file mode 100644 index 3d68f40d..00000000 --- a/benchmark/tpcds/reference_queries/q69.sql +++ /dev/null @@ -1,49 +0,0 @@ --- Same-owner EXISTS is static after entity inclusion; shipping-customer EXISTS needs the bill-owner mask. -WITH catalog_presence AS ( -SELECT cs_ship_customer_sk,bit_or(priv_hash(hash(cs_bill_customer_sk))) AS membership - FROM main.catalog_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy BETWEEN 4 AND 4+2 -GROUP BY cs_ship_customer_sk -), stats AS (SELECT cd_gender, -cd_marital_status, -cd_education_status, -cd_purchase_estimate, -cd_credit_rating, -tpcds_count((priv_hash(hash(c.c_customer_sk)) & ~coalesce(catalog_presence.membership,0::UBIGINT)),1) AS n -FROM main.customer c CROSS JOIN - main.customer_address ca CROSS JOIN - main.customer_demographics LEFT JOIN catalog_presence ON c.c_customer_sk=cs_ship_customer_sk -WHERE c.c_current_addr_sk = ca.ca_address_sk - AND ca_state IN ('KY', - 'GA', - 'NM') - AND cd_demo_sk = c.c_current_cdemo_sk - -AND EXISTS(SELECT * - FROM main.store_sales, - main.date_dim - WHERE c.c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy BETWEEN 4 AND 4+2) -AND NOT EXISTS(SELECT * - FROM main.web_sales, - main.date_dim - WHERE c.c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year = 2001 - AND d_moy BETWEEN 4 AND 4+2) -GROUP BY cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating -), answers AS (SELECT world, cd_gender, cd_marital_status, cd_education_status, n[world+1] AS cnt1, cd_purchase_estimate, n[world+1] AS cnt2, cd_credit_rating, n[world+1] AS cnt3 FROM stats CROSS JOIN range(64) worlds(world) WHERE n[world+1]>0) -SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating)<=100 ORDER BY world,cd_gender, - cd_marital_status, - cd_education_status, - cd_purchase_estimate, - cd_credit_rating; diff --git a/benchmark/tpcds/reference_queries/q72.sql b/benchmark/tpcds/reference_queries/q72.sql deleted file mode 100644 index 60a704ad..00000000 --- a/benchmark/tpcds/reference_queries/q72.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Matched and NULL-extended memberships are disjoint; NULL extension is evaluated separately in each world. -WITH left_input AS ( -SELECT i_item_desc,w_warehouse_name,d1.d_week_seq,p_promo_sk,cs_order_number,cs_item_sk,priv_hash(hash(cs_bill_customer_sk)) AS membership -FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk -JOIN main.inventory ON cs_item_sk=inv_item_sk JOIN main.warehouse ON w_warehouse_sk=inv_warehouse_sk -JOIN main.item ON i_item_sk=cs_item_sk JOIN main.customer_demographics ON cs_bill_cdemo_sk=cd_demo_sk -JOIN main.household_demographics ON cs_bill_hdemo_sk=hd_demo_sk -JOIN main.date_dim d1 ON cs_sold_date_sk=d1.d_date_sk JOIN main.date_dim d2 ON inv_date_sk=d2.d_date_sk JOIN main.date_dim d3 ON cs_ship_date_sk=d3.d_date_sk -LEFT JOIN main.promotion ON cs_promo_sk=p_promo_sk -WHERE d1.d_week_seq=d2.d_week_seq AND inv_quantity_on_handd1.d_date+5 -AND hd_buy_potential='>10000' AND d1.d_year=1999 AND cd_marital_status='D' -), valid_returns AS (SELECT r.*,priv_hash(hash(cr_returning_customer_sk)) AS membership FROM main.catalog_returns r JOIN main.customer ON cr_returning_customer_sk=c_customer_sk), -return_presence AS (SELECT cr_order_number,cr_item_sk,bit_or(membership) AS membership FROM valid_returns GROUP BY cr_order_number,cr_item_sk), -joined AS ( - SELECT l.i_item_desc,l.w_warehouse_name,l.d_week_seq,l.p_promo_sk,(l.membership & r.membership) AS membership FROM left_input l JOIN valid_returns r ON l.cs_order_number=r.cr_order_number AND l.cs_item_sk=r.cr_item_sk - UNION ALL - SELECT l.i_item_desc,l.w_warehouse_name,l.d_week_seq,l.p_promo_sk,(l.membership & ~coalesce(r.membership,0::UBIGINT)) AS membership FROM left_input l LEFT JOIN return_presence r ON l.cs_order_number=r.cr_order_number AND l.cs_item_sk=r.cr_item_sk -), stats AS (SELECT i_item_desc,w_warehouse_name,d_week_seq,tpcds_sum(membership,CASE WHEN p_promo_sk IS NULL THEN 1 ELSE 0 END) AS no_promo,tpcds_sum(membership,CASE WHEN p_promo_sk IS NOT NULL THEN 1 ELSE 0 END) AS promo,tpcds_count(membership,1) AS total_cnt,tpcds_count(membership,1) AS presence FROM joined GROUP BY i_item_desc,w_warehouse_name,d_week_seq), -answers AS (SELECT world,i_item_desc,w_warehouse_name,d_week_seq,no_promo[world+1] AS no_promo,promo[world+1] AS promo,total_cnt[world+1] AS total_cnt FROM stats CROSS JOIN range(64) worlds(world) WHERE presence[world+1]>0) -SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY total_cnt DESC NULLS FIRST,i_item_desc NULLS FIRST,w_warehouse_name NULLS FIRST,d_week_seq NULLS FIRST)<=100 ORDER BY world,total_cnt DESC NULLS FIRST,i_item_desc NULLS FIRST,w_warehouse_name NULLS FIRST,d_week_seq NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q73.sql b/benchmark/tpcds/reference_queries/q73.sql deleted file mode 100644 index b4e01e9b..00000000 --- a/benchmark/tpcds/reference_queries/q73.sql +++ /dev/null @@ -1,50 +0,0 @@ --- AS aggregation is shared; only the reduced per-ticket results are expanded by world. -WITH stats AS ( - SELECT ss_ticket_number, - ss_customer_sk, - tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS cnt, - tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS presence - FROM main.store_sales, - main.date_dim, - main.store, - main.household_demographics, - (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key - AND store_sales.ss_sold_date_sk = date_dim.d_date_sk - AND store_sales.ss_store_sk = store.s_store_sk - AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - AND date_dim.d_dom BETWEEN 1 AND 2 - AND (household_demographics.hd_buy_potential = 'Unknown' - OR household_demographics.hd_buy_potential = '>10000') - AND household_demographics.hd_vehicle_count > 0 - AND CASE - WHEN household_demographics.hd_vehicle_count > 0 THEN (household_demographics.hd_dep_count*1.000)/ household_demographics.hd_vehicle_count - ELSE NULL - END > 1 - AND date_dim.d_year IN (1999, - 1999+1, - 1999+2) - AND store.s_county IN ('Orange County', - 'Bronx County', - 'Franklin Parish', - 'Williamson County') - GROUP BY ss_ticket_number, - ss_customer_sk -), expanded AS ( - SELECT world, ss_ticket_number, ss_customer_sk, cnt[world+1] AS cnt - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1]>0 -) -SELECT world, c_last_name, - c_first_name, - c_salutation, - c_preferred_cust_flag, - ss_ticket_number, - cnt -FROM - expanded dj, - main.customer -WHERE ss_customer_sk = c_customer_sk - AND cnt BETWEEN 1 AND 5 -ORDER BY world, cnt DESC, - c_last_name ASC; diff --git a/benchmark/tpcds/reference_queries/q74.sql b/benchmark/tpcds/reference_queries/q74.sql deleted file mode 100644 index 57335d54..00000000 --- a/benchmark/tpcds/reference_queries/q74.sql +++ /dev/null @@ -1,74 +0,0 @@ --- Shared AS states per channel/customer/year; joins align the world index. -WITH year_total_stats AS ( -SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - d_year AS year_, - tpcds_sum(priv_hash(hash(c_customer_sk)), ss_net_paid) year_total, - 's' sale_type, - tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence -FROM main.customer, - main.store_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ss_customer_sk - AND ss_sold_date_sk = d_date_sk - AND d_year IN (2001, - 2001+1) - GROUP BY c_customer_id, - c_first_name, - c_last_name, - d_year - -UNION ALL - SELECT c_customer_id customer_id, - c_first_name customer_first_name, - c_last_name customer_last_name, - d_year AS year_, - tpcds_sum(priv_hash(hash(c_customer_sk)), ws_net_paid) year_total, - 'w' sale_type, - tpcds_count(priv_hash(hash(c_customer_sk)),1) AS presence -FROM main.customer, - main.web_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE c_customer_sk=asa_owner.asa_key AND c_customer_sk = ws_bill_customer_sk - AND ws_sold_date_sk = d_date_sk - AND d_year IN (2001, - 2001+1) - GROUP BY c_customer_id, - c_first_name, - c_last_name, - d_year -), year_total AS ( -SELECT world, customer_id, customer_first_name, customer_last_name, year_, year_total[world+1] AS year_total, sale_type -FROM year_total_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT t_s_firstyear.world AS world, t_s_secyear.customer_id, - t_s_secyear.customer_first_name, - t_s_secyear.customer_last_name -FROM year_total t_s_firstyear, - year_total t_s_secyear, - year_total t_w_firstyear, - year_total t_w_secyear -WHERE t_s_firstyear.world=t_s_secyear.world AND t_s_firstyear.world=t_w_firstyear.world AND t_s_firstyear.world=t_w_secyear.world AND t_s_secyear.customer_id = t_s_firstyear.customer_id - AND t_s_firstyear.customer_id = t_w_secyear.customer_id - AND t_s_firstyear.customer_id = t_w_firstyear.customer_id - AND t_s_firstyear.sale_type = 's' - AND t_w_firstyear.sale_type = 'w' - AND t_s_secyear.sale_type = 's' - AND t_w_secyear.sale_type = 'w' - AND t_s_firstyear.year_ = 2001 - AND t_s_secyear.year_ = 2001+1 - AND t_w_firstyear.year_ = 2001 - AND t_w_secyear.year_ = 2001+1 - AND t_s_firstyear.year_total > 0 - AND t_w_firstyear.year_total > 0 - AND CASE - WHEN t_w_firstyear.year_total > 0 THEN t_w_secyear.year_total / t_w_firstyear.year_total - ELSE NULL - END > CASE - WHEN t_s_firstyear.year_total > 0 THEN t_s_secyear.year_total / t_s_firstyear.year_total - ELSE NULL - END -QUALIFY row_number() OVER (PARTITION BY t_s_firstyear.world ORDER BY t_s_secyear.customer_id NULLS FIRST) <= 100 -ORDER BY t_s_firstyear.world, t_s_secyear.customer_id NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q77.sql b/benchmark/tpcds/reference_queries/q77.sql deleted file mode 100644 index 23ef4544..00000000 --- a/benchmark/tpcds/reference_queries/q77.sql +++ /dev/null @@ -1,114 +0,0 @@ --- Grouped channel results retain world-local LEFT JOIN semantics and empty-world ROLLUP totals. -WITH ss_stats AS ( -SELECT s_store_sk, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_ext_sales_price) AS sales, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) AS profit, - tpcds_count(priv_hash(hash(ss_customer_sk)),1) AS presence -FROM main.store_sales, - main.date_dim, - main.store, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - AND ss_store_sk = s_store_sk - GROUP BY s_store_sk -), -ss AS ( -SELECT world, s_store_sk, sales[world+1] AS sales, profit[world+1] AS profit -FROM ss_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -sr_stats AS ( -SELECT s_store_sk, - tpcds_sum(priv_hash(hash(sr_customer_sk)), sr_return_amt) AS returns_, - tpcds_sum(priv_hash(hash(sr_customer_sk)), sr_net_loss) AS profit_loss, - tpcds_count(priv_hash(hash(sr_customer_sk)),1) AS presence -FROM main.store_returns, - main.date_dim, - main.store, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE sr_customer_sk=asa_owner.asa_key AND sr_returned_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - AND sr_store_sk = s_store_sk - GROUP BY s_store_sk -), -sr AS ( -SELECT world, s_store_sk, returns_[world+1] AS returns_, profit_loss[world+1] AS profit_loss -FROM sr_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -cs_stats AS ( -SELECT cs_call_center_sk, - tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_ext_sales_price) AS sales, - tpcds_sum(priv_hash(hash(cs_bill_customer_sk)), cs_net_profit) AS profit, - tpcds_count(priv_hash(hash(cs_bill_customer_sk)),1) AS presence -FROM main.catalog_sales, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cs_bill_customer_sk=asa_owner.asa_key AND cs_sold_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - GROUP BY cs_call_center_sk -), -cs AS ( -SELECT world, cs_call_center_sk, sales[world+1] AS sales, profit[world+1] AS profit -FROM cs_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -cr_stats AS ( -SELECT cr_call_center_sk, - tpcds_sum(priv_hash(hash(cr_returning_customer_sk)), cr_return_amount) AS returns_, - tpcds_sum(priv_hash(hash(cr_returning_customer_sk)), cr_net_loss) AS profit_loss, - tpcds_count(priv_hash(hash(cr_returning_customer_sk)),1) AS presence -FROM main.catalog_returns, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cr_returning_customer_sk=asa_owner.asa_key AND cr_returned_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - GROUP BY cr_call_center_sk -), -cr AS ( -SELECT world, cr_call_center_sk, returns_[world+1] AS returns_, profit_loss[world+1] AS profit_loss -FROM cr_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -ws_stats AS ( -SELECT wp_web_page_sk, - tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_ext_sales_price) AS sales, - tpcds_sum(priv_hash(hash(ws_bill_customer_sk)), ws_net_profit) AS profit, - tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) AS presence -FROM main.web_sales, - main.date_dim, - main.web_page, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_sold_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - AND ws_web_page_sk = wp_web_page_sk - GROUP BY wp_web_page_sk -), -ws AS ( -SELECT world, wp_web_page_sk, sales[world+1] AS sales, profit[world+1] AS profit -FROM ws_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -wr_stats AS ( -SELECT wp_web_page_sk, - tpcds_sum(priv_hash(hash(wr_returning_customer_sk)), wr_return_amt) AS returns_, - tpcds_sum(priv_hash(hash(wr_returning_customer_sk)), wr_net_loss) AS profit_loss, - tpcds_count(priv_hash(hash(wr_returning_customer_sk)),1) AS presence -FROM main.web_returns, - main.date_dim, - main.web_page, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE wr_returning_customer_sk=asa_owner.asa_key AND wr_returned_date_sk = d_date_sk - AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) - AND wr_web_page_sk = wp_web_page_sk - GROUP BY wp_web_page_sk -), -wr AS ( -SELECT world, wp_web_page_sk, returns_[world+1] AS returns_, profit_loss[world+1] AS profit_loss -FROM wr_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -channels AS ( -SELECT ss.world,'store channel' AS channel,ss.s_store_sk AS id,sales,coalesce(returns_,0) AS returns_,profit-coalesce(profit_loss,0) AS profit FROM ss LEFT JOIN sr ON ss.world=sr.world AND ss.s_store_sk=sr.s_store_sk -UNION ALL SELECT cs.world,'catalog channel',cs_call_center_sk,sales,returns_,profit-profit_loss FROM cs JOIN cr ON cs.world=cr.world -UNION ALL SELECT ws.world,'web channel',ws.wp_web_page_sk,sales,coalesce(returns_,0),profit-coalesce(profit_loss,0) FROM ws LEFT JOIN wr ON ws.world=wr.world AND ws.wp_web_page_sk=wr.wp_web_page_sk -), -answers AS ( -SELECT w.world,g.* FROM range(64) w(world) CROSS JOIN LATERAL (SELECT channel,id,sum(sales) AS sales,sum(returns_) AS returns_,sum(profit) AS profit FROM channels WHERE channels.world=w.world GROUP BY ROLLUP(channel,id)) g -) -SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY channel NULLS FIRST,id NULLS FIRST,returns_ DESC)<=100 ORDER BY world,channel NULLS FIRST,id NULLS FIRST,returns_ DESC; diff --git a/benchmark/tpcds/reference_queries/q78.sql b/benchmark/tpcds/reference_queries/q78.sql deleted file mode 100644 index 0efa3403..00000000 --- a/benchmark/tpcds/reference_queries/q78.sql +++ /dev/null @@ -1,80 +0,0 @@ --- Return anti-memberships feed the per-customer/item/year AS sums before channel joins. -WITH ws_return_presence AS (SELECT wr_order_number AS order_number,wr_item_sk AS item_sk,bit_or(priv_hash(hash(wr_returning_customer_sk))) AS membership FROM main.web_returns JOIN main.customer ON wr_returning_customer_sk=c_customer_sk GROUP BY wr_order_number,wr_item_sk), -ws_stats AS ( -SELECT d_year AS ws_sold_year,ws_item_sk,ws_bill_customer_sk AS ws_customer_sk, - tpcds_sum((priv_hash(hash(ws_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ws_quantity) AS ws_qty,tpcds_sum((priv_hash(hash(ws_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ws_wholesale_cost) AS ws_wc,tpcds_sum((priv_hash(hash(ws_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ws_sales_price) AS ws_sp,tpcds_count((priv_hash(hash(ws_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),1) AS presence -FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk JOIN main.date_dim ON ws_sold_date_sk=d_date_sk -LEFT JOIN ws_return_presence ret ON ws_order_number=ret.order_number AND ws_item_sk=ret.item_sk -GROUP BY d_year,ws_item_sk,ws_bill_customer_sk -), -ws AS ( -SELECT world, ws_sold_year, ws_item_sk, ws_customer_sk, ws_qty[world+1] AS ws_qty, ws_wc[world+1] AS ws_wc, ws_sp[world+1] AS ws_sp -FROM ws_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -cs_return_presence AS (SELECT cr_order_number AS order_number,cr_item_sk AS item_sk,bit_or(priv_hash(hash(cr_returning_customer_sk))) AS membership FROM main.catalog_returns JOIN main.customer ON cr_returning_customer_sk=c_customer_sk GROUP BY cr_order_number,cr_item_sk), -cs_stats AS ( -SELECT d_year AS cs_sold_year,cs_item_sk,cs_bill_customer_sk AS cs_customer_sk, - tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),cs_quantity) AS cs_qty,tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),cs_wholesale_cost) AS cs_wc,tpcds_sum((priv_hash(hash(cs_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),cs_sales_price) AS cs_sp,tpcds_count((priv_hash(hash(cs_bill_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),1) AS presence -FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk JOIN main.date_dim ON cs_sold_date_sk=d_date_sk -LEFT JOIN cs_return_presence ret ON cs_order_number=ret.order_number AND cs_item_sk=ret.item_sk -GROUP BY d_year,cs_item_sk,cs_bill_customer_sk -), -cs AS ( -SELECT world, cs_sold_year, cs_item_sk, cs_customer_sk, cs_qty[world+1] AS cs_qty, cs_wc[world+1] AS cs_wc, cs_sp[world+1] AS cs_sp -FROM cs_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -ss_return_presence AS (SELECT sr_ticket_number AS order_number,sr_item_sk AS item_sk,bit_or(priv_hash(hash(sr_customer_sk))) AS membership FROM main.store_returns JOIN main.customer ON sr_customer_sk=c_customer_sk GROUP BY sr_ticket_number,sr_item_sk), -ss_stats AS ( -SELECT d_year AS ss_sold_year,ss_item_sk,ss_customer_sk AS ss_customer_sk, - tpcds_sum((priv_hash(hash(ss_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ss_quantity) AS ss_qty,tpcds_sum((priv_hash(hash(ss_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ss_wholesale_cost) AS ss_wc,tpcds_sum((priv_hash(hash(ss_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),ss_sales_price) AS ss_sp,tpcds_count((priv_hash(hash(ss_customer_sk)) & ~coalesce(ret.membership,0::UBIGINT)),1) AS presence -FROM main.store_sales JOIN main.customer ON ss_customer_sk=c_customer_sk JOIN main.date_dim ON ss_sold_date_sk=d_date_sk -LEFT JOIN ss_return_presence ret ON ss_ticket_number=ret.order_number AND ss_item_sk=ret.item_sk -GROUP BY d_year,ss_item_sk,ss_customer_sk -), -ss AS ( -SELECT world, ss_sold_year, ss_item_sk, ss_customer_sk, ss_qty[world+1] AS ss_qty, ss_wc[world+1] AS ss_wc, ss_sp[world+1] AS ss_sp -FROM ss_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT ss.world AS world, ss_sold_year, - ss_item_sk, - ss_customer_sk, - round((ss_qty*1.00)/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, - ss_qty store_qty, - ss_wc store_wholesale_cost, - ss_sp store_sales_price, - coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, - coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, - coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price -FROM ss -LEFT JOIN ws ON (ss.world=ws.world AND ws_sold_year=ss_sold_year - AND ws_item_sk=ss_item_sk - AND ws_customer_sk=ss_customer_sk) -LEFT JOIN cs ON (ss.world=cs.world AND cs_sold_year=ss_sold_year - AND cs_item_sk=ss_item_sk - AND cs_customer_sk=ss_customer_sk) -WHERE (coalesce(ws_qty,0)>0 - OR coalesce(cs_qty, 0)>0) - AND ss_sold_year=2000 -QUALIFY row_number() OVER (PARTITION BY ss.world ORDER BY ss_sold_year, - ss_item_sk, - ss_customer_sk, - ss_qty DESC, - ss_wc DESC, - ss_sp DESC, - other_chan_qty, - other_chan_wholesale_cost, - other_chan_sales_price, - ratio)<=100 -ORDER BY ss.world,ss_sold_year, - ss_item_sk, - ss_customer_sk, - ss_qty DESC, - ss_wc DESC, - ss_sp DESC, - other_chan_qty, - other_chan_wholesale_cost, - other_chan_sales_price, - ratio; diff --git a/benchmark/tpcds/reference_queries/q79.sql b/benchmark/tpcds/reference_queries/q79.sql deleted file mode 100644 index a22b04be..00000000 --- a/benchmark/tpcds/reference_queries/q79.sql +++ /dev/null @@ -1,53 +0,0 @@ --- AS aggregation is shared; only the reduced per-ticket results are expanded by world. -WITH stats AS ( - SELECT ss_ticket_number, - ss_customer_sk, - store.s_city AS s_city, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_coupon_amt) AS amt, - tpcds_sum(priv_hash(hash(ss_customer_sk)), ss_net_profit) AS profit, - tpcds_count(priv_hash(hash(ss_customer_sk)), 1) AS presence - FROM main.store_sales, - main.date_dim, - main.store, - main.household_demographics, - (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key - AND store_sales.ss_sold_date_sk = date_dim.d_date_sk - AND store_sales.ss_store_sk = store.s_store_sk - AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk - AND (household_demographics.hd_dep_count = 6 - OR household_demographics.hd_vehicle_count > 2) - AND date_dim.d_dow = 1 - AND date_dim.d_year IN (1999, - 1999+1, - 1999+2) - AND store.s_number_employees BETWEEN 200 AND 295 - GROUP BY ss_ticket_number, - ss_customer_sk, - ss_addr_sk, - main.store.s_city -), expanded AS ( - SELECT world, ss_ticket_number, ss_customer_sk, s_city, amt[world+1] AS amt, profit[world+1] AS profit - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1]>0 -) -SELECT world, c_last_name, - c_first_name, - SUBSTRING(s_city,1,30), - ss_ticket_number, - amt, - profit -FROM - expanded ms, - main.customer -WHERE ss_customer_sk = c_customer_sk -QUALIFY row_number() OVER (PARTITION BY world ORDER BY c_last_name NULLS FIRST, - c_first_name NULLS FIRST, - SUBSTRING(s_city,1,30) NULLS FIRST, - profit NULLS FIRST, - ss_ticket_number) <= 100 -ORDER BY world, c_last_name NULLS FIRST, - c_first_name NULLS FIRST, - SUBSTRING(s_city,1,30) NULLS FIRST, - profit NULLS FIRST, - ss_ticket_number; diff --git a/benchmark/tpcds/reference_queries/q80.sql b/benchmark/tpcds/reference_queries/q80.sql deleted file mode 100644 index 5fdeacf0..00000000 --- a/benchmark/tpcds/reference_queries/q80.sql +++ /dev/null @@ -1,58 +0,0 @@ --- Matched/fallback row masks preserve sampled outer joins before each channel's AS aggregates. -WITH ssr_left AS ( -SELECT s_store_id AS store_id,ss_ticket_number AS order_number,ss_item_sk AS item_sk,ss_ext_sales_price AS sales_price,ss_net_profit AS net_profit,priv_hash(hash(ss_customer_sk)) AS membership -FROM main.store_sales JOIN main.customer ON ss_customer_sk=c_customer_sk JOIN main.date_dim ON ss_sold_date_sk=d_date_sk -JOIN main.store ON ss_store_sk=s_store_sk JOIN main.item ON ss_item_sk=i_item_sk JOIN main.promotion ON ss_promo_sk=p_promo_sk -WHERE d_date BETWEEN DATE '2000-08-23' AND DATE '2000-09-22' AND i_current_price>50 AND p_channel_tv='N' -), -ssr_returns AS (SELECT sr_ticket_number AS order_number,sr_item_sk AS item_sk,sr_return_amt AS amount,sr_net_loss AS loss,priv_hash(hash(sr_customer_sk)) AS membership FROM main.store_returns JOIN main.customer ON sr_customer_sk=c_customer_sk), -ssr_return_presence AS (SELECT order_number,item_sk,bit_or(membership) AS membership FROM ssr_returns GROUP BY order_number,item_sk), -ssr_joined AS ( -SELECT l.store_id,l.sales_price,l.net_profit,r.amount,r.loss,(l.membership & r.membership) AS membership FROM ssr_left l JOIN ssr_returns r ON l.order_number=r.order_number AND l.item_sk=r.item_sk -UNION ALL SELECT l.store_id,l.sales_price,l.net_profit,0 AS amount,0 AS loss,(l.membership & ~coalesce(r.membership,0::UBIGINT)) AS membership FROM ssr_left l LEFT JOIN ssr_return_presence r ON l.order_number=r.order_number AND l.item_sk=r.item_sk -), -ssr_stats AS (SELECT store_id,tpcds_sum(membership,sales_price) AS sales,tpcds_sum(membership,coalesce(amount,0)) AS returns_,tpcds_sum(membership,net_profit-coalesce(loss,0)) AS profit,tpcds_count(membership,1) AS presence FROM ssr_joined GROUP BY store_id), -ssr AS ( -SELECT world, store_id, sales[world+1] AS sales, returns_[world+1] AS returns_, profit[world+1] AS profit -FROM ssr_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -csr_left AS ( -SELECT cp_catalog_page_id AS catalog_page_id,cs_order_number AS order_number,cs_item_sk AS item_sk,cs_ext_sales_price AS sales_price,cs_net_profit AS net_profit,priv_hash(hash(cs_bill_customer_sk)) AS membership -FROM main.catalog_sales JOIN main.customer ON cs_bill_customer_sk=c_customer_sk JOIN main.date_dim ON cs_sold_date_sk=d_date_sk -JOIN main.catalog_page ON cs_catalog_page_sk=cp_catalog_page_sk JOIN main.item ON cs_item_sk=i_item_sk JOIN main.promotion ON cs_promo_sk=p_promo_sk -WHERE d_date BETWEEN DATE '2000-08-23' AND DATE '2000-09-22' AND i_current_price>50 AND p_channel_tv='N' -), -csr_returns AS (SELECT cr_order_number AS order_number,cr_item_sk AS item_sk,cr_return_amount AS amount,cr_net_loss AS loss,priv_hash(hash(cr_returning_customer_sk)) AS membership FROM main.catalog_returns JOIN main.customer ON cr_returning_customer_sk=c_customer_sk), -csr_return_presence AS (SELECT order_number,item_sk,bit_or(membership) AS membership FROM csr_returns GROUP BY order_number,item_sk), -csr_joined AS ( -SELECT l.catalog_page_id,l.sales_price,l.net_profit,r.amount,r.loss,(l.membership & r.membership) AS membership FROM csr_left l JOIN csr_returns r ON l.order_number=r.order_number AND l.item_sk=r.item_sk -UNION ALL SELECT l.catalog_page_id,l.sales_price,l.net_profit,0 AS amount,0 AS loss,(l.membership & ~coalesce(r.membership,0::UBIGINT)) AS membership FROM csr_left l LEFT JOIN csr_return_presence r ON l.order_number=r.order_number AND l.item_sk=r.item_sk -), -csr_stats AS (SELECT catalog_page_id,tpcds_sum(membership,sales_price) AS sales,tpcds_sum(membership,coalesce(amount,0)) AS returns_,tpcds_sum(membership,net_profit-coalesce(loss,0)) AS profit,tpcds_count(membership,1) AS presence FROM csr_joined GROUP BY catalog_page_id), -csr AS ( -SELECT world, catalog_page_id, sales[world+1] AS sales, returns_[world+1] AS returns_, profit[world+1] AS profit -FROM csr_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -wsr_left AS ( -SELECT web_site_id AS web_site_id,ws_order_number AS order_number,ws_item_sk AS item_sk,ws_ext_sales_price AS sales_price,ws_net_profit AS net_profit,priv_hash(hash(ws_bill_customer_sk)) AS membership -FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk JOIN main.date_dim ON ws_sold_date_sk=d_date_sk -JOIN main.web_site ON ws_web_site_sk=web_site_sk JOIN main.item ON ws_item_sk=i_item_sk JOIN main.promotion ON ws_promo_sk=p_promo_sk -WHERE d_date BETWEEN DATE '2000-08-23' AND DATE '2000-09-22' AND i_current_price>50 AND p_channel_tv='N' -), -wsr_returns AS (SELECT wr_order_number AS order_number,wr_item_sk AS item_sk,wr_return_amt AS amount,wr_net_loss AS loss,priv_hash(hash(wr_returning_customer_sk)) AS membership FROM main.web_returns JOIN main.customer ON wr_returning_customer_sk=c_customer_sk), -wsr_return_presence AS (SELECT order_number,item_sk,bit_or(membership) AS membership FROM wsr_returns GROUP BY order_number,item_sk), -wsr_joined AS ( -SELECT l.web_site_id,l.sales_price,l.net_profit,r.amount,r.loss,(l.membership & r.membership) AS membership FROM wsr_left l JOIN wsr_returns r ON l.order_number=r.order_number AND l.item_sk=r.item_sk -UNION ALL SELECT l.web_site_id,l.sales_price,l.net_profit,0 AS amount,0 AS loss,(l.membership & ~coalesce(r.membership,0::UBIGINT)) AS membership FROM wsr_left l LEFT JOIN wsr_return_presence r ON l.order_number=r.order_number AND l.item_sk=r.item_sk -), -wsr_stats AS (SELECT web_site_id,tpcds_sum(membership,sales_price) AS sales,tpcds_sum(membership,coalesce(amount,0)) AS returns_,tpcds_sum(membership,net_profit-coalesce(loss,0)) AS profit,tpcds_count(membership,1) AS presence FROM wsr_joined GROUP BY web_site_id), -wsr AS ( -SELECT world, web_site_id, sales[world+1] AS sales, returns_[world+1] AS returns_, profit[world+1] AS profit -FROM wsr_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -channels AS (SELECT world,'store channel' AS channel,concat('store',store_id) AS id,sales,returns_,profit FROM ssr UNION ALL SELECT world,'catalog channel',concat('catalog_page',catalog_page_id),sales,returns_,profit FROM csr UNION ALL SELECT world,'web channel',concat('web_site',web_site_id),sales,returns_,profit FROM wsr), -answers AS (SELECT w.world,g.* FROM range(64) w(world) CROSS JOIN LATERAL (SELECT channel,id,sum(sales) AS sales,sum(returns_) AS returns_,sum(profit) AS profit FROM channels WHERE channels.world=w.world GROUP BY ROLLUP(channel,id)) g) -SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY channel NULLS FIRST,id NULLS FIRST)<=100 ORDER BY world,channel NULLS FIRST,id NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q81.sql b/benchmark/tpcds/reference_queries/q81.sql deleted file mode 100644 index ffe67f71..00000000 --- a/benchmark/tpcds/reference_queries/q81.sql +++ /dev/null @@ -1,77 +0,0 @@ --- Correlated thresholds use only the matching world of the reduced return aggregates. -WITH return_stats AS ( -SELECT cr_returning_customer_sk AS ctr_customer_sk , - ca_state AS ctr_state, - tpcds_sum(priv_hash(hash(cr_returning_customer_sk)), cr_return_amt_inc_tax) AS ctr_total_return, - tpcds_count(priv_hash(hash(cr_returning_customer_sk)),1) AS presence -FROM main.catalog_returns , - main.date_dim , - main.customer_address, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cr_returning_customer_sk=asa_owner.asa_key AND cr_returned_date_sk = d_date_sk - AND d_year = 2000 - AND cr_returning_addr_sk = ca_address_sk - GROUP BY cr_returning_customer_sk , - ca_state -), customer_total_return AS ( -SELECT world, ctr_customer_sk, ctr_state, ctr_total_return[world+1] AS ctr_total_return -FROM return_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT ctr1.world AS world, c_customer_id, - c_salutation, - c_first_name, - c_last_name, - ca_street_number, - ca_street_name , - ca_street_type, - ca_suite_number, - ca_city, - ca_county, - ca_state, - ca_zip, - ca_country, - ca_gmt_offset , - ca_location_type, - ctr_total_return -FROM customer_total_return ctr1 , - main.customer_address , - main.customer -WHERE ctr1.ctr_total_return > - (SELECT avg(ctr_total_return)*1.2 - FROM customer_total_return ctr2 - WHERE ctr1.world=ctr2.world AND ctr1.ctr_state = ctr2.ctr_state) - AND ca_address_sk = c_current_addr_sk - AND ca_state = 'GA' - AND ctr1.ctr_customer_sk = c_customer_sk -QUALIFY row_number() OVER (PARTITION BY ctr1.world ORDER BY c_customer_id, - c_salutation, - c_first_name, - c_last_name, - ca_street_number, - ca_street_name , - ca_street_type, - ca_suite_number, - ca_city, - ca_county, - ca_state, - ca_zip, - ca_country, - ca_gmt_offset , - ca_location_type, - ctr_total_return) <= 100 -ORDER BY ctr1.world, c_customer_id, - c_salutation, - c_first_name, - c_last_name, - ca_street_number, - ca_street_name , - ca_street_type, - ca_suite_number, - ca_city, - ca_county, - ca_state, - ca_zip, - ca_country, - ca_gmt_offset , - ca_location_type, - ctr_total_return; diff --git a/benchmark/tpcds/reference_queries/q83.sql b/benchmark/tpcds/reference_queries/q83.sql deleted file mode 100644 index ba388e54..00000000 --- a/benchmark/tpcds/reference_queries/q83.sql +++ /dev/null @@ -1,93 +0,0 @@ --- Each return channel is aggregated once, then joined on item and world. -WITH sr_items_stats AS ( -SELECT i_item_id item_id, - tpcds_sum(priv_hash(hash(sr_customer_sk)), sr_return_quantity) sr_item_qty, - tpcds_count(priv_hash(hash(sr_customer_sk)),1) AS presence -FROM main.store_returns, - main.item, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE sr_customer_sk=asa_owner.asa_key AND sr_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM main.date_dim - WHERE d_week_seq IN - (SELECT d_week_seq - FROM main.date_dim - WHERE d_date IN ('2000-06-30', - '2000-09-27', - '2000-11-17'))) - AND sr_returned_date_sk = d_date_sk - GROUP BY i_item_id -), -sr_items AS ( -SELECT world, item_id, sr_item_qty[world+1] AS sr_item_qty -FROM sr_items_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -cr_items_stats AS ( -SELECT i_item_id item_id, - tpcds_sum(priv_hash(hash(cr_returning_customer_sk)), cr_return_quantity) cr_item_qty, - tpcds_count(priv_hash(hash(cr_returning_customer_sk)),1) AS presence -FROM main.catalog_returns, - main.item, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE cr_returning_customer_sk=asa_owner.asa_key AND cr_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM main.date_dim - WHERE d_week_seq IN - (SELECT d_week_seq - FROM main.date_dim - WHERE d_date IN ('2000-06-30', - '2000-09-27', - '2000-11-17'))) - AND cr_returned_date_sk = d_date_sk - GROUP BY i_item_id -), -cr_items AS ( -SELECT world, item_id, cr_item_qty[world+1] AS cr_item_qty -FROM cr_items_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -), -wr_items_stats AS ( -SELECT i_item_id item_id, - tpcds_sum(priv_hash(hash(wr_returning_customer_sk)), wr_return_quantity) wr_item_qty, - tpcds_count(priv_hash(hash(wr_returning_customer_sk)),1) AS presence -FROM main.web_returns, - main.item, - main.date_dim, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE wr_returning_customer_sk=asa_owner.asa_key AND wr_item_sk = i_item_sk - AND d_date IN - (SELECT d_date - FROM main.date_dim - WHERE d_week_seq IN - (SELECT d_week_seq - FROM main.date_dim - WHERE d_date IN ('2000-06-30', - '2000-09-27', - '2000-11-17'))) - AND wr_returned_date_sk = d_date_sk - GROUP BY i_item_id -), -wr_items AS ( -SELECT world, item_id, wr_item_qty[world+1] AS wr_item_qty -FROM wr_items_stats CROSS JOIN range(64) worlds(world) -WHERE presence[world+1]>0 -) -SELECT sr_items.world AS world, sr_items.item_id , - sr_item_qty , - (sr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 sr_dev , - cr_item_qty , - (cr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 cr_dev , - wr_item_qty , - (wr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 wr_dev , - (sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average -FROM sr_items , - cr_items , - wr_items -WHERE sr_items.world=cr_items.world AND sr_items.world=wr_items.world AND sr_items.item_id=cr_items.item_id - AND sr_items.item_id=wr_items.item_id -QUALIFY row_number() OVER (PARTITION BY sr_items.world ORDER BY sr_items.item_id NULLS FIRST, - sr_item_qty NULLS FIRST) <= 100 -ORDER BY sr_items.world, sr_items.item_id NULLS FIRST, - sr_item_qty NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q85.sql b/benchmark/tpcds/reference_queries/q85.sql deleted file mode 100644 index 746882d7..00000000 --- a/benchmark/tpcds/reference_queries/q85.sql +++ /dev/null @@ -1,68 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT SUBSTRING(r_reason_desc,1,20) AS reason, - tpcds_avg((priv_hash(hash(ws_bill_customer_sk)) & priv_hash(hash(wr_returning_customer_sk))), ws_quantity) AS avg1, - tpcds_avg((priv_hash(hash(ws_bill_customer_sk)) & priv_hash(hash(wr_returning_customer_sk))), wr_refunded_cash) AS avg2, - tpcds_avg((priv_hash(hash(ws_bill_customer_sk)) & priv_hash(hash(wr_returning_customer_sk))), wr_fee) AS avg3, - tpcds_count((priv_hash(hash(ws_bill_customer_sk)) & priv_hash(hash(wr_returning_customer_sk))), 1) AS presence - FROM main.web_sales, - main.web_returns, - main.web_page, - main.customer_demographics cd1, - main.customer_demographics cd2, - main.customer_address, - main.date_dim, - main.reason, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0, (SELECT c_customer_sk AS asa_key1 FROM main.customer) asa_owner1 -WHERE ws_bill_customer_sk = asa_owner0.asa_key0 AND wr_returning_customer_sk = asa_owner1.asa_key1 - AND ws_web_page_sk = wp_web_page_sk - AND ws_item_sk = wr_item_sk - AND ws_order_number = wr_order_number - AND ws_sold_date_sk = d_date_sk - AND d_year = 2000 - AND cd1.cd_demo_sk = wr_refunded_cdemo_sk - AND cd2.cd_demo_sk = wr_returning_cdemo_sk - AND ca_address_sk = wr_refunded_addr_sk - AND r_reason_sk = wr_reason_sk - AND ( ( cd1.cd_marital_status = 'M' - AND cd1.cd_marital_status = cd2.cd_marital_status - AND cd1.cd_education_status = 'Advanced Degree' - AND cd1.cd_education_status = cd2.cd_education_status - AND ws_sales_price BETWEEN 100.00 AND 150.00 ) - OR ( cd1.cd_marital_status = 'S' - AND cd1.cd_marital_status = cd2.cd_marital_status - AND cd1.cd_education_status = 'College' - AND cd1.cd_education_status = cd2.cd_education_status - AND ws_sales_price BETWEEN 50.00 AND 100.00 ) - OR ( cd1.cd_marital_status = 'W' - AND cd1.cd_marital_status = cd2.cd_marital_status - AND cd1.cd_education_status = '2 yr Degree' - AND cd1.cd_education_status = cd2.cd_education_status - AND ws_sales_price BETWEEN 150.00 AND 200.00 ) ) - AND ( ( ca_country = 'United States' - AND ca_state IN ('IN', - 'OH', - 'NJ') - AND ws_net_profit BETWEEN 100 AND 200) - OR ( ca_country = 'United States' - AND ca_state IN ('WI', - 'CT', - 'KY') - AND ws_net_profit BETWEEN 150 AND 300) - OR ( ca_country = 'United States' - AND ca_state IN ('LA', - 'IA', - 'AR') - AND ws_net_profit BETWEEN 50 AND 250) ) -GROUP BY r_reason_desc -), answers AS ( - SELECT world, reason, - avg1[world+1] AS avg1, - avg2[world+1] AS avg2, - avg3[world+1] AS avg3 - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY reason, avg1, avg2, avg3) <= 100 -ORDER BY world, reason, avg1, avg2, avg3; diff --git a/benchmark/tpcds/reference_queries/q88.sql b/benchmark/tpcds/reference_queries/q88.sql deleted file mode 100644 index 4cd5e384..00000000 --- a/benchmark/tpcds/reference_queries/q88.sql +++ /dev/null @@ -1,25 +0,0 @@ --- All time-bucket counters share the fact/dimension joins. -WITH stats AS (SELECT tpcds_count(CASE WHEN t_hour=8 AND t_minute >= 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h8_30_to_9, -tpcds_count(CASE WHEN t_hour=9 AND t_minute < 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h9_to_9_30, -tpcds_count(CASE WHEN t_hour=9 AND t_minute >= 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h9_30_to_10, -tpcds_count(CASE WHEN t_hour=10 AND t_minute < 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h10_to_10_30, -tpcds_count(CASE WHEN t_hour=10 AND t_minute >= 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h10_30_to_11, -tpcds_count(CASE WHEN t_hour=11 AND t_minute < 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h11_to_11_30, -tpcds_count(CASE WHEN t_hour=11 AND t_minute >= 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h11_30_to_12, -tpcds_count(CASE WHEN t_hour=12 AND t_minute < 30 THEN priv_hash(hash(ss_customer_sk)) ELSE 0::UBIGINT END,1) AS h12_to_12_30 -FROM main.store_sales, - main.household_demographics, - main.time_dim, - main.store, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ss_customer_sk=asa_owner.asa_key AND ss_sold_time_sk = time_dim.t_time_sk - AND ss_hdemo_sk = household_demographics.hd_demo_sk - AND ss_store_sk = s_store_sk - AND ((household_demographics.hd_dep_count = 4 - AND household_demographics.hd_vehicle_count<=4+2) - OR (household_demographics.hd_dep_count = 2 - AND household_demographics.hd_vehicle_count<=2+2) - OR (household_demographics.hd_dep_count = 0 - AND household_demographics.hd_vehicle_count<=0+2)) - AND store.s_store_name = 'ese' -) -SELECT world,h8_30_to_9[world+1] AS h8_30_to_9, h9_to_9_30[world+1] AS h9_to_9_30, h9_30_to_10[world+1] AS h9_30_to_10, h10_to_10_30[world+1] AS h10_to_10_30, h10_30_to_11[world+1] AS h10_30_to_11, h11_to_11_30[world+1] AS h11_to_11_30, h11_30_to_12[world+1] AS h11_30_to_12, h12_to_12_30[world+1] AS h12_to_12_30 FROM stats CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q90.sql b/benchmark/tpcds/reference_queries/q90.sql deleted file mode 100644 index 6e279282..00000000 --- a/benchmark/tpcds/reference_queries/q90.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Scalar AS arrays are combined only after their independent shared scans. -WITH raw0 AS ( -SELECT tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) amc - FROM main.web_sales, - main.household_demographics, - main.time_dim, - main.web_page, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_sold_time_sk = time_dim.t_time_sk - AND ws_ship_hdemo_sk = household_demographics.hd_demo_sk - AND ws_web_page_sk = web_page.wp_web_page_sk - AND time_dim.t_hour BETWEEN 8 AND 8+1 - AND household_demographics.hd_dep_count = 6 - AND web_page.wp_char_count BETWEEN 5000 AND 5200 -), -raw1 AS ( -SELECT tpcds_count(priv_hash(hash(ws_bill_customer_sk)),1) pmc - FROM main.web_sales, - main.household_demographics, - main.time_dim, - main.web_page, (SELECT c_customer_sk AS asa_key FROM main.customer) asa_owner -WHERE ws_bill_customer_sk=asa_owner.asa_key AND ws_sold_time_sk = time_dim.t_time_sk - AND ws_ship_hdemo_sk = household_demographics.hd_demo_sk - AND ws_web_page_sk = web_page.wp_web_page_sk - AND time_dim.t_hour BETWEEN 19 AND 19+1 - AND household_demographics.hd_dep_count = 6 - AND web_page.wp_char_count BETWEEN 5000 AND 5200 -), -p AS (SELECT world, raw0.amc[world+1] AS amc, raw1.pmc[world+1] AS pmc FROM raw0 CROSS JOIN raw1 CROSS JOIN range(64) worlds(world)) -SELECT world, case when p.pmc=0 then null else cast(p.amc AS decimal(15,4))/cast(p.pmc AS decimal(15,4)) end am_pm_ratio - FROM p ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q91.sql b/benchmark/tpcds/reference_queries/q91.sql deleted file mode 100644 index d3c9ad21..00000000 --- a/benchmark/tpcds/reference_queries/q91.sql +++ /dev/null @@ -1,44 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT cc_call_center_id AS call_center, - cc_name AS call_center_name, - cc_manager AS manager, - tpcds_sum((priv_hash(hash(cr_returning_customer_sk))), cr_net_loss) AS returns_loss, - tpcds_count((priv_hash(hash(cr_returning_customer_sk))), 1) AS presence - FROM main.call_center, - main.catalog_returns, - main.date_dim, - main.customer, - main.customer_address, - main.customer_demographics, - main.household_demographics, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE cr_returning_customer_sk = asa_owner0.asa_key0 - AND cr_call_center_sk = cc_call_center_sk - AND cr_returned_date_sk = d_date_sk - AND cr_returning_customer_sk= c_customer_sk - AND cd_demo_sk = c_current_cdemo_sk - AND hd_demo_sk = c_current_hdemo_sk - AND ca_address_sk = c_current_addr_sk - AND d_year = 1998 - AND d_moy = 11 - AND ((cd_marital_status = 'M' - AND cd_education_status = 'Unknown') or(cd_marital_status = 'W' - AND cd_education_status = 'Advanced Degree')) - AND hd_buy_potential LIKE 'Unknown%' - AND ca_gmt_offset = -7 -GROUP BY cc_call_center_id, - cc_name, - cc_manager, - cd_marital_status, - cd_education_status -), answers AS ( - SELECT world, call_center, - call_center_name, - manager, - returns_loss[world+1] AS returns_loss - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -ORDER BY world, returns_loss DESC; diff --git a/benchmark/tpcds/reference_queries/q92.sql b/benchmark/tpcds/reference_queries/q92.sql deleted file mode 100644 index f8fc761b..00000000 --- a/benchmark/tpcds/reference_queries/q92.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Correlated aggregate predicates update the mask before the outer AS SUM. -WITH thresholds AS ( - SELECT ws_item_sk, tpcds_avg(priv_hash(hash(ws_bill_customer_sk)),ws_ext_discount_amt) AS averages - FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk JOIN main.date_dim ON ws_sold_date_sk=d_date_sk - WHERE d_date BETWEEN DATE '2000-01-27' AND DATE '2000-04-26' GROUP BY ws_item_sk -), filtered AS ( - SELECT s.ws_ext_discount_amt AS discount, - priv_select(priv_hash(hash(ws_bill_customer_sk)),list_transform(averages, lambda average: s.ws_ext_discount_amt>1.3*average)) AS membership - FROM main.web_sales s JOIN main.customer ON ws_bill_customer_sk=c_customer_sk - JOIN main.item ON s.ws_item_sk=i_item_sk JOIN main.date_dim ON s.ws_sold_date_sk=d_date_sk - JOIN thresholds t ON s.ws_item_sk=t.ws_item_sk - WHERE i_manufact_id=350 AND d_date BETWEEN DATE '2000-01-27' AND DATE '2000-04-26' -), stats AS (SELECT tpcds_sum(membership,discount) AS total FROM filtered) -SELECT world,total[world+1] AS excess_discount_amount FROM stats CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q93.sql b/benchmark/tpcds/reference_queries/q93.sql deleted file mode 100644 index 3a14d36b..00000000 --- a/benchmark/tpcds/reference_queries/q93.sql +++ /dev/null @@ -1,12 +0,0 @@ --- The reason predicate rejects unmatched returns; both owners must be present. -WITH input AS ( - SELECT ss_customer_sk,CASE WHEN sr_return_quantity IS NOT NULL THEN (ss_quantity-sr_return_quantity)*ss_sales_price ELSE ss_quantity*ss_sales_price END AS act_sales, - (priv_hash(hash(ss_customer_sk)) & priv_hash(hash(sr_customer_sk))) AS membership - FROM main.store_sales JOIN main.store_returns ON sr_item_sk=ss_item_sk AND sr_ticket_number=ss_ticket_number - JOIN main.reason ON sr_reason_sk=r_reason_sk - JOIN main.customer sale_owner ON ss_customer_sk=sale_owner.c_customer_sk - JOIN main.customer return_owner ON sr_customer_sk=return_owner.c_customer_sk - WHERE r_reason_desc='reason 28' -), stats AS (SELECT ss_customer_sk,tpcds_sum(membership,act_sales) AS sumsales,tpcds_count(membership,1) AS presence FROM input GROUP BY ss_customer_sk), -answers AS (SELECT world,ss_customer_sk,sumsales[world+1] AS sumsales FROM stats CROSS JOIN range(64) worlds(world) WHERE presence[world+1]>0) -SELECT * FROM answers QUALIFY row_number() OVER (PARTITION BY world ORDER BY sumsales NULLS FIRST,ss_customer_sk NULLS FIRST)<=100 ORDER BY world,sumsales NULLS FIRST,ss_customer_sk NULLS FIRST; diff --git a/benchmark/tpcds/reference_queries/q94.sql b/benchmark/tpcds/reference_queries/q94.sql deleted file mode 100644 index 708051dc..00000000 --- a/benchmark/tpcds/reference_queries/q94.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Order-level existence masks are combined before AS; DISTINCT unions memberships by order. -WITH warehouses AS (SELECT ws_order_number AS order_number,ws_warehouse_sk AS warehouse,bit_or(priv_hash(hash(ws_bill_customer_sk))) AS membership FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk GROUP BY ws_order_number,ws_warehouse_sk), -other_warehouses AS (SELECT a.order_number,a.warehouse,bit_or(b.membership) AS membership FROM warehouses a JOIN warehouses b ON a.order_number=b.order_number AND a.warehouse<>b.warehouse GROUP BY a.order_number,a.warehouse), -return_orders AS (SELECT wr_order_number AS order_number,bit_or(priv_hash(hash(wr_returning_customer_sk))) AS membership FROM main.web_returns JOIN main.customer ON wr_returning_customer_sk=c_customer_sk GROUP BY wr_order_number), -input AS (SELECT ws_order_number AS order_number,ws_ext_ship_cost AS shipping,ws_net_profit AS profit,(priv_hash(hash(ws_bill_customer_sk)) & other.membership & ~coalesce(ret.membership,0::UBIGINT)) AS membership -FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk JOIN main.date_dim ON ws_ship_date_sk=d_date_sk JOIN main.customer_address ON ws_ship_addr_sk=ca_address_sk JOIN main.web_site ON ws_web_site_sk=web_site_sk JOIN other_warehouses other ON ws_order_number=other.order_number AND ws_warehouse_sk=other.warehouse LEFT JOIN return_orders ret ON ws_order_number=ret.order_number WHERE d_date BETWEEN DATE '1999-02-01' AND DATE '1999-04-02' AND ca_state='IL' AND web_company_name='pri'), -totals AS (SELECT tpcds_sum(membership,shipping) AS shipping,tpcds_sum(membership,profit) AS profit FROM input), -unique_orders AS (SELECT order_number,bit_or(membership) AS membership FROM input GROUP BY order_number), -counts AS (SELECT tpcds_count(membership,1) AS n FROM unique_orders) -SELECT world,n[world+1] AS order_count,shipping[world+1] AS total_shipping_cost,profit[world+1] AS total_net_profit FROM totals CROSS JOIN counts CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q95.sql b/benchmark/tpcds/reference_queries/q95.sql deleted file mode 100644 index 7eb87985..00000000 --- a/benchmark/tpcds/reference_queries/q95.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Order-level existence masks are combined before AS; DISTINCT unions memberships by order. -WITH warehouses AS (SELECT ws_order_number AS order_number,ws_warehouse_sk AS warehouse,bit_or(priv_hash(hash(ws_bill_customer_sk))) AS membership FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk GROUP BY ws_order_number,ws_warehouse_sk), -pair_orders AS (SELECT a.order_number,bit_or(a.membership & b.membership) AS membership FROM warehouses a JOIN warehouses b ON a.order_number=b.order_number AND a.warehouse<>b.warehouse GROUP BY a.order_number), -return_orders AS (SELECT wr_order_number AS order_number,bit_or(priv_hash(hash(wr_returning_customer_sk))) AS membership FROM main.web_returns JOIN main.customer ON wr_returning_customer_sk=c_customer_sk GROUP BY wr_order_number), -input AS (SELECT ws_order_number AS order_number,ws_ext_ship_cost AS shipping,ws_net_profit AS profit,(priv_hash(hash(ws_bill_customer_sk)) & other.membership & coalesce(ret.membership,0::UBIGINT)) AS membership -FROM main.web_sales JOIN main.customer ON ws_bill_customer_sk=c_customer_sk JOIN main.date_dim ON ws_ship_date_sk=d_date_sk JOIN main.customer_address ON ws_ship_addr_sk=ca_address_sk JOIN main.web_site ON ws_web_site_sk=web_site_sk JOIN pair_orders other ON ws_order_number=other.order_number LEFT JOIN return_orders ret ON ws_order_number=ret.order_number WHERE d_date BETWEEN DATE '1999-02-01' AND DATE '1999-04-02' AND ca_state='IL' AND web_company_name='pri'), -totals AS (SELECT tpcds_sum(membership,shipping) AS shipping,tpcds_sum(membership,profit) AS profit FROM input), -unique_orders AS (SELECT order_number,bit_or(membership) AS membership FROM input GROUP BY order_number), -counts AS (SELECT tpcds_count(membership,1) AS n FROM unique_orders) -SELECT world,n[world+1] AS order_count,shipping[world+1] AS total_shipping_cost,profit[world+1] AS total_net_profit FROM totals CROSS JOIN counts CROSS JOIN range(64) worlds(world) ORDER BY world; diff --git a/benchmark/tpcds/reference_queries/q96.sql b/benchmark/tpcds/reference_queries/q96.sql deleted file mode 100644 index f14f19b0..00000000 --- a/benchmark/tpcds/reference_queries/q96.sql +++ /dev/null @@ -1,24 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS total, - tpcds_count((priv_hash(hash(ss_customer_sk))), 1) AS presence - FROM main.store_sales , - main.household_demographics, - main.time_dim, - main.store, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE ss_customer_sk = asa_owner0.asa_key0 - AND ss_sold_time_sk = time_dim.t_time_sk - AND ss_hdemo_sk = household_demographics.hd_demo_sk - AND ss_store_sk = s_store_sk - AND time_dim.t_hour = 20 - AND time_dim.t_minute >= 30 - AND household_demographics.hd_dep_count = 7 - AND store.s_store_name = 'ese' -), answers AS ( - SELECT world, total[world+1] AS total - FROM stats CROSS JOIN range(64) worlds(world) -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY total) <= 100 -ORDER BY world, total; diff --git a/benchmark/tpcds/reference_queries/q99.sql b/benchmark/tpcds/reference_queries/q99.sql deleted file mode 100644 index 2366503a..00000000 --- a/benchmark/tpcds/reference_queries/q99.sql +++ /dev/null @@ -1,42 +0,0 @@ --- Explicit AS query; tpcds_* macros normalize the public aggregate scaling and NULL semantics. -WITH stats AS ( - SELECT w_substr, - sm_type, - LOWER(cc_name) AS cc_name_lower, - tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), CASE WHEN cs_ship_date_sk - cs_sold_date_sk <= 30 THEN 1 ELSE 0 END) AS bucket0, - tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), CASE WHEN cs_ship_date_sk - cs_sold_date_sk > 30 AND cs_ship_date_sk - cs_sold_date_sk <= 60 THEN 1 ELSE 0 END) AS bucket1, - tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), CASE WHEN cs_ship_date_sk - cs_sold_date_sk > 60 AND cs_ship_date_sk - cs_sold_date_sk <= 90 THEN 1 ELSE 0 END) AS bucket2, - tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), CASE WHEN cs_ship_date_sk - cs_sold_date_sk > 90 AND cs_ship_date_sk - cs_sold_date_sk <= 120 THEN 1 ELSE 0 END) AS bucket3, - tpcds_sum((priv_hash(hash(cs_bill_customer_sk))), CASE WHEN cs_ship_date_sk - cs_sold_date_sk > 120 THEN 1 ELSE 0 END) AS bucket4, - tpcds_count((priv_hash(hash(cs_bill_customer_sk))), 1) AS presence - FROM main.catalog_sales , - (SELECT SUBSTRING(w_warehouse_name,1,20) w_substr, * - FROM main.warehouse) AS sq1 , - main.ship_mode , - main.call_center , - main.date_dim, - (SELECT c_customer_sk AS asa_key0 FROM main.customer) asa_owner0 -WHERE cs_bill_customer_sk = asa_owner0.asa_key0 - AND d_month_seq BETWEEN 1200 AND 1200 + 11 - AND cs_ship_date_sk = d_date_sk - AND cs_warehouse_sk = w_warehouse_sk - AND cs_ship_mode_sk = sm_ship_mode_sk - AND cs_call_center_sk = cc_call_center_sk -GROUP BY w_substr , - sm_type , - cc_name -), answers AS ( - SELECT world, w_substr, - sm_type, - cc_name_lower, - bucket0[world+1] AS bucket0, - bucket1[world+1] AS bucket1, - bucket2[world+1] AS bucket2, - bucket3[world+1] AS bucket3, - bucket4[world+1] AS bucket4 - FROM stats CROSS JOIN range(64) worlds(world) - WHERE presence[world+1] > 0 -) -SELECT * FROM answers -QUALIFY row_number() OVER (PARTITION BY world ORDER BY w_substr NULLS FIRST, sm_type NULLS FIRST, cc_name_lower NULLS FIRST) <= 100 -ORDER BY world, w_substr NULLS FIRST, sm_type NULLS FIRST, cc_name_lower NULLS FIRST; diff --git a/benchmark/tpcds/run.py b/benchmark/tpcds/run.py index 19bab0f8..6901c4d6 100644 --- a/benchmark/tpcds/run.py +++ b/benchmark/tpcds/run.py @@ -2,98 +2,58 @@ """Isolate TPC-DS queries in processes; all reported timings come from C++.""" import argparse import csv -import hashlib import json import pathlib import statistics import subprocess -def digest_file(path): - digest = hashlib.sha256() - with pathlib.Path(path).open("rb") as file: - for block in iter(lambda: file.read(1024 * 1024), b""): - digest.update(block) - return digest.hexdigest() - - def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", required=True) parser.add_argument("--runner", default="build/release/extension/privacy/as_tpcds_benchmark") parser.add_argument("--queries", help="Comma-separated query numbers; default: all 99") parser.add_argument("--resume", action="store_true") - validation = parser.add_mutually_exclusive_group() - validation.add_argument("--validate", action="store_true", help="Compare handwritten PAC SQL to automatic compilation") - validation.add_argument("--validate-compiled", action="store_true", help="Compare PAC output to world-zero SQL (semantics diagnostic, not compiler correctness)") - validation.add_argument("--validate-worlds", action="store_true", help="Run the separate 64-world AS reference checks") args = parser.parse_args() - if args.validate_compiled or args.validate_worlds: - args.validate = True config = json.loads(pathlib.Path(args.config).read_text()) - handwritten_dir = pathlib.Path(config.get("reference_queries_dir", "benchmark/tpcds/reference_queries") if args.validate_worlds else config.get("as_queries_dir", "benchmark/tpcds/as_queries")) - default_ids = sorted(int(p.stem[1:]) for p in handwritten_dir.glob("q[0-9][0-9].sql")) if args.validate else list(range(1, 100)) - query_ids = [int(q) for q in args.queries.split(",")] if args.queries else default_ids - suffix = "_compiled_validation" if args.validate_compiled else "_validation" if args.validate_worlds else "_pac_validation" if args.validate else "" + query_ids = [int(q) for q in args.queries.split(",")] if args.queries else list(range(1, 100)) if not query_ids or any(q < 1 or q > 99 for q in query_ids) or len(set(query_ids)) != len(query_ids): parser.error("Query numbers must be unique and between 1 and 99") output = pathlib.Path(config["out_dir"]) manifest_path = output / "run_config.json" if manifest_path.exists() and json.loads(manifest_path.read_text()) != config: parser.error("Configuration changed: use a fresh out_dir to avoid mixing experiments") - if not args.resume and any((output / f"q{q:02}{suffix}.csv").exists() for q in query_ids): + if not args.resume and any((output / f"q{q:02}.csv").exists() for q in query_ids): parser.error("Results already exist: use a fresh out_dir or --resume") subprocess.run([args.runner, args.config, "--prepare"], check=True) manifest_path.write_text(json.dumps(config, indent=2) + "\n") summary = [] - runner_digest = digest_file(args.runner) for query in query_ids: name = f"q{query:02}" - path = output / f"{name}{suffix}.csv" + path = output / f"{name}.csv" process_status = "completed" process_error = "" - process_path = output / f"{name}{suffix}_process.json" - inputs = (runner_digest + digest_file(pathlib.Path(config["queries_dir"]) / f"{name}.sql") - + digest_file(config.get("schema", "benchmark/tpcds/schema.sql"))) - hand_sql = handwritten_dir / f"{name}.sql" - if not args.validate_compiled and hand_sql.exists(): - inputs += digest_file(hand_sql) - inputs_digest = hashlib.sha256(inputs.encode()).hexdigest() - state = json.loads(process_path.read_text()) if process_path.exists() else {} - if args.resume and state and state.get("inputs_sha256") == inputs_digest: + process_path = output / f"{name}_process.json" + if args.resume and process_path.exists(): + state = json.loads(process_path.read_text()) process_status, process_error = state["status"], state["error"] else: print(f"Running {name}", flush=True) - with (output / f"{name}{suffix}.log").open("w") as log: + with (output / f"{name}.log").open("w") as log: try: result = subprocess.run( - [args.runner, args.config, str(query)] + (["--validate-compiled"] if args.validate_compiled else ["--validate-worlds"] if args.validate_worlds else ["--validate"] if args.validate else []), stdout=log, stderr=subprocess.STDOUT, + [args.runner, args.config, str(query)], stdout=log, stderr=subprocess.STDOUT, timeout=config.get("timeout_seconds", 120), ) if result.returncode: process_status = "process_error" - process_error = f"exit code {result.returncode}; see {name}{suffix}.log" + process_error = f"exit code {result.returncode}; see {name}.log" except subprocess.TimeoutExpired: process_status = "timeout" process_error = "Whole-query process timeout (both modes and all repetitions)" - process_path.write_text(json.dumps(dict(status=process_status, error=process_error, inputs_sha256=inputs_digest))) + process_path.write_text(json.dumps(dict(status=process_status, error=process_error))) rows = list(csv.DictReader(path.open())) if path.exists() else [] - if args.validate: - if not args.validate_worlds: - for row in rows: - row["method"] = "world_zero_diagnostic" if args.validate_compiled else "pac_equivalence" - if rows: - summary.extend(rows) - else: - summary.append(dict(query=name, method="validation", status=process_status if process_status != "completed" else "incomplete", error=process_error)) - fields = list(dict.fromkeys(key for row in summary for key in row)) - with (output / ("compiled_validation_summary.csv" if args.validate_compiled else "validation_summary.csv" if args.validate_worlds else "pac_validation_summary.csv")).open("w", newline="") as file: - writer = csv.DictWriter(file, fieldnames=fields) - writer.writeheader() - writer.writerows(summary) - print(f"{name}: " + (", ".join(f"{r['method']}={r['status']}" for r in rows) if rows else process_status + ": " + process_error), flush=True) - continue - for mode in ("duckdb", "simd_asa_handwritten", "simd_asa_auto", "naive_as"): + for mode in ("duckdb", "simd_asa_auto"): entries = [row for row in rows if row["mode"] == mode] measured = [row for row in entries if row["status"] == "ok"] complete = len(measured) == config.get("repetitions", 5) @@ -105,7 +65,6 @@ def main(): query=name, mode=mode, status=status, rewritten=entries[-1]["rewritten"] if entries else "", measured_runs=len(measured), - world_multiplier=entries[-1].get("world_multiplier", "") if entries else "", median_ms=statistics.median(float(row["runtime_ms"]) for row in measured) if complete else "", empty_result=all(row["rows"] == "0" for row in measured) if complete else "", correctness="not_validated", @@ -115,8 +74,8 @@ def main(): writer = csv.DictWriter(file, fieldnames=list(summary[0])) writer.writeheader() writer.writerows(summary) - print(f"{name}: " + ", ".join(f"{r['mode']}={r['status']}" for r in summary[-4:]), flush=True) - print(f"Results: {output / ('compiled_validation_summary.csv' if args.validate_compiled else 'validation_summary.csv' if args.validate_worlds else 'pac_validation_summary.csv' if args.validate else 'summary.csv')}") + print(f"{name}: " + ", ".join(f"{r['mode']}={r['status']}" for r in summary[-2:]), flush=True) + print(f"Results: {output / 'summary.csv'}") if __name__ == "__main__": diff --git a/docs/benchmark/graviton_duckdb2_rerun.md b/docs/benchmark/graviton_duckdb2_rerun.md deleted file mode 100644 index cc8d62dd..00000000 --- a/docs/benchmark/graviton_duckdb2_rerun.md +++ /dev/null @@ -1,42 +0,0 @@ -# Graviton revision rerun, November 2026 - -The DuckDB 2.0 branch is rebased onto main `52d4a3b`. The Graviton host is -`ec2-3-136-25-192.us-east-2.compute.amazonaws.com`: 16 Neoverse-V2 cores, -32 GB RAM, Ubuntu 26.04, Clang 22.1.2, and a local NVMe SSD mounted at -`/mnt/asa`. The SSD was blank after instance restart; datasets are regenerated. - -`/mnt/asa/privacy` holds main; `/mnt/asa/privacy2` holds the upgrade branch. -Both use separate build directories and database copies. The upgrade's pinned -DuckDB source is `d54cbbe3df17e8e5ce68312f483b4155836112aa`. TPC-H and TPC-DS -generator extensions are built from that same source and installed locally. - -Preparation/build services record their commits, settings, logs, and exit codes -under `/mnt/asa/repro`. Run the suite after starting these services: - -```sh -sudo systemd-run --unit=asa-full-revision-20261110 --uid=ubuntu \ - --working-directory=/mnt/asa/privacy2 \ - /bin/bash /mnt/asa/privacy2/benchmark/run_graviton_full_revision.sh \ - /mnt/asa /mnt/asa/results/duckdb2_revision_20261110 -``` - -Order: matched main TPC-H control, 2.0 TPC-H and its sample-count sweep, -ClickBench, SSB, MEPS, TPC-DS validation/performance, DP utility/stability, -then aggregation microbenchmarks. Analytical timings retain five measured -executions after warmup; DP configurations retain their existing three runs. -SSB and MEPS run approximate and exact SUM. Microbenchmarks retain their -existing four-thread setup, one warmup, three runs, and slow-query cutoff. -All stages run sequentially. A failed stage is logged and subsequent stages -continue; individual TPC-DS query failures are recorded in its CSVs. - -The microbenchmark driver uses a separate build tree and never deletes the -analytical build. Its summary now computes a true median: the old awk expression -selected the minimum sorted observation. Raw timing samples remain in the CSVs. -SUM accuracy compares exact and approximate samples on the same unscaled units -for both errors and variances. These measurement corrections must be considered -when comparing old summaries. - -Copy CSVs, configs, manifests, plans, and logs locally before stopping the instance. -The local collection is `benchmark/results/graviton_duckdb2_20261110/`; prior -paper CSVs and the October 28 five-run Graviton controls are retained under -`previous/`. No paper text or figures are updated by this driver. diff --git a/src/aggregates/as_sum.cpp b/src/aggregates/as_sum.cpp index ead72feb..eb53bab2 100644 --- a/src/aggregates/as_sum.cpp +++ b/src/aggregates/as_sum.cpp @@ -658,8 +658,8 @@ void PacSumScatterUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_d static void DpSampleSumUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count, const ClusteredAggr &clusters, idx_t count) { - PacSumUpdateClusters( - inputs, clusters, aggr_input_data.allocator, GetPrivSampleLanes(aggr_input_data)); + PacSumUpdateClusters(inputs, clusters, aggr_input_data.allocator, + GetPrivSampleLanes(aggr_input_data)); } static void DpSampleSumScatterUpdateDouble(Vector inputs[], AggregateInputData &aggr_input_data, idx_t, Vector &states, diff --git a/src/compiler/privacy_compiler.cpp b/src/compiler/privacy_compiler.cpp index 2a56bf99..8796b196 100644 --- a/src/compiler/privacy_compiler.cpp +++ b/src/compiler/privacy_compiler.cpp @@ -520,7 +520,8 @@ class PrivateUnionRewriter { ref->ResolveOperatorTypes(); memberships.emplace(ref, ColumnBinding(ref->table_index, ProjectionIndex(mask_column))); } - PRIVACY_DEBUG_PRINT("[PRIVATE UNION] Exported membership through CTE " + cte.ctename.GetIdentifierName()); + PRIVACY_DEBUG_PRINT("[PRIVATE UNION] Exported membership through CTE " + + cte.ctename.GetIdentifierName()); } Rewrite(cte.children[1]); cte.ResolveOperatorTypes(); @@ -570,8 +571,8 @@ class PrivateUnionRewriter { throw InvalidInputException("Private UNION cannot combine independent ownership streams above a UNION"); } CheckBranch(*agg.children[0]); - auto mask = - PropagateSingleBinding(*plan, found[0].second.table_index.index, found[0].second, LogicalType::UBIGINT, &agg); + auto mask = PropagateSingleBinding(*plan, found[0].second.table_index.index, found[0].second, + LogicalType::UBIGINT, &agg); if (mask.table_index.index == DConstants::INVALID_INDEX) { throw InvalidInputException("Private UNION could not propagate membership to the consuming aggregate"); } @@ -601,10 +602,11 @@ class PrivateUnionRewriter { input.optimizer.binder.GenerateTableIndex(), std::move(aggregates)); for (idx_t i = 0; i < count; i++) { dedup->groups.push_back(make_uniq(types[i], bindings[i])); - outputs.push_back(make_uniq(types[i], ColumnBinding(dedup->group_index, ProjectionIndex(i)))); + outputs.push_back(make_uniq( + types[i], ColumnBinding(dedup->group_index, ProjectionIndex(i)))); } - outputs.push_back( - make_uniq(LogicalType::UBIGINT, ColumnBinding(dedup->aggregate_index, ProjectionIndex(0)))); + outputs.push_back(make_uniq( + LogicalType::UBIGINT, ColumnBinding(dedup->aggregate_index, ProjectionIndex(0)))); dedup->children.push_back(std::move(op)); op = std::move(dedup); } else { @@ -657,7 +659,8 @@ class PrivateUnionRewriter { ProjectMembership(op, count, output_index, true); ColumnBindingReplacer replacer; for (idx_t i = 0; i < count; i++) { - replacer.replacement_bindings.emplace_back(old_bindings[i], ColumnBinding(output_index, ProjectionIndex(i))); + replacer.replacement_bindings.emplace_back(old_bindings[i], + ColumnBinding(output_index, ProjectionIndex(i))); } replacer.stop_operator = op.get(); replacer.VisitOperator(*plan); From 371825eb32bd6c1dfcc5d5bfd5e25071e55eabc7 Mon Sep 17 00:00:00 2001 From: ila Date: Wed, 11 Nov 2026 15:24:19 +0100 Subject: [PATCH 30/32] Adapt UNION source lookup to DuckDB 2.0 table indices --- src/query_processing/privacy_plan_traversal.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query_processing/privacy_plan_traversal.cpp b/src/query_processing/privacy_plan_traversal.cpp index 90141bc6..2c5bf218 100644 --- a/src/query_processing/privacy_plan_traversal.cpp +++ b/src/query_processing/privacy_plan_traversal.cpp @@ -481,7 +481,7 @@ LogicalOperator *FindOperatorByTableIndex(LogicalOperator *op, idx_t table_index return op; } } else if (op->type == LogicalOperatorType::LOGICAL_UNION) { - if (op->Cast().table_index == table_index) { + if (op->Cast().table_index.index == table_index) { return op; } } From d39f0b1b2bf6d9e88b416a982e256524ab02733c Mon Sep 17 00:00:00 2001 From: ila Date: Wed, 11 Nov 2026 16:19:27 +0100 Subject: [PATCH 31/32] Run quality checks against the official Ubuntu package mirror Keep the existing format/tidy checks and tool versions in a local reusable wrapper so dependency installation can avoid the hosted Azure mirror DNS failures. Distribution builds continue to use the pinned upstream workflow. --- .github/workflows/CodeQuality.yml | 51 +++++++++++++++++++ .../workflows/MainDistributionPipeline.yml | 7 +-- 2 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/CodeQuality.yml diff --git a/.github/workflows/CodeQuality.yml b/.github/workflows/CodeQuality.yml new file mode 100644 index 00000000..0245f10f --- /dev/null +++ b/.github/workflows/CodeQuality.yml @@ -0,0 +1,51 @@ +name: Code Quality + +on: + workflow_call: + +jobs: + check: + name: ${{ matrix.name }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - name: Format Check + runner: ubuntu-22.04 + apt_packages: ninja-build clang-format-11 + pip_packages: cmake-format 'black==24.*' cxxheaderparser pcpp 'clang_format==11.0.1' + pip_flags: '' + target: format-check + - name: Tidy Check + runner: ubuntu-24.04 + apt_packages: ninja-build clang-tidy + pip_packages: '"pybind11[global]"' + pip_flags: --break-system-packages + target: tidy-check + env: + CC: gcc + CXX: g++ + GEN: ninja + TIDY_THREADS: 4 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + submodules: recursive + + - name: Install check dependencies + shell: bash + run: | + # The hosted image's Azure mirror has repeatedly failed DNS lookup. + # Keep Ubuntu's signed packages, using its official archive directly. + if [ -f /etc/apt/apt-mirrors.txt ]; then + printf '%s\n' 'https://archive.ubuntu.com/ubuntu/' | sudo tee /etc/apt/apt-mirrors.txt + fi + sudo apt-get -o Acquire::Retries=3 update -y -qq + sudo apt-get -o Acquire::Retries=3 install -y -qq ${{ matrix.apt_packages }} + sudo pip3 install ${{ matrix.pip_packages }} ${{ matrix.pip_flags }} + + - name: Run check + run: make ${{ matrix.target }} diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 7b01c057..dd5710b6 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -22,9 +22,4 @@ jobs: code-quality-check: name: Code Quality Check - uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@ebdcc1961780b0ce1272a0d7637dc58447c34885 - with: - duckdb_version: v2.0.0 - ci_tools_version: ebdcc1961780b0ce1272a0d7637dc58447c34885 - extension_name: privacy - format_checks: 'format;tidy' + uses: ./.github/workflows/CodeQuality.yml From fb3d274bc2c18b8507555cf256a756aca079d887 Mon Sep 17 00:00:00 2001 From: ila Date: Wed, 11 Nov 2026 16:26:36 +0100 Subject: [PATCH 32/32] Use hosted Ninja and clang-tidy to avoid redundant apt installs --- .github/workflows/CodeQuality.yml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/CodeQuality.yml b/.github/workflows/CodeQuality.yml index 0245f10f..d2de0130 100644 --- a/.github/workflows/CodeQuality.yml +++ b/.github/workflows/CodeQuality.yml @@ -13,21 +13,22 @@ jobs: include: - name: Format Check runner: ubuntu-22.04 - apt_packages: ninja-build clang-format-11 pip_packages: cmake-format 'black==24.*' cxxheaderparser pcpp 'clang_format==11.0.1' pip_flags: '' + tidy_binary: '' target: format-check - name: Tidy Check runner: ubuntu-24.04 - apt_packages: ninja-build clang-tidy pip_packages: '"pybind11[global]"' pip_flags: --break-system-packages + tidy_binary: clang-tidy-18 target: tidy-check env: CC: gcc CXX: g++ GEN: ninja TIDY_THREADS: 4 + TIDY_BINARY: ${{ matrix.tidy_binary }} steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -35,17 +36,17 @@ jobs: fetch-depth: 0 submodules: recursive - - name: Install check dependencies + - name: Install Python check tools shell: bash run: | - # The hosted image's Azure mirror has repeatedly failed DNS lookup. - # Keep Ubuntu's signed packages, using its official archive directly. - if [ -f /etc/apt/apt-mirrors.txt ]; then - printf '%s\n' 'https://archive.ubuntu.com/ubuntu/' | sudo tee /etc/apt/apt-mirrors.txt - fi - sudo apt-get -o Acquire::Retries=3 update -y -qq - sudo apt-get -o Acquire::Retries=3 install -y -qq ${{ matrix.apt_packages }} + # Ninja and clang-tidy-18 are already on the hosted Ubuntu image. + # Avoid redundant apt installs that fail on the image's package mirrors. sudo pip3 install ${{ matrix.pip_packages }} ${{ matrix.pip_flags }} - name: Run check - run: make ${{ matrix.target }} + run: | + if [ '${{ matrix.target }}' = 'tidy-check' ]; then + ninja --version + "$TIDY_BINARY" --version + fi + make ${{ matrix.target }}