From cbe4522e25f1f086d16799205aa6cb0418fda192 Mon Sep 17 00:00:00 2001 From: Lalit Maganti Date: Sun, 27 Sep 2026 20:52:08 +0100 Subject: [PATCH 1/2] tp: compile each pipeline source through one relation step No behaviour change. A pipeline source and each intersection operand now go through one step, CompileRelation, which gives the plan node and columns of what the source reads, and the names in scope while a pipeline is compiled live together in one Scope. Also renames MacroRewriteBuilder to RewriteTree, as syntaqlite calls these rewrites. --- .../parser/perfetto_sql_parser.cc | 47 ++-- .../perfetto_sql/pipeline/compiler.cc | 238 ++++++++++-------- 2 files changed, 149 insertions(+), 136 deletions(-) diff --git a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc index dac2c12d178..94a1367c624 100644 --- a/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc +++ b/src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.cc @@ -114,28 +114,21 @@ base::StatusOr BuildReturnType( } // --------------------------------------------------------------------------- -// Macro rewrite tree -> SqlSource +// Rewrite tree -> SqlSource // --------------------------------------------------------------------------- -// Walks the flat list of macro rewrites produced by syntaqlite and, for a -// given AST node, builds a SqlSource whose `Rewriter` structure mirrors the -// nesting of macro calls. This lets SQLite-side error tracebacks resolve -// through macro expansions back to the authored call site. -// -// The flat list has O(N) entries reported in insertion order (outer macros -// before their nested calls). Rather than re-scanning the list for each -// rewrite we materialize a parent -> children adjacency once in the -// constructor, so every subsequent lookup descends the tree in O(subtree). -class MacroRewriteBuilder { +// The macro rewrites syntaqlite recorded for a statement, as a tree, taken +// when it is constructed. Gives the SqlSource of any AST node with them +// applied; the SqlSources nest like the rewrites, so SQLite errors trace back +// through every expansion to where the user wrote the text. +class RewriteTree { public: - // `stmt_doc_offset` is the byte offset of the current statement within - // `stmt`. Syntaqlite v0.5 reports every layer-0 offset (node extents, - // spans, macro call offsets) statement-relative, so call sites that - // slice `stmt` add this offset to translate into document coordinates. - MacroRewriteBuilder(SyntaqliteParser* p, - const SqlSource& stmt, - uint32_t stmt_doc_offset, - const base::FlatHashMap& macros) + // `stmt_doc_offset` is where the statement starts in `stmt`; syntaqlite + // reports offsets relative to the statement. + RewriteTree(SyntaqliteParser* p, + const SqlSource& stmt, + uint32_t stmt_doc_offset, + const base::FlatHashMap& macros) : p_(p), stmt_(stmt), stmt_doc_offset_(stmt_doc_offset), macros_(macros) { uint32_t total = syntaqlite_result_rewrite_count(p_); no_macros_ = (total == 0); @@ -335,7 +328,7 @@ class MacroRewriteBuilder { std::vector source_rooted_; }; -SqlSource NodeSource(const MacroRewriteBuilder& rb, uint32_t node_id) { +SqlSource NodeSource(const RewriteTree& rb, uint32_t node_id) { auto s = rb.NodeSource(node_id); PERFETTO_CHECK(s.has_value()); return std::move(*s); @@ -358,7 +351,7 @@ uint32_t CurrentStatementDocOffset(SyntaqliteParser* p) { base::StatusOr CompilePipeline( SyntaqliteParser* p, - const MacroRewriteBuilder& rb, + const RewriteTree& rb, const pipeline::Catalog* catalog, uint32_t pipeline_id) { if (!catalog) { @@ -373,7 +366,7 @@ base::StatusOr CompilePipeline( base::StatusOr ParseCreateTableBody( SyntaqliteParser* p, - const MacroRewriteBuilder& rb, + const RewriteTree& rb, const pipeline::Catalog* catalog, const SyntaqliteCreatePerfettoTableStmt& n) { using Body = PerfettoSqlParser::CreateTable::Body; @@ -386,7 +379,7 @@ base::StatusOr ParseCreateTableBody( base::StatusOr ParseCreateTable( SyntaqliteParser* p, - const MacroRewriteBuilder& rb, + const RewriteTree& rb, const pipeline::Catalog* catalog, const SyntaqliteCreatePerfettoTableStmt& n) { if (syntaqlite_node_is_present(n.table_impl)) { @@ -409,7 +402,7 @@ base::StatusOr ParseCreateTable( base::StatusOr ParseCreateView( SyntaqliteParser* p, - const MacroRewriteBuilder& rb, + const RewriteTree& rb, const SyntaqliteCreatePerfettoViewStmt& n) { ASSIGN_OR_RETURN(auto schema, BuildArgDefs(p, n.schema)); std::string name = SpanText(p, n.view_name); @@ -427,7 +420,7 @@ base::StatusOr ParseCreateView( base::StatusOr ParseCreateFunction( SyntaqliteParser* p, - const MacroRewriteBuilder& rb, + const RewriteTree& rb, const SyntaqliteCreatePerfettoFunctionStmt& n) { ASSIGN_OR_RETURN(auto args, BuildArgDefs(p, n.args)); for (const auto& arg : args) { @@ -547,7 +540,7 @@ Statement ParseCreateMacro(SyntaqliteParser* p, } base::StatusOr ParseStatement(SyntaqliteParser* p, - const MacroRewriteBuilder& rb, + const RewriteTree& rb, const SqlSource& stmt, uint32_t stmt_doc_offset, const pipeline::Catalog* catalog, @@ -734,7 +727,7 @@ bool PerfettoSqlParser::Impl::Next( // source; callers that slice `stmt` add it back on. uint32_t stmt_doc_offset = CurrentStatementDocOffset(synq); - MacroRewriteBuilder rb(synq, stmt, stmt_doc_offset, macros); + RewriteTree rb(synq, stmt, stmt_doc_offset, macros); auto root_src = rb.NodeSource(root); out_statement_sql = root_src.has_value() ? *std::move(root_src) : stmt; diff --git a/src/trace_processor/perfetto_sql/pipeline/compiler.cc b/src/trace_processor/perfetto_sql/pipeline/compiler.cc index 9b7fced40ce..7262e2f8bc4 100644 --- a/src/trace_processor/perfetto_sql/pipeline/compiler.cc +++ b/src/trace_processor/perfetto_sql/pipeline/compiler.cc @@ -82,9 +82,8 @@ class Compiler { Compiler(SyntaqliteParser* p, const NodeSourceFn& source, const Catalog& c) : p_(p), source_(source), catalog_(c) {} - base::Status CompileSource(uint32_t from); - base::Status CompileIntersection(uint32_t node); - base::Status CompileStage(uint32_t stage); + // Compiles the pipeline at `pipeline`: its source and each of its stages. + base::Status CompilePipeline(uint32_t pipeline); LogicalPlan Finish(); private: @@ -104,7 +103,23 @@ class Compiler { std::string name; std::vector columns; }; + // What names mean while a pipeline is compiled. + struct Scope { + // The operator being compiled, which prefixes every error. + const char* op = "FROM"; + // The row as of the stage being compiled, and the table aliases in scope. + std::vector row; + std::vector aliases; + }; + // A relation a source reads: the plan node producing it and its columns. + struct Relation { + PlanNodeId node = 0; + std::vector columns; + }; + base::Status CompileSource(uint32_t from); + base::Status CompileIntersection(uint32_t node); + base::Status CompileStage(uint32_t stage); base::Status CompileSelect(uint32_t stage); base::Status CompileExtend(uint32_t stage); base::Status CompileDrop(uint32_t stage); @@ -118,7 +133,7 @@ class Compiler { // Appends a column to the row. void Append(NamedColumn column, uint32_t node, bool qualified_only = false) { - row_.push_back({std::move(column), op_, node, qualified_only}); + scope_.row.push_back({std::move(column), scope_.op, node, qualified_only}); } const Alias* FindAlias(const std::string& name) const; // Forgets the table alias `name`, if there is one. @@ -137,10 +152,13 @@ class Compiler { // The name a source's columns can be qualified with, if it has one. std::optional SourceQualifier( const SyntaqlitePerfettoPipeSource&) const; + // Compiles what a source reads: a dataframe or SQL. + base::StatusOr CompileRelation(uint32_t source); // Fails unless every column of a source has a name a pipeline can use. // Crossing from SQL into a pipeline needs proper names, as creating a // PERFETTO TABLE does. - base::Status CheckSourceNames(const op::Scan&, uint32_t at) const; + base::Status CheckSourceNames(const std::vector&, + uint32_t at) const; op::Scan CompileDataframeSource(const dataframe::Dataframe&, std::string name); base::StatusOr CompileSqlSource(uint32_t from); @@ -185,17 +203,13 @@ class Compiler { const NodeSourceFn& source_; const Catalog& catalog_; LogicalPlan plan_; - // The operator being compiled, which prefixes every error. - const char* op_ = "FROM"; - // The row as of the stage being compiled, and the table aliases in scope. - std::vector row_; - std::vector aliases_; + Scope scope_; }; -// The column of `scan` named `name`, or nothing when it has none. -std::optional FindScanColumn(const op::Scan& scan, - const std::string& name) { - for (const NamedColumn& column : scan.columns) { +// The column of a relation named `name`, or nothing when it has none. +std::optional FindColumn(const std::vector& columns, + const std::string& name) { + for (const NamedColumn& column : columns) { if (base::CaseInsensitiveEqual(column.name, name)) { return column.id; } @@ -204,7 +218,7 @@ std::optional FindScanColumn(const op::Scan& scan, } base::Status Compiler::CompileIntersection(uint32_t node) { - op_ = "INTERVAL INTERSECTION"; + scope_.op = "INTERVAL INTERSECTION"; const auto* n = Node(p_, node); const auto* list = Node(p_, n->operands); uint32_t count = syntaqlite_list_count(list); @@ -237,16 +251,9 @@ base::Status Compiler::CompileIntersection(uint32_t node) { if (FindAlias(*alias)) { return Expected(source_id, "a different alias for each relation"); } - op::Scan scan; - if (const dataframe::Dataframe* dataframe = FindDirectDataframe(*source)) { - scan = - CompileDataframeSource(*dataframe, SpanText(p_, source->table_name)); - } else { - ASSIGN_OR_RETURN(scan, CompileSqlSource(source_id)); - } - RETURN_IF_ERROR(CheckSourceNames(scan, source_id)); - std::optional ts = FindScanColumn(scan, "ts"); - std::optional dur = FindScanColumn(scan, "dur"); + ASSIGN_OR_RETURN(Relation relation, CompileRelation(source_id)); + std::optional ts = FindColumn(relation.columns, "ts"); + std::optional dur = FindColumn(relation.columns, "dur"); if (!ts || !dur) { return Expected(source_id, *alias + " to have a " + (ts ? "dur" : "ts") + " column"); @@ -258,7 +265,7 @@ base::Status Compiler::CompileIntersection(uint32_t node) { uint32_t col_id = syntaqlite_list_child_id(per, k); const auto* col = Node(p_, col_id); std::string name = SpanText(p_, col->name); - std::optional key = FindScanColumn(scan, name); + std::optional key = FindColumn(relation.columns, name); if (!key) { return Expected(col_id, *alias + " to have a " + name + " column"); } @@ -268,15 +275,15 @@ base::Status Compiler::CompileIntersection(uint32_t node) { // A PER column holds the same value in every operand, so the first // operand's is also the one a bare name finds. Alias operand_alias{*alias, {}}; - for (const NamedColumn& column : scan.columns) { + for (const NamedColumn& column : relation.columns) { bool is_key = std::find(operand.keys.begin(), operand.keys.end(), column.id) != operand.keys.end(); Append(column, source_id, /*qualified_only=*/!(is_key && i == 0)); - operand_alias.columns.push_back(row_.back()); + operand_alias.columns.push_back(scope_.row.back()); operand.carried.push_back(column.id); } - aliases_.push_back(std::move(operand_alias)); - children.push_back(plan_.AddNode(std::move(scan))); + scope_.aliases.push_back(std::move(operand_alias)); + children.push_back(relation.node); isect.operands.push_back(std::move(operand)); } plan_.AddNode(std::move(isect), std::move(children)); @@ -285,21 +292,29 @@ base::Status Compiler::CompileIntersection(uint32_t node) { base::Status Compiler::CompileSource(uint32_t from) { const auto* n = Node(p_, from); + ASSIGN_OR_RETURN(Relation relation, CompileRelation(from)); + for (NamedColumn& column : relation.columns) { + Append(std::move(column), from); + } + if (std::optional qualifier = SourceQualifier(*n)) { + scope_.aliases.push_back({std::move(*qualifier), scope_.row}); + } + return base::OkStatus(); +} + +base::StatusOr Compiler::CompileRelation(uint32_t source) { + const auto* n = Node(p_, source); op::Scan scan; if (const dataframe::Dataframe* dataframe = FindDirectDataframe(*n)) { scan = CompileDataframeSource(*dataframe, SpanText(p_, n->table_name)); } else { - ASSIGN_OR_RETURN(scan, CompileSqlSource(from)); - } - RETURN_IF_ERROR(CheckSourceNames(scan, from)); - for (const NamedColumn& column : scan.columns) { - Append(column, from); + ASSIGN_OR_RETURN(scan, CompileSqlSource(source)); } - if (std::optional qualifier = SourceQualifier(*n)) { - aliases_.push_back({std::move(*qualifier), row_}); - } - plan_.AddNode(std::move(scan)); - return base::OkStatus(); + RETURN_IF_ERROR(CheckSourceNames(scan.columns, source)); + Relation relation; + relation.columns = scan.columns; + relation.node = plan_.AddNode(std::move(scan)); + return relation; } const dataframe::Dataframe* Compiler::FindDirectDataframe( @@ -330,19 +345,18 @@ std::optional Compiler::SourceQualifier( return std::nullopt; } -base::Status Compiler::CheckSourceNames(const op::Scan& scan, +base::Status Compiler::CheckSourceNames(const std::vector& columns, uint32_t at) const { - for (size_t i = 0; i < scan.columns.size(); ++i) { + for (size_t i = 0; i < columns.size(); ++i) { for (size_t j = 0; j < i; ++j) { - if (base::CaseInsensitiveEqual(scan.columns[j].name, - scan.columns[i].name)) { + if (base::CaseInsensitiveEqual(columns[j].name, columns[i].name)) { return Expected(at, "distinct column names, but there are two named '" + - scan.columns[j].name + "'"); + columns[j].name + "'"); } } } - for (size_t i = 0; i < scan.columns.size(); ++i) { - const std::string& name = scan.columns[i].name; + for (size_t i = 0; i < columns.size(); ++i) { + const std::string& name = columns[i].name; // An expression is only named by its alias. if (name.empty()) { return Expected(at, "every column to have a name, but column " + @@ -457,7 +471,7 @@ base::Status Compiler::Err(uint32_t at, {"no such table alias: '", "'"}, }; const Shape& shape = kShapes[static_cast(error)]; - return base::ErrStatus("%s%s: %s%.*s%s%s", Traceback(at).c_str(), op_, + return base::ErrStatus("%s%s: %s%.*s%s%s", Traceback(at).c_str(), scope_.op, shape.prefix, static_cast(what.size()), what.data(), shape.suffix, detail.c_str()); } @@ -465,7 +479,7 @@ base::Status Compiler::Err(uint32_t at, std::string Compiler::Origin(const RowColumn& column) const { // A qualified name is something the user can write to pick this candidate. const std::string& name = column.column.name; - for (const Alias& alias : aliases_) { + for (const Alias& alias : scope_.aliases) { for (const RowColumn& aliased : alias.columns) { if (aliased.column.id == column.column.id && base::CaseInsensitiveEqual(aliased.column.name, name)) { @@ -493,7 +507,7 @@ std::string Compiler::AmbiguousCandidates( } const Compiler::Alias* Compiler::FindAlias(const std::string& name) const { - for (const Alias& alias : aliases_) { + for (const Alias& alias : scope_.aliases) { if (base::CaseInsensitiveEqual(alias.name, name)) { return &alias; } @@ -502,12 +516,12 @@ const Compiler::Alias* Compiler::FindAlias(const std::string& name) const { } void Compiler::RemoveAlias(const std::string& name) { - aliases_.erase(std::remove_if(aliases_.begin(), aliases_.end(), - [&](const Alias& alias) { - return base::CaseInsensitiveEqual(alias.name, - name); - }), - aliases_.end()); + scope_.aliases.erase( + std::remove_if(scope_.aliases.begin(), scope_.aliases.end(), + [&](const Alias& alias) { + return base::CaseInsensitiveEqual(alias.name, name); + }), + scope_.aliases.end()); } base::Status Compiler::NoSuchColumn(const std::string& name, @@ -522,10 +536,10 @@ base::StatusOr Compiler::FindInRow(const std::string& name, uint32_t at) const { std::vector matches; size_t found = 0; - for (size_t i = 0; i < row_.size(); ++i) { - if (!row_[i].qualified_only && - base::CaseInsensitiveEqual(row_[i].column.name, name)) { - matches.push_back(&row_[i]); + for (size_t i = 0; i < scope_.row.size(); ++i) { + if (!scope_.row[i].qualified_only && + base::CaseInsensitiveEqual(scope_.row[i].column.name, name)) { + matches.push_back(&scope_.row[i]); found = i; } } @@ -555,7 +569,7 @@ base::StatusOr Compiler::Resolve(const std::string& qualifier, uint32_t at) const { if (qualifier.empty()) { ASSIGN_OR_RETURN(size_t i, FindInRow(name, at)); - return row_[i].column.id; + return scope_.row[i].column.id; } std::string full_name = qualifier + "." + name; const Alias* alias = FindAlias(qualifier); @@ -628,7 +642,7 @@ base::StatusOr Compiler::CompileSum(uint32_t agg_id, uint32_t expr) { } base::Status Compiler::CompileTreeAccumulate(uint32_t stage) { - op_ = "TREE ACCUMULATE"; + scope_.op = "TREE ACCUMULATE"; const auto* n = Node(p_, stage); op::TreeAccumulate acc; @@ -677,10 +691,10 @@ base::StatusOr> Compiler::ExpandStar( columns = alias->columns; } else { // EXTEND only takes `alias.*`: a bare star would repeat the whole row. - if (std::string_view(op_) == "EXTEND") { + if (std::string_view(scope_.op) == "EXTEND") { return Expected(star_id, "a table alias before the star, as in `t.*`"); } - columns = row_; + columns = scope_.row; } if (syntaqlite_node_is_present(star->except)) { @@ -732,7 +746,7 @@ base::StatusOr> Compiler::ExpandStar( IsPresent(item->qualifier) ? SpanText(p_, item->qualifier) : ""; ASSIGN_OR_RETURN(ColumnId value, Resolve(qualifier, SpanText(p_, item->name), item_id)); - *matches.front() = {{std::move(target), value}, op_, item_id}; + *matches.front() = {{std::move(target), value}, scope_.op, item_id}; } } @@ -763,34 +777,34 @@ base::StatusOr> Compiler::CompileItems( if (IsPresent(item->alias)) { name = SpanText(p_, item->alias); } - columns.push_back({{std::move(name), id}, op_, item_id}); + columns.push_back({{std::move(name), id}, scope_.op, item_id}); } return columns; } // Replaces the row. What it leaves is a new table: no alias reaches into it. base::Status Compiler::CompileSelect(uint32_t stage) { - op_ = "SELECT"; + scope_.op = "SELECT"; const auto* n = Node(p_, stage); - ASSIGN_OR_RETURN(row_, CompileItems(n->columns)); - aliases_.clear(); + ASSIGN_OR_RETURN(scope_.row, CompileItems(n->columns)); + scope_.aliases.clear(); return base::OkStatus(); } // Adds columns to the row. Items see only the row before the stage, not each // other. base::Status Compiler::CompileExtend(uint32_t stage) { - op_ = "EXTEND"; + scope_.op = "EXTEND"; const auto* n = Node(p_, stage); ASSIGN_OR_RETURN(std::vector columns, CompileItems(n->columns)); - row_.insert(row_.end(), columns.begin(), columns.end()); + scope_.row.insert(scope_.row.end(), columns.begin(), columns.end()); return base::OkStatus(); } // Removes every column of each name. Aliases still reach the dropped columns, // except an alias of the same name, which the name now hides. base::Status Compiler::CompileDrop(uint32_t stage) { - op_ = "DROP"; + scope_.op = "DROP"; const auto* n = Node(p_, stage); const auto* list = Node(p_, n->columns); std::vector names; @@ -799,25 +813,26 @@ base::Status Compiler::CompileDrop(uint32_t stage) { std::string name = SpanText(p_, Node(p_, item_id)->name); RETURN_IF_ERROR(CheckListedOnce(names, name, item_id)); - bool found = std::any_of(row_.begin(), row_.end(), [&](const auto& c) { - return !c.qualified_only && - base::CaseInsensitiveEqual(c.column.name, name); - }); + bool found = + std::any_of(scope_.row.begin(), scope_.row.end(), [&](const auto& c) { + return !c.qualified_only && + base::CaseInsensitiveEqual(c.column.name, name); + }); if (!found) { return NoSuchColumn(name, item_id); } } for (const std::string& name : names) { - row_.erase(std::remove_if(row_.begin(), row_.end(), - [&](const RowColumn& c) { - return !c.qualified_only && - base::CaseInsensitiveEqual(c.column.name, - name); - }), - row_.end()); + scope_.row.erase(std::remove_if(scope_.row.begin(), scope_.row.end(), + [&](const RowColumn& c) { + return !c.qualified_only && + base::CaseInsensitiveEqual( + c.column.name, name); + }), + scope_.row.end()); RemoveAlias(name); } - if (row_.empty()) { + if (scope_.row.empty()) { return Expected(stage, "a column to be left after DROP"); } return base::OkStatus(); @@ -827,7 +842,7 @@ base::Status Compiler::CompileDrop(uint32_t stage) { // renames happen at once, so two columns can swap names. Aliases still reach // the columns under their old names. base::Status Compiler::CompileRename(uint32_t stage) { - op_ = "RENAME"; + scope_.op = "RENAME"; const auto* n = Node(p_, stage); const auto* list = Node(p_, n->columns); std::vector seen; @@ -842,9 +857,9 @@ base::Status Compiler::CompileRename(uint32_t stage) { } for (const auto& [at, item_id] : renames) { const auto* item = Node(p_, item_id); - row_[at].column.name = SpanText(p_, item->alias); - row_[at].op = op_; - row_[at].node = item_id; + scope_.row[at].column.name = SpanText(p_, item->alias); + scope_.row[at].op = scope_.op; + scope_.row[at].node = item_id; } return base::OkStatus(); } @@ -854,7 +869,7 @@ base::Status Compiler::CompileRename(uint32_t stage) { // still reach the old values, except an alias of the same name, which the // name now hides. base::Status Compiler::CompileSet(uint32_t stage) { - op_ = "SET"; + scope_.op = "SET"; const auto* n = Node(p_, stage); const auto* list = Node(p_, n->items); std::vector seen; @@ -878,9 +893,9 @@ base::Status Compiler::CompileSet(uint32_t stage) { assignments.push_back({at, id, item_id}); } for (const Assignment& assignment : assignments) { - RowColumn& column = row_[assignment.at]; + RowColumn& column = scope_.row[assignment.at]; column.column.id = assignment.value; - column.op = op_; + column.op = scope_.op; column.node = assignment.node; RemoveAlias(column.column.name); } @@ -889,16 +904,34 @@ base::Status Compiler::CompileSet(uint32_t stage) { // Replaces every alias with one covering the whole row as it is now. base::Status Compiler::CompileAs(uint32_t stage) { - op_ = "AS"; + scope_.op = "AS"; const auto* n = Node(p_, stage); - aliases_.clear(); - aliases_.push_back({SpanText(p_, n->alias), row_}); + scope_.aliases.clear(); + scope_.aliases.push_back({SpanText(p_, n->alias), scope_.row}); + return base::OkStatus(); +} + +base::Status Compiler::CompilePipeline(uint32_t pipeline) { + const auto& n = Node(p_, pipeline)->perfetto_pipeline; + if (syntaqlite_node_is_present(n.intersection)) { + RETURN_IF_ERROR(CompileIntersection(n.intersection)); + } else { + RETURN_IF_ERROR(CompileSource(n.from)); + } + if (!syntaqlite_node_is_present(n.stages)) { + return base::OkStatus(); + } + const auto* stages = Node(p_, n.stages); + uint32_t count = syntaqlite_list_count(stages); + for (uint32_t i = 0; i < count; i++) { + RETURN_IF_ERROR(CompileStage(syntaqlite_list_child_id(stages, i))); + } return base::OkStatus(); } LogicalPlan Compiler::Finish() { plan_.output.clear(); - for (const RowColumn& column : row_) { + for (const RowColumn& column : scope_.row) { plan_.output.push_back(column.column); } return std::move(plan_); @@ -910,21 +943,8 @@ base::StatusOr Compile(SyntaqliteParser* p, uint32_t pipeline, const NodeSourceFn& source, const Catalog& catalog) { - const auto& n = Node(p, pipeline)->perfetto_pipeline; Compiler compiler(p, source, catalog); - if (syntaqlite_node_is_present(n.intersection)) { - RETURN_IF_ERROR(compiler.CompileIntersection(n.intersection)); - } else { - RETURN_IF_ERROR(compiler.CompileSource(n.from)); - } - if (syntaqlite_node_is_present(n.stages)) { - const auto* stages = Node(p, n.stages); - uint32_t count = syntaqlite_list_count(stages); - for (uint32_t i = 0; i < count; i++) { - RETURN_IF_ERROR( - compiler.CompileStage(syntaqlite_list_child_id(stages, i))); - } - } + RETURN_IF_ERROR(compiler.CompilePipeline(pipeline)); LogicalPlan plan = compiler.Finish(); PruneColumns(plan); return plan; From e5a8795e584ef9d5f2d9b7022be04ad499488193 Mon Sep 17 00:00:00 2001 From: Lalit Maganti Date: Sun, 27 Sep 2026 20:54:32 +0100 Subject: [PATCH 2/2] tp: let semantic analysis see CTEs Semantic analysis skipped the definitions of a WITH clause, so a query reading a CTE could not be described: a pipeline could not read SQL which uses one. Each CTE is now in scope for the rest of its WITH clause and the query it belongs to, hiding any relation of the same name there, as in SQLite. A view still sees only what its name meant where it was defined, and a recursive CTE reads itself as a relation of unknown shape. What is analyzed is also read where it is in its statement: a query, or a relation named at some node, sees the CTEs in scope there. So the SQL a pipeline reads can name a CTE of the statement the pipeline is written in, once pipelines can be written inside one. --- src/perfetto_sql/analysis/relation.cc | 277 ++++++++++++++++-- src/perfetto_sql/analysis/relation.h | 11 +- .../analysis/relation_unittest.cc | 156 +++++++++- .../perfetto_sql_connection_unittest.cc | 13 + .../perfetto_sql/pipeline/compiler.cc | 2 +- 5 files changed, 426 insertions(+), 33 deletions(-) diff --git a/src/perfetto_sql/analysis/relation.cc b/src/perfetto_sql/analysis/relation.cc index 73ddf17c46a..0ae3f90d8b6 100644 --- a/src/perfetto_sql/analysis/relation.cc +++ b/src/perfetto_sql/analysis/relation.cc @@ -103,6 +103,22 @@ bool IsNatural(SyntaqliteJoinType type) { type == SYNTAQLITE_JOIN_TYPE_NATURAL_FULL; } +// The CTE definitions of `with`, in order. +std::vector CteDefinitions( + SyntaqliteParser* p, + const SyntaqliteWithClause& with) { + std::vector out; + const void* list = syntaqlite_parser_node(p, with.ctes); + uint32_t count = list ? syntaqlite_list_count(list) : 0; + for (uint32_t i = 0; i < count; ++i) { + const SyntaqliteNode* node = Node(p, syntaqlite_list_child_id(list, i)); + if (node && node->tag == SYNTAQLITE_NODE_CTE_DEFINITION) { + out.push_back(&node->cte_definition); + } + } + return out; +} + bool ContainsName(const std::vector& names, std::string_view name) { return std::any_of(names.begin(), names.end(), [&](std::string_view n) { @@ -157,6 +173,7 @@ class RelationAnalyzer::Impl { preserves_rows_ = true; views_.clear(); leaves_.clear(); + ctes_.clear(); } // The columns of the relation `name`. Its hidden columns, if any, are added @@ -165,9 +182,24 @@ class RelationAnalyzer::Impl { std::string_view name, int depth, std::vector& hidden); + // The columns of `name` as a query reads it: the innermost CTE of that name + // in scope, or else the relation `name`. + base::StatusOr> + Read(std::string_view name, int depth, std::vector& hidden); base::StatusOr> Select(SyntaqliteParser* p, uint32_t id, int depth); + // Brings into scope the CTEs visible at `at`. Fails if `at` is not where a + // query can be read, as the CTEs in scope there are not known. + base::Status EnterScope(SqlNode at) { + if (!Within(at.parser, syntaqlite_result_root(at.parser), at.id)) { + return base::ErrStatus( + "relation analysis: could not find the scope of the node being read"); + } + // Defining the CTEs says nothing about the rows of what is read at `at`. + preserves_rows_ = true; + return base::OkStatus(); + } bool preserves_rows() const { return preserves_rows_; } @@ -182,6 +214,12 @@ class RelationAnalyzer::Impl { std::vector hidden; }; using Scope = std::vector; + // A common table expression in scope. No columns means they could not be + // worked out, as for a recursive one while it is still being defined. + struct Cte { + std::string_view name; + std::optional> columns; + }; base::Status Sources(SyntaqliteParser* p, uint32_t id, @@ -192,6 +230,32 @@ class RelationAnalyzer::Impl { static ColumnLineage Lookup(const Scope&, std::string_view table, std::string_view column); + // Brings every CTE of `with` into scope, of unknown shape, and returns the + // index in `ctes_` of the first. As in SQLite, each CTE body sees all the + // CTEs of its WITH clause, later ones and itself included, so the names + // must all be in scope before any body is analyzed. One read before it is + // worked out has unknown shape, so it fails rather than reading a relation + // of the same name. + size_t DeclareCtes(SyntaqliteParser* p, const SyntaqliteWithClause& with); + // Works out the columns of the CTE declared at `ctes_[slot]`. + void DefineCte(SyntaqliteParser* p, + const SyntaqliteCteDefinition& cte, + size_t slot, + int depth); + // Brings into scope the CTEs defined on the way from `id` down to `at`, + // through the places a query can be read: statements, WITH clauses, + // compound selects, FROM clauses and pipeline sources. Returns whether `at` + // was reached; if not, the CTEs in scope are unchanged. + bool Within(SyntaqliteParser* p, uint32_t id, uint32_t at); + bool WithinSources(SyntaqliteParser* p, uint32_t id, uint32_t at); + // The innermost CTE in scope named `name`, if any. + const Cte* FindCte(std::string_view name) const; + // Renames `columns` to the names listed at `names`, if there is a list, as + // `CREATE VIEW v(a, b)` and `WITH t(a, b)` do. + static base::Status NameColumns(SyntaqliteParser* p, + uint32_t names, + std::string_view relation, + std::vector& columns); const Catalog& catalog_; // Lineage string_views point into each view's sql string and parse tree, so @@ -201,9 +265,174 @@ class RelationAnalyzer::Impl { // Lineage string_views point into each leaf relation's strings, so they are // kept at stable addresses for the same reason. std::vector> leaves_; + // The CTEs in scope of what is being analyzed, innermost last. + std::vector ctes_; bool preserves_rows_ = true; }; +size_t RelationAnalyzer::Impl::DeclareCtes(SyntaqliteParser* p, + const SyntaqliteWithClause& with) { + size_t first = ctes_.size(); + for (const SyntaqliteCteDefinition* cte : CteDefinitions(p, with)) { + ctes_.push_back({Text(p, cte->cte_name), std::nullopt}); + } + return first; +} + +void RelationAnalyzer::Impl::DefineCte(SyntaqliteParser* p, + const SyntaqliteCteDefinition& cte, + size_t slot, + int depth) { + base::StatusOr> columns = + Select(p, cte.select, depth); + if (columns.ok() && + NameColumns(p, cte.columns, ctes_[slot].name, *columns).ok()) { + ctes_[slot].columns = std::move(*columns); + } +} + +bool RelationAnalyzer::Impl::Within(SyntaqliteParser* p, + uint32_t id, + uint32_t at) { + if (id == at) { + return true; + } + const SyntaqliteNode* node = Node(p, id); + if (!node) { + return false; + } + switch (static_cast(node->tag)) { + case SYNTAQLITE_NODE_CREATE_VIEW_STMT: + return Within(p, node->create_view_stmt.select, at); + case SYNTAQLITE_NODE_CREATE_PERFETTO_VIEW_STMT: + return Within(p, node->create_perfetto_view_stmt.select, at); + case SYNTAQLITE_NODE_CREATE_PERFETTO_TABLE_STMT: + return Within(p, node->create_perfetto_table_stmt.select, at) || + Within(p, node->create_perfetto_table_stmt.pipeline, at); + case SYNTAQLITE_NODE_CREATE_PERFETTO_FUNCTION_STMT: + return Within(p, node->create_perfetto_function_stmt.select, at); + case SYNTAQLITE_NODE_CREATE_TABLE_STMT: + return Within(p, node->create_table_stmt.as_select, at); + case SYNTAQLITE_NODE_SELECT_STMT: + return WithinSources(p, node->select_stmt.from_clause, at); + case SYNTAQLITE_NODE_COMPOUND_SELECT: + return Within(p, node->compound_select.left, at) || + Within(p, node->compound_select.right, at); + case SYNTAQLITE_NODE_PERFETTO_PIPELINE: + return Within(p, node->perfetto_pipeline.from, at) || + Within(p, node->perfetto_pipeline.intersection, at); + case SYNTAQLITE_NODE_PERFETTO_INTERVAL_INTERSECTION: { + const void* list = syntaqlite_parser_node( + p, node->perfetto_interval_intersection.operands); + uint32_t count = list ? syntaqlite_list_count(list) : 0; + for (uint32_t i = 0; i < count; ++i) { + if (Within(p, syntaqlite_list_child_id(list, i), at)) { + return true; + } + } + return false; + } + case SYNTAQLITE_NODE_PERFETTO_PIPE_SOURCE: + return Within(p, node->perfetto_pipe_source.select, at); + case SYNTAQLITE_NODE_WITH_CLAUSE: { + const SyntaqliteWithClause& with = node->with_clause; + size_t outer = DeclareCtes(p, with); + size_t slot = outer; + for (const SyntaqliteCteDefinition* cte : CteDefinitions(p, with)) { + if (Within(p, cte->select, at)) { + return true; + } + DefineCte(p, *cte, slot++, 0); + } + if (Within(p, with.select, at)) { + return true; + } + ctes_.resize(outer); + return false; + } + default: + return false; + } +} + +bool RelationAnalyzer::Impl::WithinSources(SyntaqliteParser* p, + uint32_t id, + uint32_t at) { + if (id == at) { + return true; + } + const SyntaqliteNode* node = Node(p, id); + if (!node) { + return false; + } + switch (static_cast(node->tag)) { + case SYNTAQLITE_NODE_JOIN_CLAUSE: + return WithinSources(p, node->join_clause.left, at) || + WithinSources(p, node->join_clause.right, at); + case SYNTAQLITE_NODE_JOIN_PREFIX: + return WithinSources(p, node->join_prefix.source, at); + case SYNTAQLITE_NODE_SUBQUERY_TABLE_SOURCE: + return Within(p, node->subquery_table_source.select, at); + default: + return false; + } +} + +base::StatusOr> RelationAnalyzer::Impl::Read( + std::string_view name, + int depth, + std::vector& hidden) { + const Cte* cte = FindCte(name); + if (!cte) { + return Relation(name, depth, hidden); + } + // A CTE can filter or reorder its rows. + preserves_rows_ = false; + if (!cte->columns) { + return base::ErrStatus("relation analysis: CTE '%.*s' has unknown shape", + static_cast(name.size()), name.data()); + } + return *cte->columns; +} + +const RelationAnalyzer::Impl::Cte* RelationAnalyzer::Impl::FindCte( + std::string_view name) const { + for (auto it = ctes_.rbegin(); it != ctes_.rend(); ++it) { + if (base::CaseInsensitiveEqual(it->name, name)) { + return &*it; + } + } + return nullptr; +} + +base::Status RelationAnalyzer::Impl::NameColumns( + SyntaqliteParser* p, + uint32_t names, + std::string_view relation, + std::vector& columns) { + if (!syntaqlite_node_is_present(names)) { + return base::OkStatus(); + } + const void* list = syntaqlite_parser_node(p, names); + uint32_t count = syntaqlite_list_count(list); + if (count != columns.size()) { + return base::ErrStatus( + "relation analysis: '%.*s' names %u columns for %u results", + static_cast(relation.size()), relation.data(), count, + static_cast(columns.size())); + } + for (uint32_t i = 0; i < count; ++i) { + const SyntaqliteNode* column = Node(p, syntaqlite_list_child_id(list, i)); + if (!column || column->tag != SYNTAQLITE_NODE_COLUMN_REF) { + return base::ErrStatus("relation analysis: invalid column name in '%.*s'", + static_cast(relation.size()), + relation.data()); + } + columns[i].output_name = Text(p, column->column_ref.column); + } + return base::OkStatus(); +} + ColumnLineage RelationAnalyzer::Impl::Lookup(const Scope& scope, std::string_view table, std::string_view column) { @@ -300,9 +529,13 @@ base::Status RelationAnalyzer::Impl::Sources(SyntaqliteParser* p, if (const SyntaqliteNode* a = Node(p, node->table_ref.alias)) { alias = Text(p, a->ident_name.source); } + // A CTE hides any relation of the same name. It is never qualified by + // a schema. std::vector hidden; base::StatusOr> columns = - Relation(name, depth, hidden); + Text(p, node->table_ref.schema).empty() + ? Read(name, depth, hidden) + : Relation(name, depth, hidden); if (!columns.ok()) { scope->push_back({alias, std::nullopt, {}, {}}); return base::OkStatus(); @@ -421,9 +654,19 @@ RelationAnalyzer::Impl::Select(SyntaqliteParser* p, uint32_t id, int depth) { switch (static_cast(node->tag)) { case SYNTAQLITE_NODE_SELECT_STMT: return SelectStmt(p, node->select_stmt, depth); - case SYNTAQLITE_NODE_WITH_CLAUSE: + case SYNTAQLITE_NODE_WITH_CLAUSE: { preserves_rows_ = false; - return Select(p, node->with_clause.select, depth); + size_t outer = DeclareCtes(p, node->with_clause); + size_t slot = outer; + for (const SyntaqliteCteDefinition* cte : + CteDefinitions(p, node->with_clause)) { + DefineCte(p, *cte, slot++, depth); + } + base::StatusOr> columns = + Select(p, node->with_clause.select, depth); + ctes_.resize(outer); + return columns; + } case SYNTAQLITE_NODE_COMPOUND_SELECT: { preserves_rows_ = false; base::StatusOr> left = @@ -512,28 +755,13 @@ base::StatusOr> RelationAnalyzer::Impl::Relation( return base::ErrStatus("relation analysis: '%.*s' is not a view", static_cast(name.size()), name.data()); } + // A view sees none of the CTEs of the query reading it. + std::vector outer = std::exchange(ctes_, {}); base::StatusOr> columns = Select(p, select, depth + 1); + ctes_ = std::move(outer); RETURN_IF_ERROR(columns.status()); - if (syntaqlite_node_is_present(column_names)) { - const void* list = syntaqlite_parser_node(p, column_names); - uint32_t count = syntaqlite_list_count(list); - if (count != columns->size()) { - return base::ErrStatus( - "relation analysis: view '%.*s' names %u columns for %u results", - static_cast(name.size()), name.data(), count, - static_cast(columns->size())); - } - for (uint32_t i = 0; i < count; ++i) { - const SyntaqliteNode* column = Node(p, syntaqlite_list_child_id(list, i)); - if (!column || column->tag != SYNTAQLITE_NODE_COLUMN_REF) { - return base::ErrStatus( - "relation analysis: invalid column name in '%.*s'", - static_cast(name.size()), name.data()); - } - (*columns)[i].output_name = Text(p, column->column_ref.column); - } - } + RETURN_IF_ERROR(NameColumns(p, column_names, name, *columns)); return {columns}; } @@ -573,17 +801,20 @@ RelationAnalyzer::~RelationAnalyzer() = default; base::StatusOr RelationAnalyzer::AnalyzeQuery(SqlNode query) { impl_->Begin(); + RETURN_IF_ERROR(impl_->EnterScope(query)); ASSIGN_OR_RETURN(auto columns, impl_->Select(query.parser, query.id, 0)); return RelationLineage(std::make_unique( std::move(columns), impl_->preserves_rows())); } base::StatusOr RelationAnalyzer::AnalyzeRelation( + SqlNode at, std::string_view name) { impl_->Begin(); + RETURN_IF_ERROR(impl_->EnterScope(at)); // Hidden columns are still columns of the relation itself. std::vector hidden; - ASSIGN_OR_RETURN(auto columns, impl_->Relation(name, 0, hidden)); + ASSIGN_OR_RETURN(auto columns, impl_->Read(name, 0, hidden)); return RelationLineage(std::make_unique( std::move(columns), impl_->preserves_rows())); } diff --git a/src/perfetto_sql/analysis/relation.h b/src/perfetto_sql/analysis/relation.h index a8afbb3d0eb..6ec2b34c032 100644 --- a/src/perfetto_sql/analysis/relation.h +++ b/src/perfetto_sql/analysis/relation.h @@ -31,7 +31,7 @@ struct SyntaqliteParser; namespace perfetto::perfetto_sql::analysis { -// A node in a caller-owned Syntaqlite parse tree. +// A node of the statement a caller-owned Syntaqlite parser last parsed. struct SqlNode { SyntaqliteParser* parser; uint32_t id; @@ -133,8 +133,13 @@ class RelationAnalyzer { RelationAnalyzer(const RelationAnalyzer&) = delete; RelationAnalyzer& operator=(const RelationAnalyzer&) = delete; - base::StatusOr AnalyzeQuery(SqlNode); - base::StatusOr AnalyzeRelation(std::string_view name); + // The lineage of `query`, read where it is in its statement: it sees the + // CTEs in scope there. + base::StatusOr AnalyzeQuery(SqlNode query); + // The lineage of the relation `name`, read at the node `at`: a CTE in scope + // there hides any relation of the same name. + base::StatusOr AnalyzeRelation(SqlNode at, + std::string_view name); private: class Impl; diff --git a/src/perfetto_sql/analysis/relation_unittest.cc b/src/perfetto_sql/analysis/relation_unittest.cc index a83762dc243..db05d131539 100644 --- a/src/perfetto_sql/analysis/relation_unittest.cc +++ b/src/perfetto_sql/analysis/relation_unittest.cc @@ -134,6 +134,45 @@ class RelationAnalyzerTest : public ::testing::Test { return result.ok() ? Show(*result) : std::vector{}; } + // Analyzes `name`, or with `name` empty the query itself, as read at the + // select in `sql` whose text is `at`. An error is shown as "!". + std::vector ReadAt(const std::string& sql, + const std::string& at, + const std::string& name) { + ScopedParser parser(syntaqlite_parser_create_perfetto(nullptr)); + syntaqlite_parser_set_collect_node_extents(parser.get(), 1); + syntaqlite_parser_reset(parser.get(), sql.data(), + static_cast(sql.size())); + if (syntaqlite_parser_next(parser.get()) != SYNTAQLITE_PARSE_OK) { + return {"!could not parse test query"}; + } + std::optional found; + for (uint32_t id = 0; id < syntaqlite_parser_node_count(parser.get()); + ++id) { + const auto* node = static_cast( + syntaqlite_parser_node(parser.get(), id)); + SyntaqliteLength len = 0; + SyntaqliteStmtOffset offset = 0; + const char* text = + syntaqlite_parser_node_text(parser.get(), id, &len, &offset); + if (node && node->tag == SYNTAQLITE_NODE_SELECT_STMT && text && + std::string_view(text, len) == at) { + found = id; + } + } + if (!found) { + return {"!no select '" + at + "'"}; + } + RelationAnalyzer analyzer(catalog_); + auto result = name.empty() + ? analyzer.AnalyzeQuery({parser.get(), *found}) + : analyzer.AnalyzeRelation({parser.get(), *found}, name); + if (!result.ok()) { + return {"!" + result.status().message()}; + } + return Show(*result); + } + TestCatalog catalog_; }; @@ -201,6 +240,114 @@ TEST_F(RelationAnalyzerTest, FollowsAliasesSubqueriesAndViews) { testing::ElementsAre("renamed=slice.id")); } +// CTEs are in scope for the rest of their WITH clause and its query, hiding +// relations of the same name there, as in SQLite. +TEST_F(RelationAnalyzerTest, Ctes) { + // Each sees those before it, and is renamed by its list of names. + EXPECT_THAT(Select(R"( + WITH + a(x) AS (SELECT id FROM slice), + b AS (SELECT x AS y, name FROM a, slice) + SELECT * FROM (SELECT * FROM b) + )"), + testing::ElementsAre("y=slice.id", "name=slice.name")); + // A CTE hides a table of the same name in its query, but not outside the + // query or in a view the query reads. + catalog_.AddView("v", "CREATE VIEW v AS SELECT * FROM thread"); + EXPECT_THAT(Select(R"( + SELECT * + FROM ( + WITH thread AS (SELECT id FROM slice) + SELECT * FROM thread, v + ), + thread + )"), + testing::ElementsAre("id=slice.id", "utid=thread.utid", + "name=thread.name", "utid=thread.utid", + "name=thread.name")); + // Every CTE of a WITH clause is in scope in all of its definitions, so a + // CTE read before its columns are known, by itself or by an earlier CTE, + // is never mistaken for a table of the same name. + EXPECT_THAT(Select(R"( + WITH + a AS (SELECT utid FROM thread), + thread AS (SELECT id AS utid FROM slice) + SELECT * FROM a + )"), + testing::ElementsAre("utid=")); + EXPECT_THAT(Select(R"( + WITH thread AS (SELECT utid FROM thread) + SELECT * FROM thread + )"), + testing::ElementsAre("utid=")); + // A recursive CTE reads itself before its columns are known. + EXPECT_THAT(Select(R"( + WITH RECURSIVE n(i) AS ( + SELECT id FROM slice + UNION ALL + SELECT i + 1 FROM n WHERE i < 3 + ) + SELECT * FROM n + )"), + testing::ElementsAre("i=")); + // A CTE can filter its rows, so what reads one has no row origin. + auto result = + Analyze("WITH t AS (SELECT id FROM slice WHERE ts > 5) SELECT id FROM t"); + ASSERT_TRUE(result.ok()); + EXPECT_EQ(result->row_origin(), std::nullopt); +} + +// What is read at a node of a statement sees the CTEs in scope there, and only +// those. +TEST_F(RelationAnalyzerTest, ReadsSeeTheCtesInScope) { + const std::string kHere = "SELECT 1 AS here"; + // Through statements, compound selects and joins; the innermost CTE of a + // name hides the others. + const std::string kNested = R"( + CREATE VIEW v AS + WITH t AS (SELECT utid FROM thread) + SELECT 1, 2, 3 + UNION ALL + SELECT * FROM thread JOIN ( + WITH t AS (SELECT id FROM slice) + SELECT * FROM (SELECT 1 AS here) + ) + )"; + EXPECT_THAT(ReadAt(kNested, kHere, "t"), testing::ElementsAre("id=slice.id")); + // A query read there. + EXPECT_THAT(ReadAt(R"( + WITH t AS (SELECT id FROM slice) + SELECT * FROM (SELECT * FROM t) + )", + "SELECT * FROM t", ""), + testing::ElementsAre("id=slice.id")); + // Inside a CTE's definition, all the CTEs of its WITH clause, but only those + // before it are known yet; and none of another subquery's. + const std::string kDefinition = R"( + SELECT * + FROM (WITH s AS (SELECT id FROM slice) SELECT * FROM s), + ( + WITH + a AS (SELECT id FROM slice), + b AS (SELECT * FROM (SELECT 1 AS here)), + c AS (SELECT utid FROM thread) + SELECT * FROM b + ) + )"; + EXPECT_THAT(ReadAt(kDefinition, kHere, "a"), + testing::ElementsAre("id=slice.id")); + EXPECT_THAT( + ReadAt(kDefinition, kHere, "c"), + testing::ElementsAre("!relation analysis: CTE 'c' has unknown shape")); + EXPECT_THAT(ReadAt(kDefinition, kHere, "s"), + testing::ElementsAre("!relation analysis: 's' is not known")); + // A node where no query can be read has no known scope. + EXPECT_THAT( + ReadAt("SELECT * FROM slice WHERE id IN (SELECT 1 AS here)", kHere, "t"), + testing::ElementsAre("!relation analysis: could not find the scope of " + "the node being read")); +} + TEST_F(RelationAnalyzerTest, KeepsEveryOriginOfAmbiguousJoinColumn) { EXPECT_THAT(Select("SELECT name FROM slice, thread"), testing::ElementsAre("name=slice.name,thread.name")); @@ -237,8 +384,7 @@ TEST_F(RelationAnalyzerTest, IdentifiesRowOrigin) { TEST_F(RelationAnalyzerTest, DetectsRowPreservingViewChains) { catalog_.AddView("v1", "CREATE VIEW v1 AS SELECT id, name FROM slice"); catalog_.AddView("v2", "CREATE VIEW v2 AS SELECT id AS a, name AS b FROM v1"); - RelationAnalyzer analyzer(catalog_); - auto result = analyzer.AnalyzeRelation("v2"); + auto result = Analyze("SELECT * FROM v2"); ASSERT_TRUE(result.ok()); EXPECT_EQ(result->row_origin(), std::make_optional("slice")); @@ -246,8 +392,7 @@ TEST_F(RelationAnalyzerTest, DetectsRowPreservingViewChains) { TEST_F(RelationAnalyzerTest, RejectsFilteredViewAsRowOrigin) { catalog_.AddView("v", "CREATE VIEW v AS SELECT id FROM slice WHERE ts > 5"); - RelationAnalyzer analyzer(catalog_); - auto result = analyzer.AnalyzeRelation("v"); + auto result = Analyze("SELECT * FROM v"); ASSERT_TRUE(result.ok()); EXPECT_EQ(result->row_origin(), std::nullopt); } @@ -255,8 +400,7 @@ TEST_F(RelationAnalyzerTest, RejectsFilteredViewAsRowOrigin) { TEST_F(RelationAnalyzerTest, RejectsOrderedViewAsRowOrigin) { catalog_.AddView("v", "CREATE VIEW v AS SELECT id FROM slice ORDER BY ts DESC"); - RelationAnalyzer analyzer(catalog_); - auto result = analyzer.AnalyzeRelation("v"); + auto result = Analyze("SELECT * FROM v"); ASSERT_TRUE(result.ok()); EXPECT_EQ(result->row_origin(), std::nullopt); } diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_connection_unittest.cc b/src/trace_processor/perfetto_sql/engine/perfetto_sql_connection_unittest.cc index 536309a3c42..ee06d4d49ad 100644 --- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_connection_unittest.cc +++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_connection_unittest.cc @@ -903,6 +903,19 @@ TEST_F(PerfettoSqlConnectionPipelineTest, StartsFromAnySql) { "2,0,60,60", "3,1,80,80")); } +TEST_F(PerfettoSqlConnectionPipelineTest, StartsFromSqlWithCtes) { + auto rows = Rows(R"( + FROM ( + WITH nodes AS (SELECT id, parent_id, self FROM tree) + SELECT * FROM nodes + ) + |> TREE ACCUMULATE UP SUM(self) AS total + )"); + ASSERT_TRUE(rows.ok()) << rows.status().c_message(); + EXPECT_THAT(*rows, testing::ElementsAre("0,NULL,10,100", "1,0,20,60", + "2,0,30,30", "3,1,40,40")); +} + // Perfetto tables are dataframes and are scanned directly. TEST_F(PerfettoSqlConnectionPipelineTest, StartsFromAPerfettoTable) { auto rows = Rows(R"( diff --git a/src/trace_processor/perfetto_sql/pipeline/compiler.cc b/src/trace_processor/perfetto_sql/pipeline/compiler.cc index 7262e2f8bc4..942b2a32cc7 100644 --- a/src/trace_processor/perfetto_sql/pipeline/compiler.cc +++ b/src/trace_processor/perfetto_sql/pipeline/compiler.cc @@ -404,7 +404,7 @@ base::StatusOr Compiler::CompileSqlSource(uint32_t from) { base::StatusOr lineage = syntaqlite_node_is_present(n->select) ? analyzer.AnalyzeQuery({p_, n->select}) - : analyzer.AnalyzeRelation(SpanText(p_, n->table_name)); + : analyzer.AnalyzeRelation({p_, from}, SpanText(p_, n->table_name)); if (!lineage.ok()) { return Err(from, Error::kUnsupported, "reading a relation whose columns cannot be worked out",