From 2efc2d8db207624e524210fdfbb6761442a0fba9 Mon Sep 17 00:00:00 2001 From: Lalit Maganti Date: Sun, 27 Sep 2026 20:44:43 +0100 Subject: [PATCH] tp: prepare pipelines to read SQL through semantic analysis No behaviour change. Moves out what the next change builds on: - the SQL which reads a plan, from the table function into its own pipeline_sql.h, as anything writing SQL for a plan needs it; - the pipeline catalog now extends the analysis catalog, as every catalog already implemented both; - a leaf relation owns its names, and the analyzer keeps the ones it reads for as long as the analysis, so a catalog can describe a relation it has no lasting names for. --- Android.bp | 1 + BUILD | 2 + src/perfetto_sql/analysis/relation.cc | 8 +++- src/perfetto_sql/analysis/relation.h | 7 ++- .../perfetto_sql/engine/connection_catalog.h | 3 +- .../engine/perfetto_sql_connection.cc | 5 +- .../perfetto_sql/engine/pipeline_module.cc | 23 ++-------- .../perfetto_sql/engine/pipeline_module.h | 7 --- .../perfetto_sql/pipeline/BUILD.gn | 3 ++ .../perfetto_sql/pipeline/catalog.h | 9 ++-- .../perfetto_sql/pipeline/pipeline_sql.cc | 46 +++++++++++++++++++ .../perfetto_sql/pipeline/pipeline_sql.h | 40 ++++++++++++++++ .../perfetto_sql/pipeline/test_catalog.h | 2 +- 13 files changed, 116 insertions(+), 40 deletions(-) create mode 100644 src/trace_processor/perfetto_sql/pipeline/pipeline_sql.cc create mode 100644 src/trace_processor/perfetto_sql/pipeline/pipeline_sql.h diff --git a/Android.bp b/Android.bp index 11cca4a6389..1d0e9450a5a 100644 --- a/Android.bp +++ b/Android.bp @@ -19081,6 +19081,7 @@ filegroup { "src/trace_processor/perfetto_sql/pipeline/catalog.cc", "src/trace_processor/perfetto_sql/pipeline/column_pruning.cc", "src/trace_processor/perfetto_sql/pipeline/compiler.cc", + "src/trace_processor/perfetto_sql/pipeline/pipeline_sql.cc", "src/trace_processor/perfetto_sql/pipeline/plan_serialization.cc", ], } diff --git a/BUILD b/BUILD index b2a00670d02..a8bbfafd0ff 100644 --- a/BUILD +++ b/BUILD @@ -3881,6 +3881,8 @@ perfetto_filegroup( "src/trace_processor/perfetto_sql/pipeline/compiler.cc", "src/trace_processor/perfetto_sql/pipeline/compiler.h", "src/trace_processor/perfetto_sql/pipeline/logical_plan.h", + "src/trace_processor/perfetto_sql/pipeline/pipeline_sql.cc", + "src/trace_processor/perfetto_sql/pipeline/pipeline_sql.h", "src/trace_processor/perfetto_sql/pipeline/plan_serialization.cc", "src/trace_processor/perfetto_sql/pipeline/plan_serialization.h", ], diff --git a/src/perfetto_sql/analysis/relation.cc b/src/perfetto_sql/analysis/relation.cc index 29477fa9242..468afcdc815 100644 --- a/src/perfetto_sql/analysis/relation.cc +++ b/src/perfetto_sql/analysis/relation.cc @@ -150,6 +150,7 @@ class RelationAnalyzer::Impl { void Begin() { preserves_rows_ = true; views_.clear(); + leaves_.clear(); } // The columns of the relation `name`. Its hidden columns, if any, are added @@ -191,6 +192,9 @@ class RelationAnalyzer::Impl { // every OwnedView needs a stable address: growing a std::vector // would move the elements and moving `sql` can relocate its bytes (SSO). std::vector> views_; + // Lineage string_views point into each leaf relation's strings, so they are + // kept at stable addresses for the same reason. + std::vector> leaves_; bool preserves_rows_ = true; }; @@ -445,7 +449,9 @@ base::StatusOr> RelationAnalyzer::Impl::Relation( std::string_view name, int depth, std::vector& hidden) { - if (std::optional relation = catalog_.FindLeafRelation(name)) { + if (std::optional found = catalog_.FindLeafRelation(name)) { + leaves_.push_back(std::make_unique(std::move(*found))); + const LeafRelation* relation = leaves_.back().get(); std::vector out; out.reserve(relation->columns.size()); for (const LeafColumn& column : relation->columns) { diff --git a/src/perfetto_sql/analysis/relation.h b/src/perfetto_sql/analysis/relation.h index c832464a9ad..a8afbb3d0eb 100644 --- a/src/perfetto_sql/analysis/relation.h +++ b/src/perfetto_sql/analysis/relation.h @@ -50,7 +50,7 @@ using ColumnType = base::TypeSet; // A leaf relation whose columns can be used as lineage origins. struct LeafColumn { - std::string_view name; + std::string name; // Nothing when the catalog does not know how the column is stored. std::optional type; // Left out of `*` and `table.*`, as SQLite does for HIDDEN columns, but @@ -58,12 +58,11 @@ struct LeafColumn { bool hidden = false; }; struct LeafRelation { - std::string_view name; + std::string name; std::vector columns; }; -// Supplies the schema objects referenced by parsed queries. Returned leaf -// strings only need to remain valid for the duration of an Analyze call. +// Supplies the schema objects referenced by parsed queries. class Catalog { public: virtual ~Catalog(); diff --git a/src/trace_processor/perfetto_sql/engine/connection_catalog.h b/src/trace_processor/perfetto_sql/engine/connection_catalog.h index 7221007cf1a..d343a7a3e71 100644 --- a/src/trace_processor/perfetto_sql/engine/connection_catalog.h +++ b/src/trace_processor/perfetto_sql/engine/connection_catalog.h @@ -33,8 +33,7 @@ namespace perfetto::trace_processor { // Adapts a connection to semantic analysis and pipeline compilation. Each // dataframe is served as a typed leaf relation and as a dataframe. -class ConnectionCatalog final : public perfetto_sql::analysis::Catalog, - public pipeline::Catalog { +class ConnectionCatalog final : public pipeline::Catalog { public: explicit ConnectionCatalog(PerfettoSqlConnection*); diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_connection.cc b/src/trace_processor/perfetto_sql/engine/perfetto_sql_connection.cc index 8070ed21805..cfa7433413a 100644 --- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_connection.cc +++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_connection.cc @@ -57,6 +57,7 @@ #include "src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h" #include "src/trace_processor/perfetto_sql/pipeline/logical_plan.h" #include "src/trace_processor/perfetto_sql/pipeline/physical_plan.h" +#include "src/trace_processor/perfetto_sql/pipeline/pipeline_sql.h" #include "src/trace_processor/perfetto_sql/pipeline/plan_serialization.h" #include "src/trace_processor/sqlite/bindings/sqlite_column.h" #include "src/trace_processor/sqlite/bindings/sqlite_type.h" @@ -361,7 +362,7 @@ PerfettoSqlConnection::PerfettoSqlConnection( auto ctx = std::make_unique(); ctx->pool = pool_; ctx->connection = this; - RegisterVirtualTableModule(PipelineModule::kName, + RegisterVirtualTableModule(pipeline::kPipelineFunction, std::move(ctx)); } database_->InitializeSharedSchema(connection_.get()); @@ -1051,7 +1052,7 @@ base::Status PerfettoSqlConnection::ExecuteCreateTable( base::StatusOr PerfettoSqlConnection::PreparePipeline(const pipeline::LogicalPlan& plan, const SqlSource& source) { - auto sql = PipelineModule::SelectFrom(plan); + auto sql = pipeline::SelectPipelineSql(plan); if (!sql.ok()) { return base::ErrStatus("%s%s", source.AsTraceback(0).c_str(), sql.status().c_message()); diff --git a/src/trace_processor/perfetto_sql/engine/pipeline_module.cc b/src/trace_processor/perfetto_sql/engine/pipeline_module.cc index 726f253f355..fd3f04852b9 100644 --- a/src/trace_processor/perfetto_sql/engine/pipeline_module.cc +++ b/src/trace_processor/perfetto_sql/engine/pipeline_module.cc @@ -37,9 +37,8 @@ #include "src/trace_processor/core/exec/variant.h" #include "src/trace_processor/core/util/bit_vector.h" #include "src/trace_processor/perfetto_sql/engine/perfetto_sql_connection.h" -#include "src/trace_processor/perfetto_sql/pipeline/logical_plan.h" #include "src/trace_processor/perfetto_sql/pipeline/physical_plan.h" -#include "src/trace_processor/perfetto_sql/pipeline/plan_serialization.h" +#include "src/trace_processor/perfetto_sql/pipeline/pipeline_sql.h" #include "src/trace_processor/sqlite/bindings/sqlite_result.h" #include "src/trace_processor/sqlite/sqlite_utils.h" @@ -57,7 +56,7 @@ constexpr int kFirstOutputColumn = 1; std::string Schema() { // Public names (which may repeat) are applied by the outer SELECT. std::vector columns{"pipeline HIDDEN"}; - for (uint32_t i = 0; i < PipelineModule::kMaxColumns; ++i) { + for (uint32_t i = 0; i < pipeline::kMaxPipelineColumns; ++i) { columns.push_back("c" + std::to_string(i)); } return "CREATE TABLE x(" + base::Join(columns, ", ") + ")"; @@ -198,7 +197,7 @@ PERFETTO_NO_INLINE int Load(PipelineModule::Cursor* c, sqlite3_value* value) { c->rows = std::make_unique(c->plan->source()); // One reader per declared column, so Column only indexes: arguments read as // null, and columns past the plan's outputs fail. - c->columns.assign(kFirstOutputColumn + PipelineModule::kMaxColumns, + c->columns.assign(kFirstOutputColumn + pipeline::kMaxPipelineColumns, {&c->no_view, &ResultNoColumn}); for (int i = 0; i < kFirstOutputColumn; ++i) { c->columns[static_cast(i)] = {&c->no_view, &ResultNull}; @@ -208,22 +207,6 @@ PERFETTO_NO_INLINE int Load(PipelineModule::Cursor* c, sqlite3_value* value) { } // namespace -base::StatusOr PipelineModule::SelectFrom( - const pipeline::LogicalPlan& plan) { - const std::vector& output = plan.output; - if (output.size() > kMaxColumns) { - return base::ErrStatus("A pipeline can output at most %u columns, not %zu", - kMaxColumns, output.size()); - } - std::vector columns; - for (uint32_t i = 0; i < output.size(); ++i) { - columns.push_back("c" + std::to_string(i) + " AS \"" + - base::ReplaceAll(output[i].name, "\"", "\"\"") + "\""); - } - return "SELECT " + base::Join(columns, ", ") + " FROM " + kName + "(X'" + - base::ToHex(pipeline::SerializePlan(plan)) + "')"; -} - int PipelineModule::Connect(sqlite3* db, void* raw_ctx, int, diff --git a/src/trace_processor/perfetto_sql/engine/pipeline_module.h b/src/trace_processor/perfetto_sql/engine/pipeline_module.h index 9b7db15c3be..f162779b144 100644 --- a/src/trace_processor/perfetto_sql/engine/pipeline_module.h +++ b/src/trace_processor/perfetto_sql/engine/pipeline_module.h @@ -24,10 +24,8 @@ #include #include -#include "perfetto/ext/base/status_or.h" #include "src/trace_processor/containers/string_pool.h" #include "src/trace_processor/core/exec/row_cursor.h" -#include "src/trace_processor/perfetto_sql/pipeline/logical_plan.h" #include "src/trace_processor/perfetto_sql/pipeline/physical_plan.h" #include "src/trace_processor/sqlite/bindings/sqlite_module.h" @@ -43,8 +41,6 @@ struct PipelineModule : sqlite::Module { static constexpr auto kType = kEponymousOnly; static constexpr bool kSupportsWrites = false; static constexpr bool kDoesOverloadFunctions = false; - static constexpr char kName[] = "__intrinsic_pipeline"; - static constexpr uint32_t kMaxColumns = 256; struct Context { StringPool* pool; @@ -73,9 +69,6 @@ struct PipelineModule : sqlite::Module { bool eof = true; }; - // SQL reading `plan`'s output under its own column names. - static base::StatusOr SelectFrom(const pipeline::LogicalPlan&); - static int Connect(sqlite3*, void*, int, diff --git a/src/trace_processor/perfetto_sql/pipeline/BUILD.gn b/src/trace_processor/perfetto_sql/pipeline/BUILD.gn index afa4fdc9eb0..84da7184395 100644 --- a/src/trace_processor/perfetto_sql/pipeline/BUILD.gn +++ b/src/trace_processor/perfetto_sql/pipeline/BUILD.gn @@ -26,12 +26,15 @@ source_set("logical") { "compiler.cc", "compiler.h", "logical_plan.h", + "pipeline_sql.cc", + "pipeline_sql.h", "plan_serialization.cc", "plan_serialization.h", ] deps = [ "../../../../gn:default_deps", "../../../base", + "../../../perfetto_sql/analysis", "../../../perfetto_sql/syntaqlite", "../../core/common", "../../core/dataframe", diff --git a/src/trace_processor/perfetto_sql/pipeline/catalog.h b/src/trace_processor/perfetto_sql/pipeline/catalog.h index 7ab1fe9c912..7dee828b010 100644 --- a/src/trace_processor/perfetto_sql/pipeline/catalog.h +++ b/src/trace_processor/perfetto_sql/pipeline/catalog.h @@ -20,16 +20,19 @@ #include #include "perfetto/ext/base/status_or.h" +#include "src/perfetto_sql/analysis/relation.h" #include "src/trace_processor/core/dataframe/dataframe.h" #include "src/trace_processor/perfetto_sql/pipeline/logical_plan.h" #include "src/trace_processor/sqlite/sql_source.h" namespace perfetto::trace_processor::pipeline { -// Lookup interface the compiler uses to resolve what a pipeline reads. -class Catalog { +// Lookup interface the compiler uses to resolve what a pipeline reads: the +// relations semantic analysis can describe, and the dataframes a pipeline can +// read directly. +class Catalog : public perfetto_sql::analysis::Catalog { public: - virtual ~Catalog(); + ~Catalog() override; // Dataframe registered as `name`, or null. virtual const dataframe::Dataframe* FindDataframe( diff --git a/src/trace_processor/perfetto_sql/pipeline/pipeline_sql.cc b/src/trace_processor/perfetto_sql/pipeline/pipeline_sql.cc new file mode 100644 index 00000000000..74aebb5e1c4 --- /dev/null +++ b/src/trace_processor/perfetto_sql/pipeline/pipeline_sql.cc @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/trace_processor/perfetto_sql/pipeline/pipeline_sql.h" + +#include +#include +#include + +#include "perfetto/base/status.h" +#include "perfetto/ext/base/status_or.h" +#include "perfetto/ext/base/string_utils.h" +#include "src/trace_processor/perfetto_sql/pipeline/logical_plan.h" +#include "src/trace_processor/perfetto_sql/pipeline/plan_serialization.h" + +namespace perfetto::trace_processor::pipeline { + +base::StatusOr SelectPipelineSql(const LogicalPlan& plan) { + if (plan.output.size() > kMaxPipelineColumns) { + return base::ErrStatus("A pipeline can output at most %u columns, not %zu", + kMaxPipelineColumns, plan.output.size()); + } + std::vector columns; + for (uint32_t i = 0; i < plan.output.size(); ++i) { + columns.push_back("c" + std::to_string(i) + " AS \"" + + base::ReplaceAll(plan.output[i].name, "\"", "\"\"") + + "\""); + } + return "SELECT " + base::Join(columns, ", ") + " FROM " + kPipelineFunction + + "(X'" + base::ToHex(SerializePlan(plan)) + "')"; +} + +} // namespace perfetto::trace_processor::pipeline diff --git a/src/trace_processor/perfetto_sql/pipeline/pipeline_sql.h b/src/trace_processor/perfetto_sql/pipeline/pipeline_sql.h new file mode 100644 index 00000000000..a2dbf43b27e --- /dev/null +++ b/src/trace_processor/perfetto_sql/pipeline/pipeline_sql.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_PIPELINE_PIPELINE_SQL_H_ +#define SRC_TRACE_PROCESSOR_PERFETTO_SQL_PIPELINE_PIPELINE_SQL_H_ + +#include +#include + +#include "perfetto/ext/base/status_or.h" +#include "src/trace_processor/perfetto_sql/pipeline/logical_plan.h" + +namespace perfetto::trace_processor::pipeline { + +// The table function which runs a serialized plan. +inline constexpr char kPipelineFunction[] = "__intrinsic_pipeline"; + +// The most columns the table function can output. +inline constexpr uint32_t kMaxPipelineColumns = 256; + +// SQL reading `plan`'s output under its own column names. The plan is +// serialized into the SQL, so the SQL needs nothing else to run. +base::StatusOr SelectPipelineSql(const LogicalPlan& plan); + +} // namespace perfetto::trace_processor::pipeline + +#endif // SRC_TRACE_PROCESSOR_PERFETTO_SQL_PIPELINE_PIPELINE_SQL_H_ diff --git a/src/trace_processor/perfetto_sql/pipeline/test_catalog.h b/src/trace_processor/perfetto_sql/pipeline/test_catalog.h index 10383d20f49..20a5e89eb68 100644 --- a/src/trace_processor/perfetto_sql/pipeline/test_catalog.h +++ b/src/trace_processor/perfetto_sql/pipeline/test_catalog.h @@ -43,7 +43,7 @@ namespace perfetto::trace_processor::pipeline { // Catalog over dataframes built by the test. SQLite does not know about them. // Anything else is described via `connection` (if given) and is untyped. -class TestCatalog : public Catalog, public perfetto_sql::analysis::Catalog { +class TestCatalog : public Catalog { public: explicit TestCatalog(StringPool* pool, SqliteConnection* connection = nullptr) : pool_(pool), connection_(connection) {}