tp: pass the SQL a pipeline reads to it as dataframes

Android.bp

@@ -3157,7 +3157,6 @@ cc_test {
         ":perfetto_src_trace_processor_metatrace",
         ":perfetto_src_trace_processor_metrics_metrics",
         ":perfetto_src_trace_processor_perfetto_sql_engine_engine",
-        ":perfetto_src_trace_processor_perfetto_sql_exec_exec",
         ":perfetto_src_trace_processor_perfetto_sql_generator_generator",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
         ":perfetto_src_trace_processor_perfetto_sql_parser_parser",
@@ -19000,22 +18999,6 @@ filegroup {
     ],
 }
 
-// GN: //src/trace_processor/perfetto_sql/exec:exec
-filegroup {
-    name: "perfetto_src_trace_processor_perfetto_sql_exec_exec",
-    srcs: [
-        "src/trace_processor/perfetto_sql/exec/sql_scan.cc",
-    ],
-}
-
-// GN: //src/trace_processor/perfetto_sql/exec:unittests
-filegroup {
-    name: "perfetto_src_trace_processor_perfetto_sql_exec_unittests",
-    srcs: [
-        "src/trace_processor/perfetto_sql/exec/sql_scan_unittest.cc",
-    ],
-}
-
 // GN: //src/trace_processor/perfetto_sql/generator:gen_cc_perfetto_sql_descriptor
 genrule {
     name: "perfetto_src_trace_processor_perfetto_sql_generator_gen_cc_perfetto_sql_descriptor",
@@ -19119,9 +19102,6 @@ filegroup {
 // GN: //src/trace_processor/perfetto_sql/schema:schema
 filegroup {
     name: "perfetto_src_trace_processor_perfetto_sql_schema_schema",
-    srcs: [
-        "src/trace_processor/perfetto_sql/schema/query_schema.cc",
-    ],
 }
 
 // GN: //src/trace_processor/perfetto_sql/stdlib/android:android
@@ -21513,7 +21493,6 @@ cc_library_static {
         ":perfetto_src_trace_processor_metatrace",
         ":perfetto_src_trace_processor_metrics_metrics",
         ":perfetto_src_trace_processor_perfetto_sql_engine_engine",
-        ":perfetto_src_trace_processor_perfetto_sql_exec_exec",
         ":perfetto_src_trace_processor_perfetto_sql_generator_generator",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
         ":perfetto_src_trace_processor_perfetto_sql_parser_parser",
@@ -24529,8 +24508,6 @@ cc_test {
         ":perfetto_src_trace_processor_metrics_unittests",
         ":perfetto_src_trace_processor_perfetto_sql_engine_engine",
         ":perfetto_src_trace_processor_perfetto_sql_engine_unittests",
-        ":perfetto_src_trace_processor_perfetto_sql_exec_exec",
-        ":perfetto_src_trace_processor_perfetto_sql_exec_unittests",
         ":perfetto_src_trace_processor_perfetto_sql_generator_generator",
         ":perfetto_src_trace_processor_perfetto_sql_generator_unittests",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
@@ -25689,7 +25666,6 @@ cc_library_static {
         ":perfetto_src_trace_processor_metatrace",
         ":perfetto_src_trace_processor_metrics_metrics",
         ":perfetto_src_trace_processor_perfetto_sql_engine_engine",
-        ":perfetto_src_trace_processor_perfetto_sql_exec_exec",
         ":perfetto_src_trace_processor_perfetto_sql_generator_generator",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
         ":perfetto_src_trace_processor_perfetto_sql_parser_parser",
@@ -26407,7 +26383,6 @@ cc_binary_host {
         ":perfetto_src_trace_processor_metatrace",
         ":perfetto_src_trace_processor_metrics_metrics",
         ":perfetto_src_trace_processor_perfetto_sql_engine_engine",
-        ":perfetto_src_trace_processor_perfetto_sql_exec_exec",
         ":perfetto_src_trace_processor_perfetto_sql_generator_generator",
         ":perfetto_src_trace_processor_perfetto_sql_intrinsics_types_types",
         ":perfetto_src_trace_processor_perfetto_sql_parser_parser",

BUILD

@@ -470,7 +470,6 @@ perfetto_cc_library(
         ":src_trace_processor_metatrace",
         ":src_trace_processor_metrics_metrics",
         ":src_trace_processor_perfetto_sql_engine_engine",
-        ":src_trace_processor_perfetto_sql_exec_exec",
         ":src_trace_processor_perfetto_sql_generator_generator",
         ":src_trace_processor_perfetto_sql_intrinsics_types_types",
         ":src_trace_processor_perfetto_sql_parser_parser",
@@ -795,7 +794,6 @@ perfetto_cc_library(
         ":src_trace_processor_metatrace",
         ":src_trace_processor_metrics_metrics",
         ":src_trace_processor_perfetto_sql_engine_engine",
-        ":src_trace_processor_perfetto_sql_exec_exec",
         ":src_trace_processor_perfetto_sql_generator_generator",
         ":src_trace_processor_perfetto_sql_intrinsics_types_types",
         ":src_trace_processor_perfetto_sql_parser_parser",
@@ -3825,15 +3823,6 @@ perfetto_filegroup(
     ],
 )
 
-# GN target: //src/trace_processor/perfetto_sql/exec:exec
-perfetto_filegroup(
-    name = "src_trace_processor_perfetto_sql_exec_exec",
-    srcs = [
-        "src/trace_processor/perfetto_sql/exec/sql_scan.cc",
-        "src/trace_processor/perfetto_sql/exec/sql_scan.h",
-    ],
-)
-
 # GN target: //src/trace_processor/perfetto_sql/generator:generator
 perfetto_filegroup(
     name = "src_trace_processor_perfetto_sql_generator_generator",
@@ -3901,8 +3890,6 @@ perfetto_filegroup(
 perfetto_filegroup(
     name = "src_trace_processor_perfetto_sql_schema_schema",
     srcs = [
-        "src/trace_processor/perfetto_sql/schema/query_schema.cc",
-        "src/trace_processor/perfetto_sql/schema/query_schema.h",
         "src/trace_processor/perfetto_sql/schema/type_mapping.h",
     ],
 )
@@ -11856,7 +11843,6 @@ perfetto_cc_library(
         ":src_trace_processor_metatrace",
         ":src_trace_processor_metrics_metrics",
         ":src_trace_processor_perfetto_sql_engine_engine",
-        ":src_trace_processor_perfetto_sql_exec_exec",
         ":src_trace_processor_perfetto_sql_generator_generator",
         ":src_trace_processor_perfetto_sql_intrinsics_types_types",
         ":src_trace_processor_perfetto_sql_parser_parser",
@@ -12212,7 +12198,6 @@ perfetto_cc_binary(
         ":src_trace_processor_metatrace",
         ":src_trace_processor_metrics_metrics",
         ":src_trace_processor_perfetto_sql_engine_engine",
-        ":src_trace_processor_perfetto_sql_exec_exec",
         ":src_trace_processor_perfetto_sql_generator_generator",
         ":src_trace_processor_perfetto_sql_intrinsics_types_types",
         ":src_trace_processor_perfetto_sql_parser_parser",

src/perfetto_sql/analysis/relation.cc

@@ -51,8 +51,14 @@ struct OwnedView {
   uint32_t root = 0;
 };
 
+// The text SQLite sees for the name at `span`, after macro expansion. A name is
+// one token, so its text is a slice of the one layer it is in, which lives as
+// long as the statement.
 std::string_view Text(SyntaqliteParser* p, SyntaqliteTextSpan span) {
-  return base::TrimWhitespace(SyntaqliteSpanText(p, span));
+  uint32_t len = 0;
+  const char* text = syntaqlite_parser_span_expanded_text(p, &span, &len);
+  return text ? base::TrimWhitespace(std::string_view(text, len))
+              : std::string_view();
 }
 
 const SyntaqliteNode* Node(SyntaqliteParser* p, uint32_t id) {
@@ -186,6 +192,10 @@ class RelationAnalyzer::Impl {
   static ColumnLineage Lookup(const Scope&,
                               std::string_view table,
                               std::string_view column);
+  // The columns of a leaf relation, kept for the rest of the analysis. Its
+  // hidden columns are added to `hidden`.
+  std::vector<ColumnLineage> LeafColumns(LeafRelation,
+                                         std::vector<std::string_view>& hidden);
 
   const Catalog& catalog_;
   // Lineage string_views point into each view's sql string and parse tree, so
@@ -198,6 +208,22 @@ class RelationAnalyzer::Impl {
   bool preserves_rows_ = true;
 };
 
+std::vector<ColumnLineage> RelationAnalyzer::Impl::LeafColumns(
+    LeafRelation found,
+    std::vector<std::string_view>& hidden) {
+  leaves_.push_back(std::make_unique<LeafRelation>(std::move(found)));
+  const LeafRelation* relation = leaves_.back().get();
+  std::vector<ColumnLineage> out;
+  out.reserve(relation->columns.size());
+  for (const LeafColumn& column : relation->columns) {
+    out.push_back({column.name, {{relation->name, column.name, column.type}}});
+    if (column.hidden) {
+      hidden.push_back(column.name);
+    }
+  }
+  return out;
+}
+
 ColumnLineage RelationAnalyzer::Impl::Lookup(const Scope& scope,
                                              std::string_view table,
                                              std::string_view column) {
@@ -450,18 +476,7 @@ base::StatusOr<std::vector<ColumnLineage>> RelationAnalyzer::Impl::Relation(
     int depth,
     std::vector<std::string_view>& hidden) {
   if (std::optional<LeafRelation> found = catalog_.FindLeafRelation(name)) {
-    leaves_.push_back(std::make_unique<LeafRelation>(std::move(*found)));
-    const LeafRelation* relation = leaves_.back().get();
-    std::vector<ColumnLineage> out;
-    out.reserve(relation->columns.size());
-    for (const LeafColumn& column : relation->columns) {
-      out.push_back(
-          {column.name, {{relation->name, column.name, column.type}}});
-      if (column.hidden) {
-        hidden.push_back(column.name);
-      }
-    }
-    return out;
+    return LeafColumns(std::move(*found), hidden);
   }
   if (depth >= kMaxDepth) {
     return base::ErrStatus(

src/perfetto_sql/syntaqlite/BUILD.gn

@@ -37,7 +37,6 @@ source_set("syntaqlite") {
   visibility = [
     "..:intrinsic_macro_expansion",
     "../../trace_processor/perfetto_sql/engine:unittests",
-    "../../trace_processor/perfetto_sql/exec:*",
     "../../trace_processor/perfetto_sql/parser:*",
     "../../trace_processor/perfetto_sql/pipeline:*",
     "../../trace_processor/perfetto_sql/schema:*",

src/trace_processor/BUILD.gn

@@ -473,7 +473,6 @@ perfetto_unittest_source_set("unittests") {
     deps += [
       "../perfetto_sql/analysis:unittests",
       "perfetto_sql/engine:unittests",
-      "perfetto_sql/exec:unittests",
       "perfetto_sql/parser:unittests",
       "perfetto_sql/pipeline:unittests",
       "perfetto_sql/tokenizer:unittests",

src/trace_processor/core/dataframe/adhoc_dataframe_builder.cc

@@ -42,13 +42,31 @@
 #include "src/trace_processor/core/util/flex_vector.h"
 
 namespace perfetto::trace_processor::core::dataframe {
+namespace {
+
+// The number of values collected in `storage`.
+size_t CollectedSize(const Storage& storage) {
+  switch (storage.type().index()) {
+    case StorageType::GetTypeIndex<Int64>():
+      return storage.unchecked_get<Int64>().size();
+    case StorageType::GetTypeIndex<Double>():
+      return storage.unchecked_get<Double>().size();
+    case StorageType::GetTypeIndex<String>():
+      return storage.unchecked_get<String>().size();
+    default:
+      PERFETTO_FATAL("Unexpected storage type");
+  }
+}
+
+}  // namespace
 
 AdhocDataframeBuilder::AdhocDataframeBuilder(std::vector<std::string> names,
                                              StringPool* pool,
                                              const Options& options)
     : string_pool_(pool),
       did_declare_types_(!options.types.empty()),
-      emit_auto_id_(options.emit_auto_id) {
+      emit_auto_id_(options.emit_auto_id),
+      analyze_(options.analyze) {
   PERFETTO_DCHECK(options.types.empty() ||
                   options.types.size() == names.size());
   for (uint32_t i = 0; i < names.size(); ++i) {
@@ -125,6 +143,15 @@ base::StatusOr<Dataframe> AdhocDataframeBuilder::Build() && {
           Unsorted{},
           HasDuplicates{},
       }));
+    } else if (!analyze_) {
+      non_null_row_count = CollectedSize(*state.storage);
+      columns.emplace_back(std::make_shared<Column>(Column{
+          std::move(*state.storage),
+          CreateNullStorageFromBitvector(std::move(state.null_overlay),
+                                         state.nullability_type),
+          Unsorted{},
+          HasDuplicates{},
+      }));
     } else if (state.storage->type().Is<Int64>()) {
       auto& data = state.storage->unchecked_get<Int64>();
       non_null_row_count = data.size();
@@ -227,8 +254,12 @@ base::StatusOr<Dataframe> AdhocDataframeBuilder::Build() && {
         Column{Storage{Storage::Id{static_cast<uint32_t>(row_count)}},
                NullStorage::NonNull{}, IdSorted{}, NoDuplicates{}}));
   }
-  return Dataframe(true, std::move(column_names_), std::move(columns),
-                   static_cast<uint32_t>(row_count), string_pool_);
+  Dataframe dataframe(analyze_, std::move(column_names_), std::move(columns),
+                      static_cast<uint32_t>(row_count), string_pool_);
+  if (!analyze_) {
+    dataframe.FinalizeWithoutStatistics();
+  }
+  return dataframe;
 }
 
 Storage AdhocDataframeBuilder::CreateIntegerStorage(

src/trace_processor/core/dataframe/adhoc_dataframe_builder.h

@@ -117,6 +117,12 @@ struct AdhocDataframeBuilderOptions {
   // resulting dataframe is consumed somewhere that supplies its own primary
   // key (e.g. `StaticTableFunctionModule` adds a HIDDEN `_auto_id`).
   bool emit_auto_id = true;
+
+  // If false, `Build()` keeps columns as they were collected: integers stay
+  // Int64, nothing is marked sorted or free of duplicates, and the dataframe
+  // is finalized without statistics for query planning. Cheaper, for a
+  // dataframe which is only ever scanned.
+  bool analyze = true;
 };
 
 class AdhocDataframeBuilder {
@@ -474,6 +480,7 @@ class AdhocDataframeBuilder {
   std::vector<ColumnState> column_states_;
   bool did_declare_types_ = false;
   bool emit_auto_id_ = true;
+  bool analyze_ = true;
   base::Status current_status_ = base::OkStatus();
   core::BitVector duplicate_bit_vector_;
 };

src/trace_processor/core/dataframe/adhoc_dataframe_builder_unittest.cc

@@ -76,6 +76,29 @@ TEST_F(AdhocDataframeBuilderTest, StringColumnWithNullId) {
           ColumnSpec{Id{}, NonNull{}, IdSorted{}, NoDuplicates{}}));
 }
 
+// Without analysis, columns are kept as collected: integers stay Int64 and
+// nothing is claimed about their order or duplicates.
+TEST_F(AdhocDataframeBuilderTest, WithoutAnalysisColumnsStayAsCollected) {
+  AdhocDataframeBuilder::Options options;
+  options.emit_auto_id = false;
+  options.analyze = false;
+  AdhocDataframeBuilder builder({"id", "value"}, &pool_, options);
+  for (int64_t i = 0; i < 3; ++i) {
+    builder.PushNonNull(0, i);
+    builder.PushNonNull(1, 2.5);
+  }
+  base::StatusOr<Dataframe> df = std::move(builder).Build();
+  ASSERT_OK(df.status());
+  EXPECT_EQ(df->row_count(), 3u);
+  // Finalized all the same, so its columns can be shared.
+  EXPECT_TRUE(df->finalized());
+  EXPECT_THAT(
+      df->CreateSpec().column_specs,
+      ElementsAre(
+          ColumnSpec{Int64{}, NonNull{}, Unsorted{}, HasDuplicates{}},
+          ColumnSpec{Double{}, NonNull{}, Unsorted{}, HasDuplicates{}}));
+}
+
 // Callback for reading cell values in tests.
 struct TestCellCallback : CellCallback {
   void OnCell(int64_t v) {

src/trace_processor/core/dataframe/dataframe.cc

@@ -218,6 +218,14 @@ Dataframe Dataframe::RemoveIndexAt(uint32_t pos) const {
 }
 
 void Dataframe::Finalize() {
+  FinalizeColumns(/*estimate_distinct=*/true);
+}
+
+void Dataframe::FinalizeWithoutStatistics() {
+  FinalizeColumns(/*estimate_distinct=*/false);
+}
+
+void Dataframe::FinalizeColumns(bool estimate_distinct) {
   if (finalized_) {
     return;
   }
@@ -273,7 +281,9 @@ void Dataframe::Finalize() {
       default:
         PERFETTO_FATAL("Invalid nullability type");
     }
-    c->estimated_distinct = EstimateDistinct(distinct_counts, *c);
+    if (estimate_distinct) {
+      c->estimated_distinct = EstimateDistinct(distinct_counts, *c);
+    }
   }
   // Bump the mutation counter so that any cursors with cached pointers
   // know to refresh them: shrink_to_fit() may have reallocated the internal

src/trace_processor/core/dataframe/dataframe.h

@@ -264,6 +264,11 @@ class Dataframe {
   // If the dataframe is already finalized, this function does nothing.
   void Finalize();
 
+  // As Finalize, but without estimating how many distinct values each column
+  // holds, which only query planning uses. Cheaper, for a dataframe which is
+  // only ever scanned.
+  void FinalizeWithoutStatistics();
+
   // Makes a copy of the dataframe which has been finalized. Unfinalized
   // dataframes *cannot* be copied, so this function will assert if not
   // finalized.
@@ -752,6 +757,9 @@ class Dataframe {
   uint32_t non_column_mutations_ = 0;
 
   // Whether the dataframe is "finalized". See `Finalize()`.
+  // Finalize, estimating distinct counts only if `estimate_distinct`.
+  void FinalizeColumns(bool estimate_distinct);
+
   bool finalized_ = false;
 };