perf(datasource-active-record): JOIN same-DB to-one relations and select only projected columns

packages/forest_admin_datasource_active_record/lib/forest_admin_datasource_active_record/collection.rb

@@ -26,8 +26,9 @@ def native_driver
 
     def list(_caller, filter, projection)
       query = Utils::Query.new(self, projection, filter)
+      records = query.get
 
-      query.get.map { |record| Utils::ActiveRecordSerializer.new(record).to_hash(projection) }
+      records.map { |record| Utils::ActiveRecordSerializer.new(record, query.joined_relations).to_hash(projection) }
     end
 
     def aggregate(_caller, filter, aggregation, limit = nil)

packages/forest_admin_datasource_active_record/lib/forest_admin_datasource_active_record/utils/active_record_serializer.rb

@@ -1,53 +1,85 @@
 module ForestAdminDatasourceActiveRecord
   module Utils
-    ActiveRecordSerializer = Struct.new(:object) do
+    # Relations in `joined_relations` (see Query#collect_joined_selects) are hydrated from the flat
+    # row's aliased columns; every other relation is read from its preloaded ActiveRecord association.
+    ActiveRecordSerializer = Struct.new(:object, :joined_relations) do
       def to_hash(projection)
         hash_object(object, projection)
       end
 
-      def hash_object(object, projection = nil, with_associations: true)
-        hash = {}
-
+      def hash_object(object, projection = nil, path: [])
         return if object.nil?
 
-        hash.merge! object.attributes
+        # root keeps all its selected columns (attributes + FKs); a related record is restricted to
+        # its projected columns, matching the JOINed hydration
+        hash = path.empty? || projection.nil? ? base_attributes(object) : projected_columns(object, projection)
 
-        serialize_associations(object, projection, hash) if with_associations
+        serialize_associations(object, projection, hash, path) if projection
 
         hash
       end
 
-      def serialize_associations(object, projection, hash)
+      def base_attributes(object)
+        return object.attributes if join_aliases.empty?
+
+        object.attributes.except(*join_aliases)
+      end
+
+      def projected_columns(object, projection)
+        projection.columns.to_h { |column| [column, object[column]] }
+      end
+
+      def serialize_associations(object, projection, hash, path)
         one_associations = %i[has_one belongs_to]
         many_associations = %i[has_many has_and_belongs_to_many]
 
-        # Handle one-to-one and many-to-one associations
-        object.class.reflect_on_all_associations
-              .filter { |a| one_associations.include?(a.macro) && projection.relations.key?(a.name.to_s) }
-              .each do |association|
-                association_name = association.name.to_s
-                hash[association_name] = hash_object(
-                  object.send(association_name),
-                  projection.relations[association_name],
-                  with_associations: projection.relations.key?(association_name)
-                )
-              end
-
-        # Handle one-to-many and many-to-many associations
-        object.class.reflect_on_all_associations
-              .filter { |a| many_associations.include?(a.macro) && projection.relations.key?(a.name.to_s) }
-              .each do |association|
-                association_name = association.name.to_s
-                collection = object.send(association_name)
-                # Serialize the collection as an array
-                hash[association_name] = collection.map do |item|
-                  hash_object(
-                    item,
-                    projection.relations[association_name],
-                    with_associations: projection.relations.key?(association_name)
-                  )
-                end
-              end
+        projection.relations.each_key do |association_name|
+          relation_path = path + [association_name]
+
+          if joined_relation?(relation_path)
+            hash[association_name] = hash_joined_relation(projection.relations[association_name], relation_path)
+            next
+          end
+
+          association = object.class.reflect_on_association(association_name.to_sym)
+          next if association.nil?
+
+          if one_associations.include?(association.macro)
+            hash[association_name] = hash_object(
+              object.send(association_name),
+              projection.relations[association_name],
+              path: relation_path
+            )
+          elsif many_associations.include?(association.macro)
+            hash[association_name] = object.send(association_name).map do |item|
+              hash_object(item, projection.relations[association_name], path: relation_path)
+            end
+          end
+        end
+      end
+
+      # Reads a JOINed relation's columns from the aliases on the root object (not a nested one).
+      def hash_joined_relation(projection, relation_path)
+        meta = joined_relations[relation_path.join('.')]
+        return nil if object[meta[:pk_alias]].nil?
+
+        hash = {}
+        projection.columns.each { |column| hash[column] = object[meta[:columns][column]] }
+        projection.relations.each_key do |nested_name|
+          hash[nested_name] = hash_joined_relation(projection.relations[nested_name], relation_path + [nested_name])
+        end
+
+        hash
+      end
+
+      private
+
+      def joined_relation?(relation_path)
+        !joined_relations.nil? && joined_relations.key?(relation_path.join('.'))
+      end
+
+      def join_aliases
+        @join_aliases ||= (joined_relations || {}).values.flat_map { |meta| meta[:columns].values }.uniq
       end
     end
   end

packages/forest_admin_datasource_active_record/lib/forest_admin_datasource_active_record/utils/query.rb

@@ -1,9 +1,11 @@
+require 'set'
+
 module ForestAdminDatasourceActiveRecord
   module Utils
     class Query
       include ForestAdminDatasourceToolkit::Components::Query::ConditionTree
 
-      attr_reader :query, :select
+      attr_reader :query, :select, :joined_relations
 
       def initialize(collection, projection, filter)
         @collection = collection
@@ -12,6 +14,11 @@ def initialize(collection, projection, filter)
         @filter = filter
         @arel_table = @collection.model.arel_table
         @select = []
+        # relation path (e.g. "bank_account.organizations_view") => { columns: { col => sql_alias }, pk_alias: }
+        @joined_relations = {}
+        @alias_counter = 0
+        # tables already joined by filters/sorts, so apply_select does not join them a second time
+        @filter_joined_tables = Set.new
       end
 
       def build
@@ -204,12 +211,119 @@ def build_select(collection, projection)
       end
 
       def apply_select
+        unless @projection.nil?
+          join_tree, preload_tree = split_relations
+
+          @query = @query.left_outer_joins(join_tree) unless join_tree.empty?
+          @query = @query.includes(preload_tree) unless preload_tree.empty?
+        end
         @query = @query.select(@select.join(', ')) if @select
-        @query = @query.includes(format_relation_projection(@projection)) unless @projection.nil?
 
         @query
       end
 
+      def split_relations
+        join_tree = {}
+        preload_tree = {}
+        used_tables = Set[@collection.model.table_name] | @filter_joined_tables
+
+        @projection.relations.each do |relation_name, sub_projection|
+          tables = joinable_tables(@collection, relation_name, sub_projection, used_tables)
+          if tables
+            used_tables |= tables
+            join_tree[relation_name.to_sym] = format_relation_projection(sub_projection)
+            collect_joined_selects(@collection, relation_name, sub_projection, [relation_name])
+          else
+            preload_tree[relation_name.to_sym] = format_relation_projection(sub_projection)
+          end
+        end
+
+        [join_tree, preload_tree]
+      end
+
+      def collect_joined_selects(collection, relation_name, sub_projection, path)
+        relation_schema = collection.schema[:fields][relation_name]
+        target = local_ar_collection(collection.datasource, relation_schema.foreign_collection)
+        table = target.model.table_name
+        pk_columns = Array(target.model.primary_key) # array for composite primary keys
+
+        alias_map = {}
+        # a pk column is always selected so the serializer can detect a NULL (absent) left-joined relation
+        target.model.connection_pool.with_connection do |connection|
+          (sub_projection.columns + pk_columns).uniq.each do |column|
+            sql_alias = next_join_alias
+            # quote via the adapter so identifiers are valid on every database (e.g. backticks on MySQL)
+            @select << "#{connection.quote_table_name(table)}.#{connection.quote_column_name(column)} " \
+                       "AS #{connection.quote_column_name(sql_alias)}"
+            alias_map[column] = sql_alias
+          end
+        end
+        @joined_relations[path.join('.')] = { columns: alias_map, pk_alias: alias_map[pk_columns.first] }
+
+        sub_projection.relations.each do |nested_name, nested_projection|
+          collect_joined_selects(target, nested_name, nested_projection, path + [nested_name])
+        end
+      end
+
+      def next_join_alias
+        @alias_counter += 1
+        "fa_join_#{@alias_counter}"
+      end
+
+      # Set of tables the subtree adds via JOIN, or nil if any relation in it can't be safely joined.
+      def joinable_tables(collection, relation_name, sub_projection, used_tables)
+        target = joinable_target(collection, relation_name, used_tables)
+        return nil if target.nil?
+
+        tables = Set[target.model.table_name]
+        sub_projection.relations.each do |nested_name, nested_projection|
+          nested = joinable_tables(target, nested_name, nested_projection, used_tables | tables)
+          return nil if nested.nil?
+
+          tables |= nested
+        end
+        tables
+      end
+
+      # The target collection when this hop is safe to collapse into a JOIN, else nil (-> preload).
+      def joinable_target(collection, relation_name, used_tables)
+        relation_schema = collection.schema[:fields][relation_name]
+        # belongs_to only: it joins on the target's primary key, so it can't duplicate the parent
+        # (a has_one child may not be unique)
+        return unless relation_schema.type == 'ManyToOne' && relation_schema.respond_to?(:foreign_collection)
+
+        # a scoped association applies its scope to the JOIN and may inject raw/unqualified SQL or
+        # extra joins (e.g. `belongs_to :x, -> { where('id > ?', 1) }`)
+        reflection = collection.model.reflect_on_association(relation_name.to_sym)
+        return if reflection.nil? || reflection.scope
+
+        target = local_ar_collection(collection.datasource, relation_schema.foreign_collection)
+        return if target.nil? || !target.model.default_scopes.empty? # same risk as a scoped association
+        return unless same_database?(collection.model, target.model)
+        return if used_tables.include?(target.model.table_name) # a table joined twice would be aliased by AR
+
+        target
+      end
+
+      def same_database?(model_a, model_b)
+        # compare the pools, not connection_specification_name (only an owner class name, shared across shards)
+        model_a.connection_pool == model_b.connection_pool
+      rescue StandardError
+        false
+      end
+
+      # The target collection only if it is AR-backed AND belongs to this exact datasource, else nil.
+      # Guards (concrete class + identity, not just the name) against a foreign-datasource collection.
+      def local_ar_collection(datasource, name)
+        collection = datasource.get_collection(name)
+        return nil unless collection.is_a?(ForestAdminDatasourceActiveRecord::Collection)
+        return nil unless collection.datasource.equal?(datasource)
+
+        collection
+      rescue StandardError
+        nil
+      end
+
       def add_join_relation(relation_name)
         @query = @query.left_joins(relation_name.to_sym)
 
@@ -222,6 +336,7 @@ def resolve_field(original_field)
           relation = @collection.schema[:fields][relation_name]
           related_collection = @collection.datasource.get_collection(relation.foreign_collection)
           add_join_relation(relation_name)
+          @filter_joined_tables << related_collection.model.table_name
 
           {
             formatted: "#{related_collection.model.table_name}.#{column_name}",