opensearch-project · mch2 · Aug 27, 2026 · Aug 28, 2026 · Aug 31, 2026
@@ -27,6 +27,7 @@
 import org.opensearch.core.common.Strings;
 import org.opensearch.index.IndexNotFoundException;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -352,6 +353,15 @@ private static void addLeafFields(
                 Map<String, Object> nested = (Map<String, Object>) fieldProps.get("properties");
                 if (nested != null) {
                     addLeafFields(builder, typeFactory, nested, fieldName);
+                    // Also expose the object itself as a struct (ROW) column, so a query can
+                    // address the whole object (`fields nested_metadata`, `stats … by obj`) and
+                    // not just its leaves. The object has no physical storage — the scan reads
+                    // the leaves — so ObjectStructMaterializer strips this column from the scan
+                    // and re-assembles it with make_struct in a project directly above it.
+                    RelDataType structType = buildObjectType(typeFactory, nested, fieldName);
+                    if (structType != null) {
+                        builder.add(fieldName, structType);
+                    }
                 }
                 continue;
             }
@@ -372,6 +382,54 @@ private static void addLeafFields(
         }
     }
 
+    /**
+     * Builds the struct (ROW) type for an {@code object} mapping: one struct field per supported
+     * sub-field, recursing for sub-objects. Field names are the <em>local</em> names (the struct
+     * nesting already carries the path), while the flat leaf columns added by
+     * {@link #addLeafFields} keep their dotted paths — that dotted convention is how
+     * {@code ObjectStructMaterializer} pairs a struct field back to its backing column.
+     *
+     * <p>Returns {@code null} when the object contributes no supported field, so callers omit the
+     * column entirely rather than declaring an empty struct.
+     */
+    @SuppressWarnings("unchecked")
+    private static RelDataType buildObjectType(RelDataTypeFactory typeFactory, Map<String, Object> properties, String pathPrefix) {
+        List<RelDataType> types = new ArrayList<>();
+        List<String> names = new ArrayList<>();
+        for (Map.Entry<String, Object> fieldEntry : properties.entrySet()) {
+            String localName = fieldEntry.getKey();
+            Map<String, Object> fieldProps = (Map<String, Object>) fieldEntry.getValue();
+            String fieldType = (String) fieldProps.get("type");
+            RelDataType childType;
+            if (fieldType == null || "object".equals(fieldType)) {
+                Map<String, Object> nested = (Map<String, Object>) fieldProps.get("properties");
+                childType = nested == null ? null : buildObjectType(typeFactory, nested, pathPrefix + "." + localName);
+            } else if ("nested".equals(fieldType)) {
+                // Array-of-sub-docs needs LIST<STRUCT> + UNNEST; out of scope here.
+                childType = null;
+            } else {
+                // An ordinary scalar leaf (keyword, long, date_nanos, …). Typed exactly as
+                // addLeafFields types the equivalent flat column — including `scaling_factor`,
+                // the mapping parameter of `scaled_float` (a float persisted as a scaled long),
+                // which buildLeafType needs to type that field. parseScalingFactor yields NaN
+                // when absent, which buildLeafType treats as "not a scaled_float".
+                double scalingFactor = parseScalingFactor(fieldProps.get(SCALING_FACTOR_FIELD));
+                childType = buildLeafType(fieldType, (String) fieldProps.get("format"), scalingFactor, typeFactory);
+            }
+            if (childType == null) {
+                // Unsupported sub-field (geo_point, nested, …) — dropped from the struct for the
+                // same reason it is dropped from the flat columns.
+                continue;
+            }
+            names.add(localName);
+            types.add(childType);
+        }
+        if (names.isEmpty()) {
+            return null;
+        }
+        return typeFactory.createTypeWithNullability(typeFactory.createStructType(types, names), true);
+    }
+
     /**
      * Normalizes a scaling_factor value (Number or String from mapping JSON) to a double.
      * Returns {@link Double#NaN} when the value is null or unparseable, signaling the caller

@@ -144,6 +144,11 @@ public static FieldType fromSqlTypeName(SqlTypeName sqlTypeName) {
             case BINARY, VARBINARY -> FieldType.BINARY;
             case ARRAY -> FieldType.ARRAY;
             case MAP -> FieldType.MAP;
+            // A Calcite ROW is the struct shape an `object` mapping materializes into (see
+            // ObjectStructMaterializer). Without this case, any struct-typed expression fails
+            // capability resolution in OpenSearchProjectRule and surfaces as
+            // UnsupportedFunctionException rather than dispatching to a backend.
+            case ROW -> OBJECT;
             default -> null;
         };
     }

@@ -0,0 +1,95 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.analytics.spi;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.SqlTypeName;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * {@code make_struct('name0', value0, 'name1', value1, ...)} → ROW.
+ *
+ * <p>Materializes an OpenSearch {@code object} field as a struct from the flat dotted leaf
+ * columns the parquet scan produces. The engine stores {@code object} sub-fields as flat
+ * columns ({@code a.b.c}); this function re-assembles them into the nested shape at query
+ * time so a projection or aggregate can address the object as a single value. Nested objects
+ * nest the call:
+ *
+ * <pre>
+ * make_struct('top', $1, 'properties', make_struct('name', $2, 'value', $3))
+ * </pre>
+ *
+ * <p>Executes as DataFusion's {@code named_struct}, which takes the same
+ * (name, value, name, value, …) calling convention. The call does <em>not</em> reach DataFusion
+ * through Substrait's function-signature matching: {@code MakeStructCallConverter} intercepts it
+ * and builds the invocation directly, because isthmus cannot match a variadic function whose
+ * operands are deliberately of differing types. That is what makes the number of struct fields
+ * unbounded.
+ *
+ * <p>The return type is always supplied explicitly by the caller via
+ * {@link #makeCall(RexBuilder, RelDataType, List, List)} — the operand-driven inference is a
+ * placeholder ({@code ANY}) because the authoritative ROW type comes from the index mapping,
+ * not from the operands.
+ *
+ * @opensearch.internal
+ */
+public final class MakeStructFunction {
+
+    /** The function name used in Calcite plans and Substrait serialization. */
+    public static final String NAME = "make_struct";
+
+    /** Singleton Calcite SqlFunction: {@code make_struct(VARCHAR, ANY, ...) → ROW}. */
+    public static final SqlFunction FUNCTION = new SqlFunction(
+        NAME,
+        SqlKind.OTHER_FUNCTION,
+        // Placeholder: callers always construct with an explicit ROW type (see makeCall).
+        opBinding -> opBinding.getTypeFactory().createSqlType(SqlTypeName.ANY),
+        null,
+        OperandTypes.VARIADIC,
+        SqlFunctionCategory.USER_DEFINED_FUNCTION
+    );
+
+    private MakeStructFunction() {}
+
+    /**
+     * Builds {@code make_struct('f0', v0, 'f1', v1, ...)} with an explicit ROW return type.
+     *
+     * @param rexBuilder  builder for the enclosing plan
+     * @param structType  the ROW type this call produces (from the index mapping)
+     * @param fieldNames  struct field names, in order
+     * @param fieldValues struct field value expressions, positionally paired with {@code fieldNames}
+     */
+    public static RexNode makeCall(RexBuilder rexBuilder, RelDataType structType, List<String> fieldNames, List<RexNode> fieldValues) {
+        if (fieldNames.size() != fieldValues.size()) {
+            throw new IllegalArgumentException(
+                "make_struct requires one value per field name; got " + fieldNames.size() + " names and " + fieldValues.size() + " values"
+            );
+        }
+        // Field names are VARCHAR, not CHAR. rexBuilder.makeLiteral(String) would produce a
+        // CHAR(n) literal, which Substrait types as the fixed-width `char<n>`; DataFusion's
+        // named_struct expects a variable-length string for a field name, so VARCHAR (→ Substrait
+        // `string`) is the faithful type. It also keeps the literal free of the padding semantics
+        // CHAR carries.
+        RelDataType nameType = rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR);
+        List<RexNode> operands = new ArrayList<>(fieldNames.size() * 2);
+        for (int i = 0; i < fieldNames.size(); i++) {
+            operands.add(rexBuilder.makeLiteral(fieldNames.get(i), nameType, true));
+            operands.add(fieldValues.get(i));
+        }
+        return rexBuilder.makeCall(structType, FUNCTION, operands);
+    }
+}
@@ -455,7 +455,16 @@ public enum ScalarFunction {
     MINSPAN_BUCKET(Category.SCALAR, SqlKind.OTHER_FUNCTION),
 
     /** PPL range_bucket(value, data_min, data_max, start_param, end_param). VARCHAR label. */
-    RANGE_BUCKET(Category.SCALAR, SqlKind.OTHER_FUNCTION);
+    RANGE_BUCKET(Category.SCALAR, SqlKind.OTHER_FUNCTION),
+
+    // ── Composite (object / struct) construction ─────────────────────
+    /**
+     * {@code make_struct('name0', v0, 'name1', v1, ...)} → ROW. Materializes an OpenSearch
+     * {@code object} field from the flat dotted leaf columns the parquet scan produces
+     * (see {@code ObjectStructMaterializer}). Maps to DataFusion's {@code named_struct}.
+     * Nests for sub-objects. See {@link MakeStructFunction}.
+     */
+    MAKE_STRUCT(Category.SCALAR, SqlKind.OTHER_FUNCTION);
 
     /**
      * Category of scalar function.

@@ -473,6 +473,18 @@ public class DataFusionAnalyticsBackendPlugin implements AnalyticsSearchBackendP
      */
     private static final Set<ScalarFunction> POLYMORPHIC_RETURN_PROJECT_OPS = Set.of(ScalarFunction.CAST, ScalarFunction.SAFE_CAST);
 
+    /**
+     * Project-side scalar functions whose return type is a struct (Calcite {@code ROW} →
+     * {@link FieldType#OBJECT}). Registered separately for the same reason as
+     * {@link #MAP_RETURNING_PROJECT_OPS}: the capability lookup in
+     * {@code OpenSearchProjectRule.resolveScalarViableBackends} keys on the call's return type.
+     *
+     * <p>{@code MAKE_STRUCT} materializes an OpenSearch {@code object} field from the flat dotted
+     * leaf columns the parquet scan produces (see {@code ObjectStructMaterializer}); it executes as
+     * DataFusion's {@code named_struct}, emitted by {@code MakeStructCallConverter}.
+     */
+    private static final Set<ScalarFunction> OBJECT_RETURNING_PROJECT_OPS = Set.of(ScalarFunction.MAKE_STRUCT);
+
     private static final Set<AggregateFunction> AGG_FUNCTIONS = Set.of(
         AggregateFunction.SUM,
         AggregateFunction.SUM0,
@@ -635,6 +647,9 @@ public Set<ProjectCapability> projectCapabilities() {
                 for (ScalarFunction op : MAP_RETURNING_PROJECT_OPS) {
                     caps.add(new ProjectCapability.Scalar(op, Set.of(FieldType.MAP), formats, true));
                 }
+                for (ScalarFunction op : OBJECT_RETURNING_PROJECT_OPS) {
+                    caps.add(new ProjectCapability.Scalar(op, Set.of(FieldType.OBJECT), formats, true));
+                }
                 for (ScalarFunction op : POLYMORPHIC_RETURN_PROJECT_OPS) {
                     for (FieldType ft : FieldType.values()) {
                         caps.add(new ProjectCapability.Scalar(op, Set.of(ft), formats, true));
@@ -647,6 +662,18 @@ public Set<ProjectCapability> projectCapabilities() {
             public Set<AggregateCapability> aggregateCapabilities() {
                 Set<String> formats = Set.copyOf(plugin.getSupportedFormats());
                 Set<AggregateCapability> caps = new HashSet<>();
+                // Aggregates over a struct-typed (object) column. Registered separately because
+                // FieldType.OBJECT is deliberately NOT in SUPPORTED_FIELD_TYPES — most aggregates
+                // are meaningless on a struct (no ordering ⇒ MIN/MAX, no arithmetic ⇒ SUM/AVG),
+                // and adding OBJECT there would also wrongly claim filter/sort support.
+                //
+                // COUNT is the meaningful one (non-null count) and is what the SQL plugin's
+                // Lucene path supports today, i.e. `stats count(<object>)`. Without this the
+                // capability lookup for (COUNT, OBJECT) misses and the query fails at plan time
+                // with "Function [COUNT] is not currently supported as an aggregate function".
+                for (FieldType objectType : Set.of(FieldType.OBJECT)) {
+                    caps.add(new AggregateCapability(AggregateFunction.COUNT, Set.of(objectType), formats));
+                }
                 for (AggregateFunction func : AGG_FUNCTIONS) {
                     for (FieldType type : SUPPORTED_FIELD_TYPES) {
                         // 3-arg constructor leaves decomposition=null so the

@@ -57,6 +57,7 @@
 import java.math.MathContext;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
 
@@ -577,7 +578,16 @@ private byte[] convertToSubstrait(RelNode fragment) {
             throw new IllegalStateException("Substrait conversion rejected the plan: " + e.getMessage(), e);
         }
 
-        List<String> fieldNames = root.fields.stream().map(field -> field.getValue()).toList();
+        // Output column names Substrait attaches to the plan root (Plan.Root.names); DataFusion
+        // names the result schema from them. Flattened so struct columns also contribute their
+        // nested field names — see flattenNamesForSubstrait.
+        //
+        // Each root field carries its own index into the row type, and that index is NOT always
+        // the field's position in this list (RelRoot may project or permute), so the type must be
+        // looked up by field.getKey() rather than by list position — otherwise a name gets paired
+        // with the wrong type and a mispaired struct injects nested names that do not belong,
+        // corrupting the schema.
+        List<String> fieldNames = flattenNamesForSubstrait(root.fields, preprocessed.getRowType());
 
         Plan.Root substraitRoot = Plan.Root.builder().input(substraitRel).names(fieldNames).build();
         Plan plan = Plan.builder().addRoots(substraitRoot).build();
@@ -610,7 +620,73 @@ static Plan rewire(Plan inner, Rel wrapper, List<String> wrapperNames) {
 
     /** Wrapper's output column names from its Calcite row type. */
     private static List<String> fieldNames(RelNode fragment) {
-        return fragment.getRowType().getFieldList().stream().map(RelDataTypeField::getName).toList();
+        return flattenNamesForSubstrait(fragment.getRowType());
+    }
+
+    /**
+     * Flattens column names for a Substrait {@code Plan.Root} / {@code NamedStruct}.
+     *
+     * <p>Substrait carries schema names as ONE depth-first list covering every field at every
+     * nesting level — a struct contributes its own name followed by its children's names,
+     * recursively. A Substrait struct <em>expression</em> is positional (it holds only field
+     * values, no names), so the names live exclusively in this list; that is why it has to be
+     * complete.
+     *
+     * <p>Example — for the output row type
+     * <pre>
+     * id   INTEGER
+     * meta ROW(top VARCHAR, props ROW(name VARCHAR, value VARCHAR))
+     * </pre>
+     * Substrait expects six names, depth-first:
+     * <pre>
+     * ["id", "meta", "top", "props", "name", "value"]
+     *          └────┬────┘   └──────┬──────┘
+     *         meta's children   props' children
+     * </pre>
+     * Emitting only the top-level {@code ["id", "meta"]} makes DataFusion's substrait consumer
+     * reject the plan with {@code "Named schema must contain names for all fields"} — which is
+     * exactly what happened for an {@code object} materialized by
+     * {@code ObjectStructMaterializer} before this flattening was added.
+     *
+     * <p>Names are taken from {@code rootFields} rather than from {@code rowType} because a
+     * {@link RelRoot} may alias its output columns; only the nested names come from the type. Each
+     * root field also carries its own index into {@code rowType}, which is not necessarily its
+     * position in the list ({@code RelRoot} may project or permute), so the type is looked up by
+     * {@link java.util.Map.Entry#getKey()}.
+     */
+    private static List<String> flattenNamesForSubstrait(List<? extends Map.Entry<Integer, String>> rootFields, RelDataType rowType) {
+        List<RelDataTypeField> fields = rowType.getFieldList();
+        List<String> flattened = new ArrayList<>(rootFields.size());
+        for (Map.Entry<Integer, String> rootField : rootFields) {
+            flattened.add(rootField.getValue());
+            int index = rootField.getKey();
+            if (index >= 0 && index < fields.size() && fields.get(index).getType().isStruct()) {
+                appendNestedNames(flattened, fields.get(index).getType());
+            }
+        }
+        return flattened;
+    }
+
+    /** Row-type-driven overload: names come from the row type itself, so positions align by construction. */
+    private static List<String> flattenNamesForSubstrait(RelDataType rowType) {
+        List<String> flattened = new ArrayList<>(rowType.getFieldCount());
+        for (RelDataTypeField field : rowType.getFieldList()) {
+            flattened.add(field.getName());
+            if (field.getType().isStruct()) {
+                appendNestedNames(flattened, field.getType());
+            }
+        }
+        return flattened;
+    }
+
+    /** Appends a struct's field names depth-first (children before the next sibling). */
+    private static void appendNestedNames(List<String> out, RelDataType structType) {
+        for (RelDataTypeField child : structType.getFieldList()) {
+            out.add(child.getName());
+            if (child.getType().isStruct()) {
+                appendNestedNames(out, child.getType());
+            }
+        }
     }
 
     private static Rel replaceInput(Rel wrapper, Rel newInput) {
@@ -798,7 +874,20 @@ protected ImmutableList<FunctionMappings.Sig> getSigs() {
             aggConverter,
             windowConverter,
             typeConverter
-        );
+        ) {
+            @Override
+            public List<io.substrait.isthmus.CallConverter> getCallConverters() {
+                // Offer struct construction to MakeStructCallConverter before function matching.
+                // RexExpressionConverter.visitCall walks this list and only throws "Unable to
+                // convert call ..." once every converter declines, so building the invocation
+                // there sidesteps isthmus' SingularArgumentMatcher — which is what removes the
+                // arity ceiling a declared named_struct signature would impose.
+                List<io.substrait.isthmus.CallConverter> converters = new ArrayList<>();
+                converters.add(new MakeStructCallConverter(extensions, typeConverter));
+                converters.addAll(super.getCallConverters());
+                return converters;
+            }
+        };
         return new SubstraitRelVisitor(converterProvider) {
             @Override
             public Rel visit(org.apache.calcite.rel.core.Aggregate aggregate) {