indexingExecutionEngine = indexingEngine();
+// //Create Writer
+// ParquetWriter writer = (ParquetWriter) indexingExecutionEngine.createWriter();
+// for (int i=0;i<10;i++) {
+// //Get DocumentInput
+// DocumentInput documentInput = writer.newDocumentInput();
+// ParquetDocumentInput parquetDocumentInput = (ParquetDocumentInput) documentInput;
+// //Populate data
+// DummyDataUtils.populateDocumentInput(parquetDocumentInput);
+// //Write document
+// writer.addDoc(parquetDocumentInput);
+// }
+// writer.flush(null);
+// writer.close();
+// //refresh engine
+// indexingExecutionEngine.refresh(null);
+ }
+
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/ArrowExport.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/ArrowExport.java
new file mode 100644
index 0000000000000..1adf01e5989d1
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/ArrowExport.java
@@ -0,0 +1,39 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.bridge;
+
+import org.apache.arrow.c.ArrowArray;
+import org.apache.arrow.c.ArrowSchema;
+
+/**
+ * Container for Arrow C Data Interface exports.
+ * Provides a safe wrapper around ArrowArray and ArrowSchema with proper resource management.
+ */
+public record ArrowExport(ArrowArray arrowArray, ArrowSchema arrowSchema) implements AutoCloseable {
+
+ public long getArrayAddress() {
+ return arrowArray.memoryAddress();
+ }
+
+ public long getSchemaAddress() {
+ return arrowSchema.memoryAddress();
+ }
+
+ @Override
+ public void close() {
+ if (arrowArray != null) {
+ arrowArray.release();
+ arrowArray.close();
+ }
+ if (arrowSchema != null) {
+ arrowSchema.release();
+ arrowSchema.close();
+ }
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/NativeLibraryLoader.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/NativeLibraryLoader.java
new file mode 100644
index 0000000000000..d994f9721ee84
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/NativeLibraryLoader.java
@@ -0,0 +1,98 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.bridge;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.vectorized.execution.jni.NativeLoaderException;
+import org.opensearch.vectorized.execution.jni.PlatformHelper;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.Optional;
+
+/**
+ * Handles loading of the native JNI library.
+ * TODO move to common lib once we switch to passing absolute lib paths
+ */
+public final class NativeLibraryLoader {
+
+ private static volatile boolean loaded = false;
+
+ private static final String DEFAULT_PATH = "native";
+
+ private static final Logger logger = LogManager.getLogger(NativeLibraryLoader.class);
+
+ NativeLibraryLoader() {}
+
+ /**
+ * Load the native library by name.
+ * Supports loading from resources and platform-specific directories.
+ *
+ * @throws UnsatisfiedLinkError if the library cannot be loaded
+ */
+ public static synchronized void load(String libraryName) {
+ if (loaded) return;
+ try {
+ System.loadLibrary(libraryName);
+ loaded = true;
+ return;
+ } catch (UnsatisfiedLinkError ignored) {
+ logger.warn("Failed to load library '" + libraryName + "' from system path");
+ }
+
+ //Look-up with default path
+ try {
+ loadFromResources(DEFAULT_PATH, libraryName);
+ return;
+ } catch (UnsatisfiedLinkError | IOException ignored) {
+ logger.warn("Failed to load library '" + libraryName + "' from default path");
+ }
+
+ // Try platform-specific directory
+ try {
+ String platformDir = PlatformHelper.getPlatformDirectory();
+ String currentDir = Optional.of(System.getProperty("user.dir")).orElse("/");
+ String path = Paths.get(currentDir, "native", platformDir,
+ PlatformHelper.getPlatformLibraryName(libraryName)).toString();
+ loadFromResources(path, libraryName);
+ } catch (UnsatisfiedLinkError | IOException e) {
+ throw new NativeLoaderException(
+ "Failed to load library '" + libraryName + "' from all attempted locations", e);
+ }
+ }
+
+ private static void loadFromResources(String providedPath, String libraryName) throws IOException {
+ String platformDir = PlatformHelper.getPlatformDirectory();
+ String libName = PlatformHelper.getPlatformLibraryName(libraryName);
+ String resourcePath = Paths.get("/", providedPath, platformDir, libName).toString();
+ try (InputStream is = NativeLibraryLoader.class.getResourceAsStream(resourcePath)) {
+ if (is == null) {
+ throw new IOException("Native library not found: " + resourcePath);
+ }
+ Path tempFile = Files.createTempFile(libraryName, PlatformHelper.getNativeExtension());
+ tempFile.toFile().deleteOnExit();
+ Files.copy(is, tempFile, StandardCopyOption.REPLACE_EXISTING);
+ // Register deletion hook on JVM shutdown
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ try {
+ Files.deleteIfExists(tempFile);
+ } catch (IOException ignored) {}
+ }));
+ System.load(tempFile.toAbsolutePath().toString());
+ loaded = true;
+ } catch (IOException e) {
+ throw e;
+ }
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/NativeParquetWriter.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/NativeParquetWriter.java
new file mode 100644
index 0000000000000..1ffa170c0ad98
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/NativeParquetWriter.java
@@ -0,0 +1,62 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.bridge;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Type-safe handle for native Parquet writer with lifecycle management.
+ */
+public class NativeParquetWriter implements Closeable {
+
+ private final String filePath;
+
+ /**
+ * Creates a new native Parquet writer.
+ * @param filePath path to the Parquet file
+ * @param schemaAddress Arrow C Data Interface schema pointer
+ * @throws IOException if writer creation fails
+ */
+ public NativeParquetWriter(String filePath, long schemaAddress) throws IOException {
+ this.filePath = filePath;
+ RustBridge.createWriter(filePath, schemaAddress);
+ }
+
+ /**
+ * Writes a batch to the Parquet file.
+ * @param arrayAddress Arrow C Data Interface array pointer
+ * @param schemaAddress Arrow C Data Interface schema pointer
+ * @throws IOException if write fails
+ */
+ public void write(long arrayAddress, long schemaAddress) throws IOException {
+ RustBridge.write(filePath, arrayAddress, schemaAddress);
+ }
+
+ /**
+ * Flushes buffered data to disk.
+ * @throws IOException if flush fails
+ */
+ public void flush() throws IOException {
+ RustBridge.flushToDisk(filePath);
+ }
+
+ @Override
+ public void close() {
+ try {
+ RustBridge.closeWriter(filePath);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to close Parquet writer for " + filePath, e);
+ }
+ }
+
+ public String getFilePath() {
+ return filePath;
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/RustBridge.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/RustBridge.java
new file mode 100644
index 0000000000000..408ef74ea44f7
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/RustBridge.java
@@ -0,0 +1,42 @@
+package com.parquet.parquetdataformat.bridge;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+
+/**
+ * JNI bridge to the native Rust Parquet writer implementation.
+ *
+ * This class provides the interface between Java and the native Rust library
+ * that handles low-level Parquet file operations. It automatically loads the
+ * appropriate native library for the current platform and architecture.
+ *
+ *
The native library is extracted from resources and loaded as a temporary file,
+ * which is automatically cleaned up on JVM shutdown.
+ *
+ *
All native methods operate on Arrow C Data Interface pointers and return
+ * integer status codes for error handling.
+ */
+public class RustBridge {
+
+ static {
+ NativeLibraryLoader.load("parquet_dataformat_jni");
+
+ initLogger();
+ }
+
+ // Logger initialization method
+ public static native void initLogger();
+
+ // Enhanced native methods that handle validation and provide better error reporting
+ public static native void createWriter(String file, long schemaAddress) throws IOException;
+ public static native void write(String file, long arrayAddress, long schemaAddress) throws IOException;
+ public static native void closeWriter(String file) throws IOException;
+ public static native void flushToDisk(String file) throws IOException;
+
+ public static native long getFilteredNativeBytesUsed(String pathPrefix);
+
+
+ // Native method declarations - these will be implemented in the JNI library
+ public static native void mergeParquetFilesInRust(List inputFiles, String outputFile);
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/RustLoggerBridge.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/RustLoggerBridge.java
new file mode 100644
index 0000000000000..02b2f0953e309
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/bridge/RustLoggerBridge.java
@@ -0,0 +1,34 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.bridge;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RustLoggerBridge {
+
+ private static final Logger logger = LoggerFactory.getLogger(RustLoggerBridge.class);
+
+ // Instance methods for direct Java usage
+ public static void logInfo(String message) {
+ logger.info(message);
+ }
+
+ public static void logWarn(String message) {
+ logger.warn(message);
+ }
+
+ public static void logError(String message) {
+ logger.error(message);
+ }
+
+ public static void logDebug(String message) {
+ logger.debug(message);
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/converter/FieldTypeConverter.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/converter/FieldTypeConverter.java
new file mode 100644
index 0000000000000..b4ace7c4b1953
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/converter/FieldTypeConverter.java
@@ -0,0 +1,135 @@
+package com.parquet.parquetdataformat.converter;
+
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.lucene.search.Query;
+import org.opensearch.index.mapper.MappedFieldType;
+import org.opensearch.index.mapper.TextSearchInfo;
+import org.opensearch.index.mapper.ValueFetcher;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Utility class for converting between OpenSearch field types and Arrow/Parquet types.
+ *
+ * This converter provides bidirectional mapping between OpenSearch's field type system
+ * and Apache Arrow's type system, which serves as the bridge to Parquet data representation.
+ * It handles the complete conversion pipeline from OpenSearch indexed data to columnar
+ * Parquet storage format.
+ *
+ *
Supported type conversions:
+ *
+ * - OpenSearch numeric types (long, integer, short, byte, double, float) → Arrow Int/FloatingPoint
+ * - OpenSearch boolean → Arrow Bool
+ * - OpenSearch date → Arrow Timestamp
+ * - OpenSearch text/keyword → Arrow Utf8
+ *
+ *
+ * The converter also provides reverse mapping capabilities to reconstruct OpenSearch
+ * field types from Arrow types, enabling proper schema reconstruction during read operations.
+ *
+ *
All conversion methods are static and thread-safe, making them suitable for concurrent
+ * use across multiple writer instances.
+ */
+public class FieldTypeConverter {
+
+ public static Map convertToArrowFieldMap(MappedFieldType mappedFieldType, Object value) {
+ Map fieldMap = new HashMap<>();
+ FieldType arrowFieldType = convertToArrowFieldType(mappedFieldType);
+ fieldMap.put(arrowFieldType, value);
+ return fieldMap;
+ }
+
+ public static FieldType convertToArrowFieldType(MappedFieldType mappedFieldType) {
+ ArrowType arrowType = getArrowType(mappedFieldType.typeName());
+ return new FieldType(true, arrowType, null);
+ }
+
+ public static ParquetFieldType convertToParquetFieldType(MappedFieldType mappedFieldType) {
+ ArrowType arrowType = getArrowType(mappedFieldType.typeName());
+ return new ParquetFieldType(mappedFieldType.name(), arrowType);
+ }
+
+ public static MappedFieldType convertToMappedFieldType(String name, ArrowType arrowType) {
+ String opensearchType = getOpenSearchType(arrowType);
+ return new MockMappedFieldType(name, opensearchType);
+ }
+
+ private static ArrowType getArrowType(String opensearchType) {
+ switch (opensearchType) {
+ case "long":
+ return new ArrowType.Int(64, true);
+ case "integer":
+ return new ArrowType.Int(32, true);
+ case "short":
+ return new ArrowType.Int(16, true);
+ case "byte":
+ return new ArrowType.Int(8, true);
+ case "double":
+ return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
+ case "float":
+ return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE);
+ case "boolean":
+ return new ArrowType.Bool();
+ case "date":
+ return new ArrowType.Timestamp(TimeUnit.MILLISECOND, null);
+ default:
+ return new ArrowType.Utf8();
+ }
+ }
+
+ private static String getOpenSearchType(ArrowType arrowType) {
+ switch (arrowType) {
+ case ArrowType.Int intType -> {
+ return switch (intType.getBitWidth()) {
+ case 8 -> "byte";
+ case 16 -> "short";
+ case 32 -> "integer";
+ case 64 -> "long";
+ default -> "integer";
+ };
+ }
+ case ArrowType.FloatingPoint fpType -> {
+ return fpType.getPrecision() == FloatingPointPrecision.DOUBLE ? "double" : "float";
+ }
+ case ArrowType.Bool bool -> {
+ return "boolean";
+ }
+ case ArrowType.Timestamp timestamp -> {
+ return "date";
+ }
+ case null, default -> {
+ return "text";
+ }
+ }
+ }
+
+ private static class MockMappedFieldType extends MappedFieldType {
+ private final String type;
+
+ public MockMappedFieldType(String name, String type) {
+ super(name, true, false, false, TextSearchInfo.NONE, null);
+ this.type = type;
+ }
+
+ @Override
+ public String typeName() {
+ return type;
+ }
+
+ @Override
+ public ValueFetcher valueFetcher(org.opensearch.index.query.QueryShardContext context,
+ org.opensearch.search.lookup.SearchLookup searchLookup,
+ String format) {
+ return null;
+ }
+
+ @Override
+ public Query termQuery(Object value, org.opensearch.index.query.QueryShardContext context) {
+ return null;
+ }
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/converter/ParquetFieldType.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/converter/ParquetFieldType.java
new file mode 100644
index 0000000000000..84f1b9a4bedd2
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/converter/ParquetFieldType.java
@@ -0,0 +1,48 @@
+package com.parquet.parquetdataformat.converter;
+
+import org.apache.arrow.vector.types.pojo.ArrowType;
+
+/**
+ * Represents a field type for Parquet-based document fields.
+ *
+ * This class encapsulates the field name and Arrow type information
+ * required for proper type mapping between OpenSearch fields and Parquet
+ * column definitions. It serves as the intermediate representation used
+ * throughout the Parquet processing pipeline.
+ *
+ *
The Arrow type system provides a rich set of data types that can
+ * accurately represent various field types from OpenSearch, ensuring
+ * proper data serialization and deserialization.
+ *
+ *
Key features:
+ *
+ * - Field name preservation for schema mapping
+ * - Arrow type integration for precise data representation
+ * - Simple mutable structure for field definition building
+ *
+ */
+public class ParquetFieldType {
+ private String name;
+ private ArrowType type;
+
+ public ParquetFieldType(String name, ArrowType type) {
+ this.name = name;
+ this.type = type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public ArrowType getType() {
+ return type;
+ }
+
+ public void setType(ArrowType type) {
+ this.type = type;
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/DummyDataUtils.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/DummyDataUtils.java
new file mode 100644
index 0000000000000..0d6c2519d463a
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/DummyDataUtils.java
@@ -0,0 +1,60 @@
+package com.parquet.parquetdataformat.engine;
+
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.opensearch.common.SuppressForbidden;
+import org.opensearch.index.engine.exec.DocumentInput;
+import org.opensearch.index.mapper.MappedFieldType;
+import com.parquet.parquetdataformat.converter.FieldTypeConverter;
+
+import java.util.Arrays;
+import java.util.Random;
+
+@SuppressForbidden(reason = "Need random for creating temp files")
+public class DummyDataUtils {
+ public static Schema getSchema() {
+ // Create the most minimal schema possible - just one string field
+ return new Schema(Arrays.asList(
+ Field.notNullable(ID, new ArrowType.Int(32, true)),
+ Field.nullable(NAME, new ArrowType.Utf8()),
+ Field.nullable(DESIGNATION, new ArrowType.Utf8()),
+ Field.nullable(SALARY, new ArrowType.Int(32, true))
+ ));
+ }
+
+ public static void populateDocumentInput(DocumentInput documentInput) {
+ MappedFieldType idField = FieldTypeConverter.convertToMappedFieldType(ID, new ArrowType.Int(32, true));
+ documentInput.addField(idField, generateRandomId());
+ MappedFieldType nameField = FieldTypeConverter.convertToMappedFieldType(NAME, new ArrowType.Utf8());
+ documentInput.addField(nameField, generateRandomName());
+ MappedFieldType designationField = FieldTypeConverter.convertToMappedFieldType(DESIGNATION, new ArrowType.Utf8());
+ documentInput.addField(designationField, generateRandomDesignation());
+ MappedFieldType salaryField = FieldTypeConverter.convertToMappedFieldType(SALARY, new ArrowType.Int(32, true));
+ documentInput.addField(salaryField, random.nextInt(100000));
+ }
+
+ private static final String ID = "id";
+ private static final String NAME = "name";
+ private static final String DESIGNATION = "designation";
+ private static final String SALARY = "salary";
+ private static final String INCREMENT = "increment";
+ private static final Random random = new Random();
+ private static final String[] NAMES = {"John Doe", "Jane Smith", "Alice Johnson", "Bob Wilson", "Carol Brown"};
+ private static final String[] DESIGNATIONS = {"Software Engineer", "Senior Developer", "Team Lead", "Manager", "Architect"};
+
+ private static int generateRandomId() {
+ return random.nextInt(1000000);
+ }
+
+ private static String generateRandomName() {
+ return NAMES[random.nextInt(NAMES.length)];
+ }
+
+ private static String generateRandomDesignation() {
+ return DESIGNATIONS[random.nextInt(DESIGNATIONS.length)];
+ }
+
+
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/ParquetDataFormat.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/ParquetDataFormat.java
new file mode 100644
index 0000000000000..240a33c10531e
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/ParquetDataFormat.java
@@ -0,0 +1,58 @@
+package com.parquet.parquetdataformat.engine;
+
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.engine.exec.DataFormat;
+
+/**
+ * Data format implementation for Parquet-based document storage.
+ *
+ * This class integrates with OpenSearch's DataFormat interface to provide
+ * Parquet file format support within the OpenSearch indexing pipeline. It
+ * defines the configuration and behavior for the "parquet" data format.
+ *
+ *
The implementation provides hooks for:
+ *
+ * - Data format specific settings configuration
+ * - Cluster-level settings management
+ * - Store configuration for Parquet-specific optimizations
+ * - Format identification through the "parquet" name
+ *
+ *
+ * This class serves as the entry point for registering Parquet format
+ * capabilities with OpenSearch's execution engine framework, allowing
+ * the system to recognize and utilize Parquet-based storage operations.
+ */
+public class ParquetDataFormat implements DataFormat {
+ @Override
+ public Setting dataFormatSettings() {
+ return null;
+ }
+
+ @Override
+ public Setting clusterLeveldataFormatSettings() {
+ return null;
+ }
+
+ @Override
+ public String name() {
+ return "parquet";
+ }
+
+ @Override
+ public void configureStore() {
+
+ }
+
+ public static ParquetDataFormat PARQUET_DATA_FORMAT = new ParquetDataFormat();
+
+ @Override
+ public boolean equals(Object obj) {
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/ParquetExecutionEngine.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/ParquetExecutionEngine.java
new file mode 100644
index 0000000000000..705ddebaec64c
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/ParquetExecutionEngine.java
@@ -0,0 +1,166 @@
+package com.parquet.parquetdataformat.engine;
+
+import com.parquet.parquetdataformat.bridge.RustBridge;
+import com.parquet.parquetdataformat.memory.ArrowBufferPool;
+import com.parquet.parquetdataformat.merge.CompactionStrategy;
+import com.parquet.parquetdataformat.merge.ParquetMergeExecutor;
+import com.parquet.parquetdataformat.merge.ParquetMerger;
+import com.parquet.parquetdataformat.writer.ParquetDocumentInput;
+import com.parquet.parquetdataformat.writer.ParquetWriter;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.engine.exec.DataFormat;
+import org.opensearch.index.engine.exec.IndexingExecutionEngine;
+import org.opensearch.index.engine.exec.Merger;
+import org.opensearch.index.engine.exec.RefreshInput;
+import org.opensearch.index.engine.exec.RefreshResult;
+import org.opensearch.index.engine.exec.Writer;
+import org.opensearch.index.engine.exec.WriterFileSet;
+import org.opensearch.index.shard.ShardPath;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.StreamSupport;
+
+import static com.parquet.parquetdataformat.engine.ParquetDataFormat.PARQUET_DATA_FORMAT;
+
+/**
+ * Main execution engine for Parquet-based indexing operations in OpenSearch.
+ *
+ * This engine implements OpenSearch's IndexingExecutionEngine interface to provide
+ * Parquet file generation capabilities within the indexing pipeline. It manages the
+ * lifecycle of Parquet writers and coordinates the overall document processing workflow.
+ *
+ *
Key responsibilities:
+ *
+ * - Writer creation with unique file naming and Arrow schema integration
+ * - Schema-based field type support and validation
+ * - Refresh operations for completing indexing cycles
+ * - Integration with the broader Parquet data format ecosystem
+ *
+ *
+ * The engine uses an atomic counter to ensure unique Parquet file names across
+ * concurrent operations, following the naming pattern "parquet_file_generation_N.parquet"
+ * where N is an incrementing sequence number.
+ *
+ *
Each writer instance created by this engine is configured with:
+ *
+ * - A unique file name for output isolation
+ * - The Arrow schema provided during engine construction
+ * - Full access to the Parquet processing pipeline via {@link ParquetWriter}
+ *
+ *
+ * The engine is designed to work with {@link ParquetDocumentInput} for document
+ * processing and integrates seamlessly with OpenSearch's execution framework.
+ */
+public class ParquetExecutionEngine implements IndexingExecutionEngine {
+
+ private static final Logger logger = LogManager.getLogger(ParquetExecutionEngine.class);
+
+ public static final String FILE_NAME_PREFIX = "_parquet_file_generation";
+ private static final Pattern FILE_PATTERN = Pattern.compile(".*_(\\d+)\\.parquet$", Pattern.CASE_INSENSITIVE);
+ public static final String FILE_NAME_EXT = ".parquet";
+
+ private final Supplier schema;
+ private final List filesWrittenAlready = new ArrayList<>();
+ private final ShardPath shardPath;
+ private final ParquetMerger parquetMerger = new ParquetMergeExecutor(CompactionStrategy.RECORD_BATCH);
+ private final ArrowBufferPool arrowBufferPool;
+
+ public ParquetExecutionEngine(Settings settings, Supplier schema, ShardPath shardPath) {
+ this.schema = schema;
+ this.shardPath = shardPath;
+ this.arrowBufferPool = new ArrowBufferPool(settings);
+ }
+
+ @Override
+ public void loadWriterFiles() throws IOException {
+ try (DirectoryStream stream = Files.newDirectoryStream(shardPath.getDataPath(), "*" + FILE_NAME_EXT)) {
+ StreamSupport.stream(stream.spliterator(), false)
+ .map(Path::getFileName)
+ .map(Path::toString)
+ .map(FILE_PATTERN::matcher)
+ .filter(Matcher::matches)
+ .map(m -> WriterFileSet.builder()
+ .directory(shardPath.getDataPath())
+ .writerGeneration(Long.parseLong(m.group(1)))
+ .addFile(m.group(0))
+ .build())
+ .forEach(filesWrittenAlready::add);
+ }
+ }
+
+ @Override
+ public void deleteFiles(Map> filesToDelete) throws IOException {
+ if (filesToDelete.get(PARQUET_DATA_FORMAT.name()) != null) {
+ Collection parquetFilesToDelete = filesToDelete.get(PARQUET_DATA_FORMAT.name());
+ for (String fileName : parquetFilesToDelete) {
+ Path filePath = Paths.get(fileName);
+ logger.info("Deleting file [ParquetExecutionEngine]: {}", filePath);
+ try {
+ Files.delete(filePath);
+ } catch (Exception e) {
+ logger.error("Failed to delete file [ParquetExecutionEngine]: {}", filePath, e);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+ @Override
+ public List supportedFieldTypes() {
+ return List.of();
+ }
+
+ @Override
+ public Writer createWriter(long writerGeneration) throws IOException {
+ String fileName = Path.of(shardPath.getDataPath().toString(), FILE_NAME_PREFIX + "_" + writerGeneration + FILE_NAME_EXT).toString();
+ return new ParquetWriter(fileName, schema.get(), writerGeneration, arrowBufferPool);
+ }
+
+ @Override
+ public Merger getMerger() {
+ return parquetMerger;
+ }
+
+ @Override
+ public RefreshResult refresh(RefreshInput refreshInput) throws IOException {
+ RefreshResult refreshResult = new RefreshResult();
+ // NO-OP, as refresh is being handled at CompositeIndexingExecution Engin
+ return refreshResult;
+ }
+
+ @Override
+ public DataFormat getDataFormat() {
+ return new ParquetDataFormat();
+ }
+
+ @Override
+ public long getNativeBytesUsed() {
+ long vsrMemory = arrowBufferPool.getTotalAllocatedBytes();
+ String shardDataPath = shardPath.getDataPath().toString();
+ long filteredArrowWriterMemory = RustBridge.getFilteredNativeBytesUsed(shardDataPath);
+ logger.debug("Native memory used by VSR Buffer Pool: {}", vsrMemory);
+ logger.debug("Native memory used by ArrowWriters in shard path {}: {}", shardDataPath, filteredArrowWriterMemory);
+ return vsrMemory + filteredArrowWriterMemory;
+ }
+
+ @Override
+ public void close() throws IOException {
+ arrowBufferPool.close();
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/read/ParquetDataSourceCodec.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/read/ParquetDataSourceCodec.java
new file mode 100644
index 0000000000000..5e60e949cd527
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/read/ParquetDataSourceCodec.java
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.engine.read;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.vectorized.execution.search.DataFormat;
+import org.opensearch.vectorized.execution.search.spi.DataSourceCodec;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Datasource codec implementation for parquet files
+ */
+public class ParquetDataSourceCodec implements DataSourceCodec {
+
+ private static final Logger logger = LogManager.getLogger(ParquetDataSourceCodec.class);
+
+ // JNI library loading
+ static {
+ try {
+ //JniLibraryLoader.loadLibrary();
+ logger.info("DataFusion JNI library loaded successfully");
+ } catch (Exception e) {
+ logger.error("Failed to load DataFusion JNI library", e);
+ throw new RuntimeException("Failed to initialize DataFusion JNI library", e);
+ }
+ }
+
+ public DataFormat getDataFormat() {
+ return DataFormat.CSV;
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/read/package-info.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/read/package-info.java
new file mode 100644
index 0000000000000..bd486fa1e26f4
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/read/package-info.java
@@ -0,0 +1,13 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * CSV data format implementation for DataFusion integration.
+ * Provides CSV file reading capabilities through DataFusion query engine.
+ */
+package com.parquet.parquetdataformat.engine.read;
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/ArrowFieldRegistry.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/ArrowFieldRegistry.java
new file mode 100644
index 0000000000000..1a65f7a116623
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/ArrowFieldRegistry.java
@@ -0,0 +1,163 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields;
+
+import com.parquet.parquetdataformat.fields.core.data.number.LongParquetField;
+import com.parquet.parquetdataformat.plugins.fields.CoreDataFieldPlugin;
+import com.parquet.parquetdataformat.plugins.fields.MetadataFieldPlugin;
+import com.parquet.parquetdataformat.plugins.fields.ParquetFieldPlugin;
+import org.opensearch.index.mapper.SeqNoFieldMapper;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Registry for mapping OpenSearch field types to their corresponding Parquet field implementations.
+ * This class maintains a centralized mapping between OpenSearch field type names and their
+ * Arrow/Parquet field representations, enabling efficient field type resolution during
+ * schema creation and data processing.
+ *
+ * The registry is initialized once during class loading and provides thread-safe
+ * read-only access to field mappings.
+ */
+public final class ArrowFieldRegistry {
+
+ /**
+ * All registered field mappings (thread-safe, mutable)
+ */
+ private static final Map FIELD_REGISTRY = new ConcurrentHashMap<>();
+
+ // Static initialization block to populate the field registry
+ static {
+ initialize();
+ }
+
+ // Private constructor to prevent instantiation of utility class
+ private ArrowFieldRegistry() {
+ throw new UnsupportedOperationException("Registry class should not be instantiated");
+ }
+
+ /**
+ * Initialize the registry with all available plugins.
+ * This method should be called during node startup after all plugins are loaded.
+ */
+ public static synchronized void initialize() {
+ // Always register core plugins first
+ registerCorePlugins();
+ }
+
+ /**
+ * Register core OpenSearch field plugins.
+ * These are always available and provide the foundation field type support.
+ */
+ private static void registerCorePlugins() {
+ // Register core data fields
+ registerPlugin(new CoreDataFieldPlugin(), "CoreDataFields");
+
+ // REgister metadata fields
+ registerPlugin(new MetadataFieldPlugin(), "MetadataFields");
+ }
+ /**
+ * Register a single plugin's field types.
+ */
+ private static void registerPlugin(ParquetFieldPlugin plugin, String pluginName) {
+ Map fields = plugin.getParquetFields();
+
+ if (fields != null && !fields.isEmpty()) {
+ for (Map.Entry entry : fields.entrySet()) {
+ String fieldType = entry.getKey();
+ ParquetField parquetField = entry.getValue();
+
+ // Validate registration
+ validateFieldRegistration(fieldType, parquetField, pluginName);
+
+ // Check for conflicts
+ if (FIELD_REGISTRY.containsKey(fieldType)) {
+ throw new IllegalArgumentException(
+ String.format("Field type [%s] is already registered. Plugin [%s] cannot override it.",
+ fieldType, pluginName)
+ );
+ }
+
+ FIELD_REGISTRY.put(fieldType, parquetField);
+ }
+
+ FIELD_REGISTRY.put(SeqNoFieldMapper.PRIMARY_TERM_NAME, new LongParquetField());
+ }
+ }
+
+ private static void validateFieldRegistration(String fieldType, ParquetField parquetField, String source) {
+ if (fieldType == null || fieldType.trim().isEmpty()) {
+ throw new IllegalArgumentException("Field type name cannot be null or empty");
+ }
+
+ if (parquetField == null) {
+ throw new IllegalArgumentException("ParquetField implementation cannot be null");
+ }
+
+ // Validate that the ParquetField can provide required Arrow types
+ try {
+ parquetField.getArrowType();
+ parquetField.getFieldType();
+ } catch (Exception e) {
+ throw new IllegalArgumentException(
+ String.format("Invalid ParquetField implementation for type [%s] from source [%s]: %s",
+ fieldType, source, e.getMessage()), e
+ );
+ }
+ }
+
+ /**
+ * Get registry statistics for monitoring and debugging.
+ */
+ public static RegistryStats getStats() {
+ Set allTypes = getRegisteredFieldNames();
+
+ return new RegistryStats(
+ FIELD_REGISTRY.size(), // Single source of truth
+ allTypes
+ );
+ }
+
+ /**
+ * Get all registered field type names.
+ */
+ public static Set getRegisteredFieldNames() {
+ return Collections.unmodifiableSet(FIELD_REGISTRY.keySet());
+ }
+
+ /**
+ * Returns the ParquetField implementation for the specified OpenSearch field type, or null if not found.
+ */
+ public static ParquetField getParquetField(String fieldType) {
+ return FIELD_REGISTRY.get(fieldType);
+ }
+
+ public static class RegistryStats {
+ private final int totalFields;
+ private final Set allFieldTypes;
+
+ public RegistryStats(int totalFields, Set allFieldTypes) {
+ this.totalFields = totalFields;
+ this.allFieldTypes = allFieldTypes;
+ }
+
+ // Getters
+ public int getTotalFields() { return totalFields; }
+ public Set getAllFieldTypes() { return allFieldTypes; }
+
+ @Override
+ public String toString() {
+ return String.format("RegistryStats{total=%d, }", totalFields);
+ }
+ }
+
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/ArrowSchemaBuilder.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/ArrowSchemaBuilder.java
new file mode 100644
index 0000000000000..5430b7fa03101
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/ArrowSchemaBuilder.java
@@ -0,0 +1,119 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields;
+
+import com.parquet.parquetdataformat.fields.core.data.number.LongParquetField;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.opensearch.index.engine.exec.composite.CompositeDataFormatWriter;
+import org.opensearch.index.mapper.FieldNamesFieldMapper;
+import org.opensearch.index.mapper.IndexFieldMapper;
+import org.opensearch.index.mapper.Mapper;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.MetadataFieldMapper;
+import org.opensearch.index.mapper.NestedPathFieldMapper;
+import org.opensearch.index.mapper.SeqNoFieldMapper;
+import org.opensearch.index.mapper.SourceFieldMapper;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Utility class for creating Apache Arrow schemas from OpenSearch mapper services.
+ * This class provides methods to convert OpenSearch field mappings into Arrow schema definitions
+ * that can be used for Parquet data format operations.
+ */
+public final class ArrowSchemaBuilder {
+
+ // Private constructor to prevent instantiation of utility class
+ private ArrowSchemaBuilder() {
+ throw new UnsupportedOperationException("Utility class should not be instantiated");
+ }
+
+ /**
+ * Creates an Apache Arrow Schema from the provided MapperService.
+ * This method extracts all non-metadata field mappers and converts them to Arrow fields.
+ *
+ * @param mapperService the OpenSearch mapper service containing field definitions
+ * @return a new Schema containing Arrow field definitions for all mapped fields
+ * @throws IllegalArgumentException if mapperService is null
+ * @throws IllegalStateException if no valid fields are found or if a field type is not supported
+ */
+ public static Schema getSchema(final MapperService mapperService) {
+ Objects.requireNonNull(mapperService, "MapperService cannot be null");
+
+ final List fields = extractFieldsFromMappers(mapperService);
+
+ if (fields.isEmpty()) {
+ throw new IllegalStateException("No valid fields found in mapper service");
+ }
+
+ return new Schema(fields);
+ }
+
+ /**
+ * Extracts Arrow fields from the mapper service, filtering out metadata fields.
+ *
+ * @param mapperService the mapper service to extract fields from
+ * @return a list of Arrow fields
+ */
+ private static List extractFieldsFromMappers(final MapperService mapperService) {
+ final List fields = new ArrayList<>();
+
+ for (final Mapper mapper : mapperService.documentMapper().mappers()) {
+ if (notSupportedMetadataField(mapper)) {
+ continue;
+ }
+
+ final Field arrowField = createArrowField(mapper);
+ fields.add(arrowField);
+ }
+
+ fields.add(new Field(CompositeDataFormatWriter.ROW_ID, new LongParquetField().getFieldType(), null));
+ fields.add(new Field(SeqNoFieldMapper.PRIMARY_TERM_NAME, new LongParquetField().getFieldType(), null));
+
+ return fields;
+ }
+
+ /**
+ * Checks if the given mapper represents a not supported metadata field.
+ *
+ * @param mapper the mapper to check
+ * @return true if the mapper is a not supported metadata field, false otherwise
+ */
+ private static boolean notSupportedMetadataField(final Mapper mapper) {
+ return mapper instanceof SourceFieldMapper
+ || mapper instanceof FieldNamesFieldMapper
+ || mapper instanceof IndexFieldMapper
+ || mapper instanceof NestedPathFieldMapper
+ || Objects.equals(mapper.typeName(), "_feature")
+ || Objects.equals(mapper.typeName(), "_data_stream_timestamp");
+ }
+
+ /**
+ * Creates an Arrow Field from an OpenSearch Mapper.
+ *
+ * @param mapper the mapper to convert
+ * @return a new Arrow Field
+ * @throws IllegalStateException if the mapper type is not supported
+ */
+ private static Field createArrowField(final Mapper mapper) {
+ final ParquetField parquetField = ArrowFieldRegistry.getParquetField(mapper.typeName());
+
+ if (parquetField == null) {
+ throw new IllegalStateException(
+ String.format("Unsupported field type '%s' for field '%s'",
+ mapper.typeName(), mapper.name())
+ );
+ }
+
+ return new Field(mapper.name(), parquetField.getFieldType(), null);
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/ParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/ParquetField.java
new file mode 100644
index 0000000000000..dc1a7e369d430
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/ParquetField.java
@@ -0,0 +1,124 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields;
+
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+import java.util.Objects;
+
+/**
+ * Abstract base class for all Parquet field implementations that handle the conversion
+ * between OpenSearch field types and Apache Arrow/Parquet data structures.
+ *
+ * This class defines the contract for field-specific operations including:
+ *
+ * - Adding field data to vector groups for columnar storage
+ * - Creating field instances with proper type validation
+ * - Providing Arrow type definitions for schema generation
+ * - Generating field type metadata for Arrow schemas
+ *
+ *
+ * Implementations of this class should be thread-safe and stateless, as they
+ * may be shared across multiple processing contexts.
+ *
+ * @see ArrowFieldRegistry
+ * @see ManagedVSR
+ */
+public abstract class ParquetField {
+
+ /**
+ * Adds the parsed field value to the appropriate vector group within the managed VSR.
+ * This method is responsible for the actual data conversion and storage in the
+ * columnar format specific to each field type.
+ *
+ * Implementations must handle null values appropriately and ensure type safety
+ * when casting the parseValue to the expected type.
+ *
+ * @param mappedFieldType the OpenSearch field type metadata containing field configuration
+ * @param managedVSR the managed vector schema root for columnar data storage
+ * @param parseValue the parsed field value to be stored, may be null
+ * @throws IllegalArgumentException if any parameter is invalid for this field type
+ * @throws ClassCastException if parseValue cannot be cast to the expected type
+ */
+ protected abstract void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue);
+
+ /**
+ * Creates and processes a field entry if the field type supports columnar storage.
+ * This method serves as the main entry point for field processing and includes
+ * validation logic to ensure only columnar fields are processed.
+ *
+ * The method performs the following operations:
+ *
+ * - Validates input parameters
+ * - Checks if the field supports columnar storage
+ * - Delegates to {@link #addToGroup} for actual data processing
+ *
+ *
+ * @param mappedFieldType the OpenSearch field type metadata, must not be null
+ * @param managedVSR the managed vector schema root, must not be null
+ * @param parseValue the parsed field value to be processed, may be null
+ * @throws IllegalArgumentException if mappedFieldType or managedVSR is null
+ */
+ public final void createField(final MappedFieldType mappedFieldType,
+ final ManagedVSR managedVSR,
+ final Object parseValue) {
+ Objects.requireNonNull(mappedFieldType, "MappedFieldType cannot be null");
+ Objects.requireNonNull(managedVSR, "ManagedVSR cannot be null");
+
+ if (mappedFieldType.isColumnar()) {
+ // TODO: support dynamic mapping update
+ // for now ignore the field
+ if (managedVSR.getVector(mappedFieldType.name()) != null) {
+ addToGroup(mappedFieldType, managedVSR, parseValue);
+ }
+ }
+ }
+
+ /**
+ * Returns the Apache Arrow type definition for this field.
+ * This type definition is used for schema generation and data type validation
+ * in the Arrow/Parquet ecosystem.
+ *
+ * The returned ArrowType should be consistent across all instances of the
+ * same field implementation and should accurately represent the data type
+ * that will be stored.
+ *
+ * @return the Arrow type definition for this field, never null
+ */
+ public abstract ArrowType getArrowType();
+
+ /**
+ * Returns the Apache Arrow field type with metadata for schema generation.
+ * This includes the base Arrow type along with additional metadata such as
+ * nullability constraints and custom properties.
+ *
+ * The returned FieldType is used when constructing Arrow schemas and
+ * should include appropriate nullability settings based on the field's
+ * characteristics.
+ *
+ * @return the complete field type definition including metadata, never null
+ */
+ public abstract FieldType getFieldType();
+
+ /**
+ * Provides a string representation of this ParquetField for debugging purposes.
+ * The default implementation includes the class name and Arrow type information.
+ *
+ * @return a string representation of this field
+ */
+ @Override
+ public String toString() {
+ return String.format("%s{arrowType=%s}",
+ this.getClass().getSimpleName(),
+ getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/BinaryParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/BinaryParquetField.java
new file mode 100644
index 0000000000000..eaa4d5209bfc2
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/BinaryParquetField.java
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling binary data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch binary fields and Apache Arrow
+ * variable-length binary vectors for columnar storage in Parquet format. Binary values are stored using
+ * Apache Arrow's {@link VarBinaryVector}, which handles variable-length byte arrays.
+ *
+ * This field type corresponds to OpenSearch's {@code binary} field mapping and supports
+ * arbitrary byte sequences. All binary data is stored as-is without transformation.
+ *
+ * Usage Example:
+ * {@code
+ * BinaryParquetField binaryField = new BinaryParquetField();
+ * ArrowType arrowType = binaryField.getArrowType(); // Returns Binary type
+ * FieldType fieldType = binaryField.getFieldType(); // Returns nullable binary field type
+ * }
+ *
+ * @see ParquetField
+ * @see VarBinaryVector
+ * @see ArrowType.Binary
+ * @since 1.0
+ */
+public class BinaryParquetField extends ParquetField {
+
+ @Override
+ protected void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ final VarBinaryVector varBinaryVector = (VarBinaryVector) managedVSR.getVector(mappedFieldType.name());
+ int rowCount = managedVSR.getRowCount();
+ varBinaryVector.set(rowCount, (byte[]) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Binary();
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/BooleanParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/BooleanParquetField.java
new file mode 100644
index 0000000000000..4b2237bf1aa1f
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/BooleanParquetField.java
@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data;
+
+import com.parquet.parquetdataformat.fields.ArrowFieldRegistry;
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.BitVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling boolean data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch boolean fields and Apache Arrow
+ * boolean vectors for columnar storage in Parquet format. Boolean values are stored using
+ * Apache Arrow's {@link BitVector}, which provides efficient bit-level storage for boolean data.
+ *
+ * This field type corresponds to OpenSearch's {@code boolean} field mapping and is
+ * automatically registered in the {@link ArrowFieldRegistry} for use during document processing.
+ *
+ * Usage Example:
+ * {@code
+ * BooleanParquetField boolField = new BooleanParquetField();
+ * ArrowType arrowType = boolField.getArrowType(); // Returns ArrowType.Bool
+ * FieldType fieldType = boolField.getFieldType(); // Returns non-nullable boolean field type
+ * }
+ *
+ * @see ParquetField
+ * @see BitVector
+ * @see ArrowType.Bool
+ * @since 1.0
+ */
+public class BooleanParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ BitVector bitVector = (BitVector) managedVSR.getVector(mappedFieldType.name());
+ int rowIndex = managedVSR.getRowCount();
+ bitVector.setSafe(rowIndex, (Boolean) parseValue ? 1 : 0);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Bool();
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/IpParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/IpParquetField.java
new file mode 100644
index 0000000000000..be16d3154b66a
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/IpParquetField.java
@@ -0,0 +1,67 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.util.BytesRef;
+import org.opensearch.index.mapper.MappedFieldType;
+
+import java.net.InetAddress;
+
+/**
+ * Parquet field implementation for handling IP address data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch IP fields and Apache Arrow
+ * Binary string vectors for columnar storage in Parquet format. IP address values are encoded
+ * using Lucene's {@link InetAddressPoint} encoding and stored using Apache Arrow's
+ * {@link VarBinaryVector}, which provides efficient variable-length binary storage.
+ *
+ * This field type corresponds to OpenSearch's {@code ip} field mapping, which is used
+ * for storing IPv4 and IPv6 addresses. The IP addresses are internally encoded as binary
+ * data using Lucene's point encoding for efficient range queries and storage optimization.
+ *
+ * Usage Example:
+ * {@code
+ * IpParquetField ipField = new IpParquetField();
+ * ArrowType arrowType = ipField.getArrowType(); // Returns ArrowType.Binary
+ * FieldType fieldType = ipField.getFieldType(); // Returns nullable Binary field type
+ * }
+ *
+ * @see ParquetField
+ * @see VarBinaryVector
+ * @see InetAddressPoint
+ * @see ArrowType.Utf8
+ * @since 1.0
+ */
+public class IpParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ VarBinaryVector varBinaryVector = (VarBinaryVector) managedVSR.getVector(mappedFieldType.name());
+ int rowIndex = managedVSR.getRowCount();
+ final BytesRef bytesRef = new BytesRef(InetAddressPoint.encode((InetAddress) parseValue));
+ varBinaryVector.setSafe(rowIndex, bytesRef.bytes, bytesRef.offset, bytesRef.length);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Binary();
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/KeywordParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/KeywordParquetField.java
new file mode 100644
index 0000000000000..1814e20891f4e
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/KeywordParquetField.java
@@ -0,0 +1,62 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Parquet field implementation for handling keyword data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch keyword fields and Apache Arrow
+ * UTF-8 string vectors for columnar storage in Parquet format. Keyword values are stored using
+ * Apache Arrow's {@link VarCharVector}, which provides efficient variable-length string storage
+ * with UTF-8 encoding.
+ *
+ * This field type corresponds to OpenSearch's {@code keyword} field mapping, which is
+ * typically used for exact-match searches, aggregations, and sorting. Unlike text fields,
+ * keyword fields are not analyzed and are stored as-is for precise matching.
+ *
+ * Usage Example:
+ * {@code
+ * KeywordParquetField keywordField = new KeywordParquetField();
+ * ArrowType arrowType = keywordField.getArrowType(); // Returns ArrowType.Utf8
+ * FieldType fieldType = keywordField.getFieldType(); // Returns non-nullable UTF-8 field type
+ * }
+ *
+ * @see ParquetField
+ * @see VarCharVector
+ * @see ArrowType.Utf8
+ * @since 1.0
+ */
+public class KeywordParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ VarCharVector textVector = (VarCharVector) managedVSR.getVector(mappedFieldType.name());
+ int rowIndex = managedVSR.getRowCount();
+ textVector.setSafe(rowIndex, parseValue.toString().getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Utf8();
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/TextParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/TextParquetField.java
new file mode 100644
index 0000000000000..e4c93aa9f608f
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/TextParquetField.java
@@ -0,0 +1,63 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data;
+
+import com.parquet.parquetdataformat.fields.ArrowFieldRegistry;
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Parquet field implementation for handling text data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch text fields and Apache Arrow
+ * vectors for columnar storage in Parquet format. Text values are stored using Apache Arrow's
+ * {@link VarCharVector}, which provides efficient variable-length string storage with UTF-8 encoding.
+ *
+ * This field type corresponds to OpenSearch's {@code text} field mapping, which is
+ * typically used for full-text search operations. Text fields are usually analyzed during
+ * indexing, but this implementation stores the original text content for columnar access.
+ *
+ * Usage Example:
+ * {@code
+ * TextParquetField textField = new TextParquetField();
+ * ArrowType arrowType = textField.getArrowType(); // Returns ArrowType.Utf8
+ * FieldType fieldType = textField.getFieldType(); // Returns non-nullable integer field type
+ * }
+ *
+ * @see ParquetField
+ * @see ArrowFieldRegistry
+ * @see VarCharVector
+ * @see ArrowType.Int
+ * @since 1.0
+ */
+public class TextParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ VarCharVector textVector = (VarCharVector) managedVSR.getVector(mappedFieldType.name());
+ int rowIndex = managedVSR.getRowCount();
+ textVector.setSafe(rowIndex, parseValue.toString().getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Utf8();
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/TokenCountParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/TokenCountParquetField.java
new file mode 100644
index 0000000000000..603189bddc80b
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/TokenCountParquetField.java
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling token count data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch token count fields and Apache Arrow
+ * 32-bit signed integer vectors for columnar storage in Parquet format. Token count values are stored
+ * using Apache Arrow's {@link IntVector}, which provides efficient storage for signed integer values
+ * representing the number of tokens in analyzed text fields.
+ *
+ * This field type corresponds to OpenSearch's {@code token_count} field mapping, which is used
+ * for storing the count of tokens produced by text analysis. This field is particularly useful
+ * for implementing token-based queries, analyzing text complexity, and performing aggregations
+ * based on document length in terms of token count.
+ *
+ * Usage Example:
+ * {@code
+ * TokenCountParquetField tokenCountField = new TokenCountParquetField();
+ * ArrowType arrowType = tokenCountField.getArrowType(); // Returns ArrowType.Int(32, true)
+ * FieldType fieldType = tokenCountField.getFieldType(); // Returns non-nullable 32-bit signed integer field type
+ * }
+ *
+ * @see ParquetField
+ * @see IntVector
+ * @see ArrowType.Int
+ * @since 1.0
+ */
+public class TokenCountParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ IntVector intVector = (IntVector) managedVSR.getVector(mappedFieldType.name());
+ int rowCount = managedVSR.getRowCount();
+ intVector.setSafe(rowCount, (Integer) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Int(32, true);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/date/DateNanosParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/date/DateNanosParquetField.java
new file mode 100644
index 0000000000000..09ca4d50c9fe7
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/date/DateNanosParquetField.java
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data.date;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.TimeStampNanoVector;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling date and timestamp data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch date fields and Apache Arrow
+ * timestamp vectors for columnar storage in Parquet format. Date values are stored using
+ * Apache Arrow's {@link TimeStampNanoVector}, which stores timestamps as nanoseconds since the
+ * Unix epoch (January 1, 1970, 00:00:00 UTC).
+ *
+ * This field type corresponds to OpenSearch's {@code date_nanos} field mapping and supports
+ * various date formats as configured in the field mapping. All dates are normalized to
+ * nanosecond timestamps before storage in the Arrow vector.
+ *
+ * Usage Example:
+ * {@code
+ * DateParquetField dateField = new DateParquetField();
+ * ArrowType arrowType = dateField.getArrowType(); // Returns Timestamp with NANOSECOND precision
+ * FieldType fieldType = dateField.getFieldType(); // Returns non-nullable timestamp field type
+ * }
+ *
+ * @see ParquetField
+ * @see TimeStampNanoVector
+ * @see ArrowType.Timestamp
+ * @since 1.0
+ */
+public class DateNanosParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ TimeStampNanoVector timeStampNanoVector = (TimeStampNanoVector) managedVSR.getVector(mappedFieldType.name());
+ int rowIndex = managedVSR.getRowCount();
+ timeStampNanoVector.setSafe(rowIndex, (long) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Timestamp(TimeUnit.NANOSECOND, null);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/date/DateParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/date/DateParquetField.java
new file mode 100644
index 0000000000000..8554314e722a7
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/date/DateParquetField.java
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data.date;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.types.TimeUnit;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling date and timestamp data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch date fields and Apache Arrow
+ * timestamp vectors for columnar storage in Parquet format. Date values are stored using
+ * Apache Arrow's {@link TimeStampMilliVector}, which stores timestamps as milliseconds since the
+ * Unix epoch (January 1, 1970, 00:00:00 UTC).
+ *
+ * This field type corresponds to OpenSearch's {@code date} field mapping and supports
+ * various date formats as configured in the field mapping. All dates are normalized to
+ * millisecond timestamps before storage in the Arrow vector.
+ *
+ * Usage Example:
+ * {@code
+ * DateParquetField dateField = new DateParquetField();
+ * ArrowType arrowType = dateField.getArrowType(); // Returns Timestamp with MILLISECOND precision
+ * FieldType fieldType = dateField.getFieldType(); // Returns non-nullable timestamp field type
+ * }
+ *
+ * @see ParquetField
+ * @see TimeStampMilliVector
+ * @see ArrowType.Timestamp
+ * @since 1.0
+ */
+public class DateParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ TimeStampMilliVector timeStampMilliVector = (TimeStampMilliVector) managedVSR.getVector(mappedFieldType.name());
+ int rowIndex = managedVSR.getRowCount();
+ timeStampMilliVector.setSafe(rowIndex, (long) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Timestamp(TimeUnit.MILLISECOND, null);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/ByteParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/ByteParquetField.java
new file mode 100644
index 0000000000000..d9d45faeb3872
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/ByteParquetField.java
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data.number;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.TinyIntVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling 8-bit signed byte integer data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch byte fields and Apache Arrow
+ * 8-bit signed integer vectors for columnar storage in Parquet format. Byte values are stored
+ * using Apache Arrow's {@link TinyIntVector}, which provides efficient fixed-width 8-bit integer storage.
+ *
+ * This field type corresponds to OpenSearch's {@code byte} number field mapping and
+ * supports the full range of 8-bit signed integer values.
+ *
+ * Usage Example:
+ * {@code
+ * ByteParquetField byteField = new ByteParquetField();
+ * ArrowType arrowType = byteField.getArrowType(); // Returns 8-bit signed integer type
+ * FieldType fieldType = byteField.getFieldType(); // Returns non-nullable byte field type
+ * }
+ *
+ * @see ParquetField
+ * @see TinyIntVector
+ * @see ArrowType.Int
+ * @since 1.0
+ */
+public class ByteParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ TinyIntVector tinyIntVector = (TinyIntVector) managedVSR.getVector(mappedFieldType.name());
+ int rowCount = managedVSR.getRowCount();
+ tinyIntVector.setSafe(rowCount, (Byte) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Int(8, true);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/DoubleParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/DoubleParquetField.java
new file mode 100644
index 0000000000000..ac2b3a6e62927
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/DoubleParquetField.java
@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data.number;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling double-precision floating-point data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch double fields and Apache Arrow
+ * double-precision floating-point vectors for columnar storage in Parquet format. Double values are stored
+ * using Apache Arrow's {@link Float8Vector}, which provides efficient 64-bit IEEE 754 double-precision
+ * floating-point storage.
+ *
+ * This field type corresponds to OpenSearch's {@code double} number field mapping and
+ * supports the full range of IEEE 754 double-precision floating-point values.
+ *
+ * Usage Example:
+ * {@code
+ * DoubleParquetField doubleField = new DoubleParquetField();
+ * ArrowType arrowType = doubleField.getArrowType(); // Returns double-precision floating-point type
+ * FieldType fieldType = doubleField.getFieldType(); // Returns non-nullable double field type
+ * }
+ *
+ * @see ParquetField
+ * @see Float8Vector
+ * @see ArrowType.FloatingPoint
+ * @since 1.0
+ */
+public class DoubleParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ Float8Vector float8Vector = (Float8Vector) managedVSR.getVector(mappedFieldType.name());
+ int rowCount = managedVSR.getRowCount();
+ float8Vector.setSafe(rowCount, (Double) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/FloatParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/FloatParquetField.java
new file mode 100644
index 0000000000000..a516efd2f990f
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/FloatParquetField.java
@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data.number;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling single-precision floating-point data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch float fields and Apache Arrow
+ * single-precision floating-point vectors for columnar storage in Parquet format. Float values are stored
+ * using Apache Arrow's {@link Float4Vector}, which provides efficient 32-bit IEEE 754 single-precision
+ * floating-point storage.
+ *
+ * This field type corresponds to OpenSearch's {@code float} number field mapping and
+ * supports the full range of IEEE 754 single-precision floating-point values.
+ *
+ * Usage Example:
+ * {@code
+ * FloatParquetField floatField = new FloatParquetField();
+ * ArrowType arrowType = floatField.getArrowType(); // Returns single-precision floating-point type
+ * FieldType fieldType = floatField.getFieldType(); // Returns non-nullable float field type
+ * }
+ *
+ * @see ParquetField
+ * @see Float4Vector
+ * @see ArrowType.FloatingPoint
+ * @since 1.0
+ */
+public class FloatParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ Float4Vector float4Vector = (Float4Vector) managedVSR.getVector(mappedFieldType.name());
+ int rowCount = managedVSR.getRowCount();
+ float4Vector.setSafe(rowCount, (Float) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/HalfFloatParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/HalfFloatParquetField.java
new file mode 100644
index 0000000000000..3019773e6bd42
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/HalfFloatParquetField.java
@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data.number;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.Float2Vector;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling half-precision (16-bit) floating-point data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch half_float fields and Apache Arrow
+ * half-precision floating-point vectors for columnar storage in Parquet format. Half-float values are stored
+ * using Apache Arrow's {@link Float2Vector}, which provides efficient 16-bit IEEE 754 half-precision
+ * floating-point storage.
+ *
+ * This field type corresponds to OpenSearch's {@code half_float} number field mapping and
+ * supports IEEE 754 half-precision floating-point values.
+ *
+ * Usage Example:
+ * {@code
+ * HalfFloatParquetField halfFloatField = new HalfFloatParquetField();
+ * ArrowType arrowType = halfFloatField.getArrowType(); // Returns half-precision floating-point type
+ * FieldType fieldType = halfFloatField.getFieldType(); // Returns non-nullable half-float field type
+ * }
+ *
+ * @see ParquetField
+ * @see Float2Vector
+ * @see ArrowType.FloatingPoint
+ * @since 1.0
+ */
+public class HalfFloatParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ Float2Vector float2Vector = (Float2Vector) managedVSR.getVector(mappedFieldType.name());
+ int rowCount = managedVSR.getRowCount();
+ float2Vector.setSafe(rowCount, (Short) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.FloatingPoint(FloatingPointPrecision.HALF);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/IntegerParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/IntegerParquetField.java
new file mode 100644
index 0000000000000..b11d49b666799
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/IntegerParquetField.java
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data.number;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling 32-bit signed integer data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch integer fields and Apache Arrow
+ * 32-bit signed integer vectors for columnar storage in Parquet format. Integer values are stored
+ * using Apache Arrow's {@link IntVector}, which provides efficient fixed-width integer storage.
+ *
+ * This field type corresponds to OpenSearch's {@code integer} number field mapping and
+ * supports the full range of 32-bit signed integer values.
+ *
+ * Usage Example:
+ * {@code
+ * IntegerParquetField intField = new IntegerParquetField();
+ * ArrowType arrowType = intField.getArrowType(); // Returns 32-bit signed integer type
+ * FieldType fieldType = intField.getFieldType(); // Returns non-nullable integer field type
+ * }
+ *
+ * @see ParquetField
+ * @see IntVector
+ * @see ArrowType.Int
+ * @since 1.0
+ */
+public class IntegerParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ IntVector intVector = (IntVector) managedVSR.getVector(mappedFieldType.name());
+ int rowCount = managedVSR.getRowCount();
+ intVector.setSafe(rowCount, (Integer) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Int(32, true);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/LongParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/LongParquetField.java
new file mode 100644
index 0000000000000..850ac0f004649
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/LongParquetField.java
@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data.number;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling 64-bit signed long integer data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch long fields and Apache Arrow
+ * 64-bit signed integer vectors for columnar storage in Parquet format. Long values are stored
+ * using Apache Arrow's {@link BigIntVector}, which provides efficient fixed-width 64-bit integer storage.
+ *
+ * This field type corresponds to OpenSearch's {@code long} number field mapping and
+ * supports the full range of 64-bit signed integer values. The implementation includes proper
+ * null handling, setting explicit null markers when null values are encountered.
+ *
+ * Usage Example:
+ * {@code
+ * LongParquetField longField = new LongParquetField();
+ * ArrowType arrowType = longField.getArrowType(); // Returns 64-bit signed integer type
+ * FieldType fieldType = longField.getFieldType(); // Returns non-nullable long field type
+ * }
+ *
+ * @see ParquetField
+ * @see BigIntVector
+ * @see ArrowType.Int
+ * @since 1.0
+ */
+public class LongParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ BigIntVector bigIntVector = (BigIntVector) managedVSR.getVector(mappedFieldType.name());
+ int rowCount = managedVSR.getRowCount();
+ bigIntVector.setSafe(rowCount, (Long) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Int(64, true);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/ShortParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/ShortParquetField.java
new file mode 100644
index 0000000000000..07ee5c1b54814
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/ShortParquetField.java
@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data.number;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling 16-bit signed short integer data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch short fields and Apache Arrow
+ * 16-bit signed integer vectors for columnar storage in Parquet format. Short values are stored
+ * using Apache Arrow's {@link SmallIntVector}, which provides efficient fixed-width 16-bit integer storage.
+ *
+ * This field type corresponds to OpenSearch's {@code short} number field mapping and
+ * supports the full range of 16-bit signed integer values. The implementation includes proper
+ * null handling, setting explicit null markers when null values are encountered.
+ *
+ * Usage Example:
+ * {@code
+ * ShortParquetField shortField = new ShortParquetField();
+ * ArrowType arrowType = shortField.getArrowType(); // Returns 16-bit signed integer type
+ * FieldType fieldType = shortField.getFieldType(); // Returns non-nullable short field type
+ * }
+ *
+ * @see ParquetField
+ * @see SmallIntVector
+ * @see ArrowType.Int
+ * @since 1.0
+ */
+public class ShortParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ SmallIntVector smallIntVector = (SmallIntVector) managedVSR.getVector(mappedFieldType.name());
+ int rowCount = managedVSR.getRowCount();
+ smallIntVector.setSafe(rowCount, (Short) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Int(16, true);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/UnsignedLongParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/UnsignedLongParquetField.java
new file mode 100644
index 0000000000000..7f8e407f29092
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/data/number/UnsignedLongParquetField.java
@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.data.number;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.UInt8Vector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling 64-bit unsigned long integer data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch unsigned_long fields and Apache Arrow
+ * 64-bit unsigned integer vectors for columnar storage in Parquet format. Unsigned long values are stored
+ * using Apache Arrow's {@link UInt8Vector}, which provides efficient fixed-width 64-bit unsigned integer storage.
+ *
+ * This field type corresponds to OpenSearch's {@code unsigned_long} number field mapping and
+ * supports the full range of 64-bit unsigned integer values. The implementation includes proper
+ * null handling, setting explicit null markers when null values are encountered.
+ *
+ * Usage Example:
+ * {@code
+ * UnsignedLongParquetField unsignedLongField = new UnsignedLongParquetField();
+ * ArrowType arrowType = unsignedLongField.getArrowType(); // Returns 64-bit unsigned integer type
+ * FieldType fieldType = unsignedLongField.getFieldType(); // Returns non-nullable unsigned long field type
+ * }
+ *
+ * @see ParquetField
+ * @see UInt8Vector
+ * @see ArrowType.Int
+ * @since 1.0
+ */
+public class UnsignedLongParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ UInt8Vector uInt8Vector = (UInt8Vector) managedVSR.getVector(mappedFieldType.name());
+ int rowCount = managedVSR.getRowCount();
+ long longValue = ((Number) parseValue).longValue();
+ uInt8Vector.setSafe(rowCount, longValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Int(64, false);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/IdParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/IdParquetField.java
new file mode 100644
index 0000000000000..413a3938836fc
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/IdParquetField.java
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.metadata;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.apache.lucene.util.BytesRef;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling document ID metadata in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch document ID fields and Apache Arrow
+ * binary vectors for columnar storage in Parquet format. Document ID values are stored
+ * using Apache Arrow's {@link VarBinaryVector}, which stores raw bytes without UTF-8 validation.
+ *
+ * This field type corresponds to OpenSearch's {@code _id} metadata field and
+ * supports unique document identifiers. The ID values are processed from {@link BytesRef} objects
+ * and stored directly in the Arrow vector with proper offset and length handling.
+ *
+ * Usage Example:
+ * {@code
+ * IdParquetField idField = new IdParquetField();
+ * ArrowType arrowType = idField.getArrowType(); // Returns Binary type
+ * FieldType fieldType = idField.getFieldType(); // Returns nullable Binary field type
+ * }
+ *
+ * @see ParquetField
+ * @see VarBinaryVector
+ * @see ArrowType.Binary
+ * @since 1.0
+ */
+public class IdParquetField extends ParquetField {
+
+ @Override
+ protected void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ VarBinaryVector idVector = (VarBinaryVector) managedVSR.getVector(mappedFieldType.name());
+ int rowIndex = managedVSR.getRowCount();
+ BytesRef bytesRef = (BytesRef) parseValue;
+ idVector.setSafe(rowIndex, bytesRef.bytes, bytesRef.offset, bytesRef.length);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Binary();
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/IgnoredParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/IgnoredParquetField.java
new file mode 100644
index 0000000000000..c31e3932c2295
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/IgnoredParquetField.java
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.metadata;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Parquet field implementation for handling ignored field data types in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch ignored fields and Apache Arrow
+ * UTF-8 string vectors for columnar storage in Parquet format. Ignored field values are stored
+ * using Apache Arrow's {@link VarCharVector}, which provides efficient variable-length string storage.
+ *
+ * This field type corresponds to OpenSearch's {@code ignored} field mapping and
+ * supports fields that are indexed but not stored in the document source. The field values
+ * are converted to UTF-8 string representation before storage in the Arrow vector.
+ *
+ * Usage Example:
+ * {@code
+ * IgnoredParquetField ignoredField = new IgnoredParquetField();
+ * ArrowType arrowType = ignoredField.getArrowType(); // Returns UTF-8 string type
+ * FieldType fieldType = ignoredField.getFieldType(); // Returns nullable UTF-8 field type
+ * }
+ *
+ * @see ParquetField
+ * @see VarCharVector
+ * @see ArrowType.Utf8
+ * @since 1.0
+ */
+public class IgnoredParquetField extends ParquetField {
+
+ @Override
+ protected void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ VarCharVector varCharVector = (VarCharVector) managedVSR.getVector(mappedFieldType.name());
+ int rowIndex = managedVSR.getRowCount();
+ varCharVector.setSafe(rowIndex, parseValue.toString().getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Utf8();
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/RoutingParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/RoutingParquetField.java
new file mode 100644
index 0000000000000..ffacfa1995ed4
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/RoutingParquetField.java
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.metadata;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Parquet field implementation for handling routing metadata in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch routing fields and Apache Arrow
+ * UTF-8 string vectors for columnar storage in Parquet format. Routing values are stored
+ * using Apache Arrow's {@link VarCharVector}, which provides efficient variable-length string storage.
+ *
+ * This field type corresponds to OpenSearch's {@code _routing} metadata field and
+ * supports custom routing values that determine which shard a document is stored on. The routing
+ * value is converted to UTF-8 bytes before storage in the Arrow vector.
+ *
+ * Usage Example:
+ * {@code
+ * RoutingParquetField routingField = new RoutingParquetField();
+ * ArrowType arrowType = routingField.getArrowType(); // Returns UTF-8 string type
+ * FieldType fieldType = routingField.getFieldType(); // Returns nullable UTF-8 field type
+ * }
+ *
+ * @see ParquetField
+ * @see VarCharVector
+ * @see ArrowType.Utf8
+ * @since 1.0
+ */
+public class RoutingParquetField extends ParquetField {
+
+ @Override
+ protected void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ VarCharVector routingVector = (VarCharVector) managedVSR.getVector(mappedFieldType.name());
+ int rowIndex = managedVSR.getRowCount();
+ routingVector.setSafe(rowIndex, parseValue.toString().getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Utf8();
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/SizeParquetField.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/SizeParquetField.java
new file mode 100644
index 0000000000000..1367cc7542155
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/fields/core/metadata/SizeParquetField.java
@@ -0,0 +1,61 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.fields.core.metadata;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.opensearch.index.mapper.MappedFieldType;
+
+/**
+ * Parquet field implementation for handling document size metadata in OpenSearch documents.
+ *
+ * This class provides the conversion logic between OpenSearch document size fields and Apache Arrow
+ * 32-bit signed integer vectors for columnar storage in Parquet format. Document size values are stored
+ * using Apache Arrow's {@link IntVector}, which provides efficient storage for signed integer values
+ * representing the size of documents in bytes.
+ *
+ * This field type corresponds to OpenSearch's {@code _size} metadata field, which is used
+ * for storing the size of the original document source in bytes. This field is particularly useful
+ * for monitoring storage usage, implementing size-based queries, and analyzing document distribution
+ * by size across indices.
+ *
+ * Usage Example:
+ * {@code
+ * SizeParquetField sizeField = new SizeParquetField();
+ * ArrowType arrowType = sizeField.getArrowType(); // Returns ArrowType.Int(32, true)
+ * FieldType fieldType = sizeField.getFieldType(); // Returns non-nullable 32-bit signed integer field type
+ * }
+ *
+ * @see ParquetField
+ * @see IntVector
+ * @see ArrowType.Int
+ * @since 1.0
+ */
+public class SizeParquetField extends ParquetField {
+
+ @Override
+ public void addToGroup(MappedFieldType mappedFieldType, ManagedVSR managedVSR, Object parseValue) {
+ IntVector intVector = (IntVector) managedVSR.getVector(mappedFieldType.name());
+ int rowCount = managedVSR.getRowCount();
+ intVector.setSafe(rowCount, (Integer) parseValue);
+ }
+
+ @Override
+ public ArrowType getArrowType() {
+ return new ArrowType.Int(32, true);
+ }
+
+ @Override
+ public FieldType getFieldType() {
+ return FieldType.nullable(getArrowType());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/memory/ArrowBufferPool.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/memory/ArrowBufferPool.java
new file mode 100644
index 0000000000000..4a71187f188ab
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/memory/ArrowBufferPool.java
@@ -0,0 +1,73 @@
+package com.parquet.parquetdataformat.memory;
+
+import com.parquet.parquetdataformat.ParquetDataFormatPlugin;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.RatioValue;
+import org.opensearch.monitor.jvm.JvmInfo;
+import org.opensearch.monitor.os.OsProbe;
+
+import java.io.Closeable;
+
+/**
+ * Manages BufferAllocator lifecycle with configurable allocation strategies.
+ * Provides factory methods for creating allocators with different policies
+ * based on OpenSearch settings and memory pressure conditions.
+ */
+public class ArrowBufferPool implements Closeable {
+
+ private static final Logger logger = LogManager.getLogger(ArrowBufferPool.class);
+
+ private final RootAllocator rootAllocator;
+ private final long maxChildAllocation;
+
+ public ArrowBufferPool(Settings settings) {
+ long maxAllocationInBytes = 10L * 1024 * 1024 * 1024;
+
+ logger.info("Max native memory allocation for ArrowBufferPool: {} bytes", maxAllocationInBytes);
+ this.rootAllocator = new RootAllocator(maxAllocationInBytes);
+ this.maxChildAllocation = 1024 * 1024 * 1024;
+ }
+
+ /**
+ * Creates a new child allocator with the configured strategy and limits.
+ *
+ * @param name Unique name for the allocator
+ * @return BufferAllocator configured with pool settings
+ */
+ public BufferAllocator createChildAllocator(String name) {
+ return createChildAllocator(name, maxChildAllocation);
+ }
+
+ /**
+ * Creates a new child allocator with custom limits.
+ *
+ * @param name Unique name for the allocator
+ * @param maxAllocation Maximum allocation limit
+ * @return BufferAllocator configured with specified limits
+ */
+ private BufferAllocator createChildAllocator(String name, long maxAllocation) {
+ return rootAllocator.newChildAllocator(name, 0, maxAllocation);
+ }
+
+ public long getTotalAllocatedBytes() {
+ return rootAllocator.getAllocatedMemory();
+ }
+
+ /**
+ * Closes all active allocators and cleans up the pool.
+ */
+ @Override
+ public void close() {
+ rootAllocator.close();
+ }
+
+ private static long getMaxAllocationInBytes(Settings settings) {
+ long totalAvailableSystemMemory = OsProbe.getInstance().getTotalPhysicalMemorySize() - JvmInfo.jvmInfo().getConfiguredMaxHeapSize();
+ RatioValue maxAllocationPercentage = RatioValue.parseRatioValue(settings.get(ParquetDataFormatPlugin.INDEX_MAX_NATIVE_ALLOCATION.getKey(), ParquetDataFormatPlugin.DEFAULT_MAX_NATIVE_ALLOCATION));
+ return (long) (totalAvailableSystemMemory * maxAllocationPercentage.getAsRatio());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/CompactionStrategy.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/CompactionStrategy.java
new file mode 100644
index 0000000000000..b2d3f6c16ea53
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/CompactionStrategy.java
@@ -0,0 +1,17 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.merge;
+
+/**
+ * Defines supported Parquet compaction strategies.
+ */
+public enum CompactionStrategy {
+ RECORD_BATCH
+}
+
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMergeExecutor.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMergeExecutor.java
new file mode 100644
index 0000000000000..f792f249b7356
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMergeExecutor.java
@@ -0,0 +1,34 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.merge;
+
+import org.opensearch.index.engine.exec.FileMetadata;
+import org.opensearch.index.engine.exec.WriterFileSet;
+import org.opensearch.index.engine.exec.merge.MergeResult;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Executes Parquet merge operations using a chosen compaction strategy.
+ */
+public class ParquetMergeExecutor extends ParquetMerger {
+
+ private final ParquetMergeStrategy strategy;
+
+ public ParquetMergeExecutor(CompactionStrategy compactionStrategy) {
+ this.strategy = ParquetMergeStrategyFactory.getStrategy(compactionStrategy);
+ }
+
+ @Override
+ public MergeResult merge(List fileMetadataList, long writerGeneration) {
+ MergeResult result = strategy.mergeParquetFiles(fileMetadataList, writerGeneration);
+ strategy.postMerge();
+ return result;
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMergeStrategy.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMergeStrategy.java
new file mode 100644
index 0000000000000..f84ccb795fc94
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMergeStrategy.java
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.merge;
+
+
+import org.opensearch.index.engine.exec.FileMetadata;
+import org.opensearch.index.engine.exec.WriterFileSet;
+import org.opensearch.index.engine.exec.merge.MergeResult;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Interface defining a Parquet merge strategy.
+ */
+public interface ParquetMergeStrategy {
+
+ /**
+ * Performs the actual Parquet merge.
+ */
+ MergeResult mergeParquetFiles(List files, long writerGeneration);
+
+ /**
+ * Optional post-merge hook.
+ */
+ default void postMerge() {
+ // No-op by default
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMergeStrategyFactory.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMergeStrategyFactory.java
new file mode 100644
index 0000000000000..361263531bf57
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMergeStrategyFactory.java
@@ -0,0 +1,23 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.merge;
+
+/**
+ * Factory for creating appropriate merge strategies based on compaction type.
+ */
+public class ParquetMergeStrategyFactory {
+
+ public static ParquetMergeStrategy getStrategy(CompactionStrategy compactionStrategy) {
+ switch (compactionStrategy) {
+ case RECORD_BATCH:
+ default:
+ return new RecordBatchMergeStrategy();
+ }
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMerger.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMerger.java
new file mode 100644
index 0000000000000..555e8a10c88f5
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/ParquetMerger.java
@@ -0,0 +1,25 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.merge;
+
+import org.opensearch.index.engine.exec.FileMetadata;
+import org.opensearch.index.engine.exec.Merger;
+import org.opensearch.index.engine.exec.WriterFileSet;
+import org.opensearch.index.engine.exec.merge.MergeResult;
+import org.opensearch.index.engine.exec.merge.RowIdMapping;
+
+import java.util.Collection;
+import java.util.List;
+
+public abstract class ParquetMerger implements Merger {
+ @Override
+ public MergeResult merge(List fileMetadataList, RowIdMapping rowIdMapping, long writerGeneration) {
+ throw new UnsupportedOperationException("Not supported parquet as secondary data format yet.");
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/RecordBatchMergeStrategy.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/RecordBatchMergeStrategy.java
new file mode 100644
index 0000000000000..6f2d8baf9f97c
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/merge/RecordBatchMergeStrategy.java
@@ -0,0 +1,79 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.merge;
+
+import com.parquet.parquetdataformat.engine.ParquetDataFormat;
+import com.parquet.parquetdataformat.engine.ParquetExecutionEngine;
+import org.opensearch.index.engine.exec.DataFormat;
+import org.opensearch.index.engine.exec.FileMetadata;
+import org.opensearch.index.engine.exec.WriterFileSet;
+import org.opensearch.index.engine.exec.merge.MergeResult;
+import org.opensearch.index.engine.exec.merge.RowId;
+import org.opensearch.index.engine.exec.merge.RowIdMapping;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+import static com.parquet.parquetdataformat.bridge.RustBridge.mergeParquetFilesInRust;
+
+/**
+ * Implements record-batch-based merging of Parquet files.
+ */
+public class RecordBatchMergeStrategy implements ParquetMergeStrategy {
+
+ @Override
+ public MergeResult mergeParquetFiles(List files, long writerGeneration) {
+
+ if (files.isEmpty()) {
+ throw new IllegalArgumentException("No files to merge");
+ }
+
+ List filePaths = new ArrayList<>();
+ files.forEach(writerFileSet -> writerFileSet.getFiles().forEach(
+ file -> filePaths.add(Path.of(writerFileSet.getDirectory(), file))));
+
+ String outputDirectory = files.iterator().next().getDirectory();
+ String mergedFilePath = getMergedFilePath(writerGeneration, outputDirectory);
+ String mergedFileName = getMergedFileName(writerGeneration);
+
+ // Merge files in Rust
+ mergeParquetFilesInRust(filePaths, mergedFilePath);
+
+ // Build row ID mapping
+ Map rowIdMapping = new HashMap<>();
+
+ WriterFileSet mergedWriterFileSet =
+ WriterFileSet.builder().directory(Path.of(outputDirectory)).addFile(mergedFileName).writerGeneration(writerGeneration).build();
+
+
+ Map mergedWriterFileSetMap = Collections.singletonMap(
+ new ParquetDataFormat(),
+ mergedWriterFileSet
+ );
+
+ return new MergeResult(new RowIdMapping(rowIdMapping, mergedFileName), mergedWriterFileSetMap);
+ }
+
+ private String getMergedFileName(long generation) {
+ // TODO
+ // For debuging we have added extra "merged" in file name, later we can remove and keep same as writer
+ return ParquetExecutionEngine.FILE_NAME_PREFIX + "_merged_" + generation + ParquetExecutionEngine.FILE_NAME_EXT;
+ }
+
+ private String getMergedFilePath(long generation, String outputDirectory) {
+ return Path.of(outputDirectory, getMergedFileName(generation)).toString();
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/plugins/fields/CoreDataFieldPlugin.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/plugins/fields/CoreDataFieldPlugin.java
new file mode 100644
index 0000000000000..20bdfc9610d13
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/plugins/fields/CoreDataFieldPlugin.java
@@ -0,0 +1,126 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.plugins.fields;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.fields.core.data.BinaryParquetField;
+import com.parquet.parquetdataformat.fields.core.data.date.DateNanosParquetField;
+import com.parquet.parquetdataformat.fields.core.data.TokenCountParquetField;
+import com.parquet.parquetdataformat.fields.core.data.BooleanParquetField;
+import com.parquet.parquetdataformat.fields.core.data.date.DateParquetField;
+import com.parquet.parquetdataformat.fields.core.data.IpParquetField;
+import com.parquet.parquetdataformat.fields.core.data.KeywordParquetField;
+import com.parquet.parquetdataformat.fields.core.data.TextParquetField;
+import com.parquet.parquetdataformat.fields.core.data.number.ByteParquetField;
+import com.parquet.parquetdataformat.fields.core.data.number.DoubleParquetField;
+import com.parquet.parquetdataformat.fields.core.data.number.FloatParquetField;
+import com.parquet.parquetdataformat.fields.core.data.number.HalfFloatParquetField;
+import com.parquet.parquetdataformat.fields.core.data.number.IntegerParquetField;
+import com.parquet.parquetdataformat.fields.core.data.number.LongParquetField;
+import com.parquet.parquetdataformat.fields.core.data.number.ShortParquetField;
+import com.parquet.parquetdataformat.fields.core.data.number.UnsignedLongParquetField;
+import org.opensearch.index.mapper.BinaryFieldMapper;
+import org.opensearch.index.mapper.BooleanFieldMapper;
+import org.opensearch.index.mapper.DateFieldMapper;
+import org.opensearch.index.mapper.IpFieldMapper;
+import org.opensearch.index.mapper.KeywordFieldMapper;
+import org.opensearch.index.mapper.NumberFieldMapper;
+import org.opensearch.index.mapper.TextFieldMapper;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Core data fields plugin that provides Parquet field implementations for all built-in OpenSearch field types.
+ * This plugin is automatically registered and provides the foundation field type support for Parquet storage.
+ */
+public class CoreDataFieldPlugin implements ParquetFieldPlugin {
+
+ @Override
+ public Map getParquetFields() {
+ final Map fieldMap = new HashMap<>();
+
+ // Register numeric field types
+ registerNumericFields(fieldMap);
+
+ // Register temporal field types
+ registerTemporalFields(fieldMap);
+
+ // Register boolean field types
+ registerBooleanFields(fieldMap);
+
+ // Register text-based field types
+ registerTextFields(fieldMap);
+
+ // Register binary field types
+ registerBinaryFields(fieldMap);
+
+ return fieldMap;
+ }
+
+ /**
+ * Registers all numeric field type mappings.
+ *
+ * @param fieldMap the map to populate with numeric field mappings
+ */
+ private static void registerNumericFields(final Map fieldMap) {
+ // Floating point types
+ fieldMap.put(NumberFieldMapper.NumberType.HALF_FLOAT.typeName(), new HalfFloatParquetField());
+ fieldMap.put(NumberFieldMapper.NumberType.FLOAT.typeName(), new FloatParquetField());
+ fieldMap.put(NumberFieldMapper.NumberType.DOUBLE.typeName(), new DoubleParquetField());
+
+ // Integer types
+ fieldMap.put(NumberFieldMapper.NumberType.BYTE.typeName(), new ByteParquetField());
+ fieldMap.put(NumberFieldMapper.NumberType.SHORT.typeName(), new ShortParquetField());
+ fieldMap.put(NumberFieldMapper.NumberType.INTEGER.typeName(), new IntegerParquetField());
+ fieldMap.put(NumberFieldMapper.NumberType.LONG.typeName(), new LongParquetField());
+ fieldMap.put(NumberFieldMapper.NumberType.UNSIGNED_LONG.typeName(), new UnsignedLongParquetField());
+ fieldMap.put("token_count", new TokenCountParquetField());
+ fieldMap.put("scaled_float", new LongParquetField());
+ }
+
+ /**
+ * Registers all temporal field type mappings.
+ *
+ * @param fieldMap the map to populate with temporal field mappings
+ */
+ private static void registerTemporalFields(final Map fieldMap) {
+ fieldMap.put(DateFieldMapper.CONTENT_TYPE, new DateParquetField());
+ fieldMap.put(DateFieldMapper.DATE_NANOS_CONTENT_TYPE, new DateNanosParquetField());
+ }
+
+ /**
+ * Registers all boolean field type mappings.
+ *
+ * @param fieldMap the map to populate with boolean field mappings
+ */
+ private static void registerBooleanFields(final Map fieldMap) {
+ fieldMap.put(BooleanFieldMapper.CONTENT_TYPE, new BooleanParquetField());
+ }
+
+ /**
+ * Registers all binary field type mappings.
+ *
+ * @param fieldMap the map to populate with binary field mappings
+ */
+ private static void registerBinaryFields(final Map fieldMap) {
+ fieldMap.put(BinaryFieldMapper.CONTENT_TYPE, new BinaryParquetField());
+ }
+
+ /**
+ * Registers all text-based field type mappings.
+ *
+ * @param fieldMap the map to populate with text field mappings
+ */
+ private static void registerTextFields(final Map fieldMap) {
+ fieldMap.put(TextFieldMapper.CONTENT_TYPE, new TextParquetField());
+ fieldMap.put(KeywordFieldMapper.CONTENT_TYPE, new KeywordParquetField());
+ fieldMap.put(IpFieldMapper.CONTENT_TYPE, new IpParquetField());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/plugins/fields/MetadataFieldPlugin.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/plugins/fields/MetadataFieldPlugin.java
new file mode 100644
index 0000000000000..69cc7e4548fd6
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/plugins/fields/MetadataFieldPlugin.java
@@ -0,0 +1,53 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.plugins.fields;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+import com.parquet.parquetdataformat.fields.core.data.number.LongParquetField;
+import com.parquet.parquetdataformat.fields.core.metadata.IdParquetField;
+import com.parquet.parquetdataformat.fields.core.metadata.IgnoredParquetField;
+import com.parquet.parquetdataformat.fields.core.metadata.RoutingParquetField;
+import com.parquet.parquetdataformat.fields.core.metadata.SizeParquetField;
+import org.opensearch.index.mapper.DocCountFieldMapper;
+import org.opensearch.index.mapper.IdFieldMapper;
+import org.opensearch.index.mapper.IgnoredFieldMapper;
+import org.opensearch.index.mapper.RoutingFieldMapper;
+import org.opensearch.index.mapper.SeqNoFieldMapper;
+import org.opensearch.index.mapper.VersionFieldMapper;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class MetadataFieldPlugin implements ParquetFieldPlugin {
+
+ @Override
+ public Map getParquetFields() {
+ final Map fieldMap = new HashMap<>();
+
+ // Register metadata field types
+ registerMetadataFields(fieldMap);
+
+ return fieldMap;
+ }
+
+ /**
+ * Registers all metadata field type mappings.
+ *
+ * @param fieldMap the map to populate with metadata field mappings
+ */
+ private static void registerMetadataFields(final Map fieldMap) {
+ fieldMap.put(DocCountFieldMapper.CONTENT_TYPE, new LongParquetField());
+ fieldMap.put("_size", new SizeParquetField());
+ fieldMap.put(RoutingFieldMapper.CONTENT_TYPE, new RoutingParquetField());
+ fieldMap.put(IgnoredFieldMapper.CONTENT_TYPE, new IgnoredParquetField());
+ fieldMap.put(IdFieldMapper.CONTENT_TYPE, new IdParquetField());
+ fieldMap.put(SeqNoFieldMapper.CONTENT_TYPE, new LongParquetField());
+ fieldMap.put(VersionFieldMapper.CONTENT_TYPE, new LongParquetField());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/plugins/fields/ParquetFieldPlugin.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/plugins/fields/ParquetFieldPlugin.java
new file mode 100644
index 0000000000000..09af099dc028a
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/plugins/fields/ParquetFieldPlugin.java
@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.plugins.fields;
+
+import com.parquet.parquetdataformat.fields.ParquetField;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Plugin interface for registering custom Parquet field implementations.
+ * Plugins implementing this interface can register their field types with the ArrowFieldRegistry.
+ */
+public interface ParquetFieldPlugin {
+
+ /**
+ * Returns additional Parquet field implementations added by this plugin.
+ *
+ * @return a map where keys are OpenSearch field type names and values are ParquetField instances
+ */
+ default Map getParquetFields() {
+ return Collections.emptyMap();
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/rowid/RowIdGenerator.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/rowid/RowIdGenerator.java
new file mode 100644
index 0000000000000..8735efc2b21dc
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/rowid/RowIdGenerator.java
@@ -0,0 +1,81 @@
+package com.parquet.parquetdataformat.rowid;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Atomic, monotonic row ID generator as specified in the Project Mustang design.
+ * Ensures that each parquet file has sequential row IDs starting from 0,
+ * maintaining a 1:1 mapping between docs indexed in Lucene and parquet rows.
+ */
+public class RowIdGenerator {
+
+ private final AtomicLong globalCounter;
+ private final String generatorId;
+
+ public RowIdGenerator(String generatorId) {
+ this.generatorId = generatorId;
+ this.globalCounter = new AtomicLong(0);
+ }
+
+ /**
+ * Generates the next monotonic row ID.
+ * Thread-safe and atomic operation.
+ *
+ * @return Next sequential row ID
+ */
+ public long nextRowId() {
+ return globalCounter.getAndIncrement();
+ }
+
+ /**
+ * Gets the current counter value without incrementing.
+ * Useful for determining the number of rows generated so far.
+ *
+ * @return Current counter value
+ */
+ public long getCurrentCount() {
+ return globalCounter.get();
+ }
+
+ /**
+ * Resets the counter to zero.
+ * Should only be used during testing or system reinitialization.
+ */
+ public void reset() {
+ globalCounter.set(0);
+ }
+
+ /**
+ * Gets the generator ID for tracking purposes.
+ *
+ * @return Generator identifier
+ */
+ public String getGeneratorId() {
+ return generatorId;
+ }
+
+ /**
+ * Gets generation statistics.
+ *
+ * @return GenerationStats with current state
+ */
+ public GenerationStats getStats() {
+ return new GenerationStats(generatorId, globalCounter.get());
+ }
+
+ /**
+ * Statistics for row ID generation.
+ */
+ public static class GenerationStats {
+ private final String generatorId;
+ private final long totalGenerated;
+
+ public GenerationStats(String generatorId, long totalGenerated) {
+ this.generatorId = generatorId;
+ this.totalGenerated = totalGenerated;
+ }
+
+ public String getGeneratorId() { return generatorId; }
+ public long getTotalGenerated() { return totalGenerated; }
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/rowid/RowIdTracker.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/rowid/RowIdTracker.java
new file mode 100644
index 0000000000000..418c96efa07ce
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/rowid/RowIdTracker.java
@@ -0,0 +1,204 @@
+package com.parquet.parquetdataformat.rowid;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Tracks row ID ranges per parquet file for Lucene segment mapping.
+ * Maintains the 1:1 mapping between docs indexed in Lucene and parquet rows
+ * as specified in the Project Mustang design.
+ */
+public class RowIdTracker {
+
+ private final ConcurrentMap fileRanges;
+ private final AtomicLong totalRowsTracked;
+
+ public RowIdTracker() {
+ this.fileRanges = new ConcurrentHashMap<>();
+ this.totalRowsTracked = new AtomicLong(0);
+ }
+
+ /**
+ * Starts tracking a new row ID range for a parquet file.
+ *
+ * @param fileName Name of the parquet file
+ * @param startRowId Starting row ID for this file
+ * @return RowIdRange tracker for this file
+ */
+ public RowIdRange startTracking(String fileName, long startRowId) {
+ RowIdRange range = new RowIdRange(fileName, startRowId);
+ fileRanges.put(fileName, range);
+ return range;
+ }
+
+ /**
+ * Completes tracking for a parquet file by setting the end row ID.
+ *
+ * @param fileName Name of the parquet file
+ * @param endRowId Final row ID for this file (exclusive)
+ * @return true if tracking was successfully completed
+ */
+ public boolean completeTracking(String fileName, long endRowId) {
+ RowIdRange range = fileRanges.get(fileName);
+ if (range != null) {
+ range.setEndRowId(endRowId);
+ long rowCount = endRowId - range.getStartRowId();
+ totalRowsTracked.addAndGet(rowCount);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Gets the row ID range for a specific parquet file.
+ *
+ * @param fileName Name of the parquet file
+ * @return RowIdRange for the file, or null if not found
+ */
+ public RowIdRange getRangeForFile(String fileName) {
+ return fileRanges.get(fileName);
+ }
+
+ /**
+ * Finds which parquet file contains the given row ID.
+ *
+ * @param rowId Row ID to search for
+ * @return File name containing the row ID, or null if not found
+ */
+ public String findFileForRowId(long rowId) {
+ for (RowIdRange range : fileRanges.values()) {
+ if (range.containsRowId(rowId)) {
+ return range.getFileName();
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Gets all tracked file ranges.
+ *
+ * @return ConcurrentMap of fileName -> RowIdRange
+ */
+ public ConcurrentMap getAllRanges() {
+ return new ConcurrentHashMap<>(fileRanges);
+ }
+
+ /**
+ * Gets tracking statistics.
+ *
+ * @return TrackingStats with current state
+ */
+ public TrackingStats getStats() {
+ return new TrackingStats(
+ fileRanges.size(),
+ totalRowsTracked.get(),
+ fileRanges.values().stream().mapToLong(RowIdRange::getRowCount).sum()
+ );
+ }
+
+ /**
+ * Removes tracking for a parquet file.
+ * Used during cleanup or file deletion.
+ *
+ * @param fileName Name of the parquet file
+ * @return true if tracking was removed
+ */
+ public boolean removeTracking(String fileName) {
+ RowIdRange removed = fileRanges.remove(fileName);
+ if (removed != null) {
+ totalRowsTracked.addAndGet(-removed.getRowCount());
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Clears all tracking data.
+ * Should only be used during testing or system reset.
+ */
+ public void clear() {
+ fileRanges.clear();
+ totalRowsTracked.set(0);
+ }
+
+ /**
+ * Represents a row ID range for a specific parquet file.
+ */
+ public static class RowIdRange {
+ private final String fileName;
+ private final long startRowId;
+ private volatile long endRowId;
+ private volatile boolean completed;
+
+ public RowIdRange(String fileName, long startRowId) {
+ this.fileName = fileName;
+ this.startRowId = startRowId;
+ this.endRowId = startRowId;
+ this.completed = false;
+ }
+
+ /**
+ * Sets the end row ID and marks the range as completed.
+ *
+ * @param endRowId Final row ID (exclusive)
+ */
+ public void setEndRowId(long endRowId) {
+ this.endRowId = endRowId;
+ this.completed = true;
+ }
+
+ /**
+ * Checks if the given row ID falls within this range.
+ *
+ * @param rowId Row ID to check
+ * @return true if row ID is within range
+ */
+ public boolean containsRowId(long rowId) {
+ return completed && rowId >= startRowId && rowId < endRowId;
+ }
+
+ /**
+ * Gets the number of rows in this range.
+ *
+ * @return Row count, or 0 if not completed
+ */
+ public long getRowCount() {
+ return completed ? endRowId - startRowId : 0;
+ }
+
+ // Getters
+ public String getFileName() { return fileName; }
+ public long getStartRowId() { return startRowId; }
+ public long getEndRowId() { return endRowId; }
+ public boolean isCompleted() { return completed; }
+
+ @Override
+ public String toString() {
+ return String.format("RowIdRange{file='%s', start=%d, end=%d, completed=%s}",
+ fileName, startRowId, endRowId, completed);
+ }
+ }
+
+ /**
+ * Statistics for row ID tracking.
+ */
+ public static class TrackingStats {
+ private final int trackedFiles;
+ private final long totalRowsTracked;
+ private final long activeRows;
+
+ public TrackingStats(int trackedFiles, long totalRowsTracked, long activeRows) {
+ this.trackedFiles = trackedFiles;
+ this.totalRowsTracked = totalRowsTracked;
+ this.activeRows = activeRows;
+ }
+
+ public int getTrackedFiles() { return trackedFiles; }
+ public long getTotalRowsTracked() { return totalRowsTracked; }
+ public long getActiveRows() { return activeRows; }
+ public double getAverageRowsPerFile() {
+ return trackedFiles > 0 ? (double) activeRows / trackedFiles : 0.0;
+ }
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/ManagedVSR.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/ManagedVSR.java
new file mode 100644
index 0000000000000..3b4113b0abc00
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/ManagedVSR.java
@@ -0,0 +1,231 @@
+package com.parquet.parquetdataformat.vsr;
+
+import com.parquet.parquetdataformat.bridge.ArrowExport;
+import org.apache.arrow.memory.ArrowBuf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.c.ArrowArray;
+import org.apache.arrow.c.ArrowSchema;
+import org.apache.arrow.c.Data;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.arrow.vector.BitVectorHelper.byteIndex;
+
+/**
+ * Managed wrapper around VectorSchemaRoot that handles state transitions
+ * and provides thread-safe access for the ACTIVE/FROZEN lifecycle.
+ */
+public class ManagedVSR implements AutoCloseable {
+
+ private static final Logger logger = LogManager.getLogger(ManagedVSR.class);
+
+ private final String id;
+ private final VectorSchemaRoot vsr;
+ private final BufferAllocator allocator;
+ private final AtomicReference state;
+ private final ReadWriteLock lock;
+ private final long createdTime;
+
+
+ public ManagedVSR(String id, VectorSchemaRoot vsr, BufferAllocator allocator) {
+ this.id = id;
+ this.vsr = vsr;
+ this.allocator = allocator;
+ this.state = new AtomicReference<>(VSRState.ACTIVE);
+ this.lock = new ReentrantReadWriteLock();
+ this.createdTime = System.currentTimeMillis();
+ }
+
+ /**
+ * Gets the underlying VectorSchemaRoot.
+ * Should only be used when holding appropriate locks.
+ *
+ * @return VectorSchemaRoot instance
+ */
+ public VectorSchemaRoot getVSR() {
+ return vsr;
+ }
+
+ /**
+ * Gets the current row count in this VSR.
+ * Thread-safe read operation.
+ *
+ * @return Number of rows currently in the VSR
+ */
+ public int getRowCount() {
+ lock.readLock().lock();
+ try {
+ return vsr.getRowCount();
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Sets the row count for this VSR.
+ * Only allowed when VSR is in ACTIVE state.
+ *
+ * @param rowCount New row count
+ * @throws IllegalStateException if VSR is not active or is immutable
+ */
+ public void setRowCount(int rowCount) {
+ lock.writeLock().lock();
+ try {
+ if (state.get() != VSRState.ACTIVE) {
+ throw new IllegalStateException("Cannot modify VSR in state: " + state.get());
+ }
+ vsr.setRowCount(rowCount);
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ /**
+ * Gets a field vector by name.
+ * Thread-safe read operation.
+ *
+ * @param fieldName Name of the field
+ * @return FieldVector for the field, or null if not found
+ */
+ public FieldVector getVector(String fieldName) {
+ lock.readLock().lock();
+ try {
+ return vsr.getVector(fieldName);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Changes the state of this VSR.
+ * Handles state transition logic and immutability.
+ *
+ * @param newState New state to transition to
+ */
+ public void setState(VSRState newState) {
+ VSRState oldState = state.getAndSet(newState);
+
+ logger.debug("State transition: {} -> {} for VSR {}", oldState, newState, id);
+ }
+
+ /**
+ * Gets the current state of this VSR.
+ *
+ * @return Current VSRState
+ */
+ public VSRState getState() {
+ return state.get();
+ }
+
+ /**
+ * Exports this VSR to Arrow C Data Interface for Rust handoff.
+ * Only allowed when VSR is FROZEN or FLUSHING.
+ *
+ * @return ArrowExport containing ArrowArray and ArrowSchema
+ * @throws IllegalStateException if VSR is not in correct state
+ */
+ public ArrowExport exportToArrow() {
+ VSRState currentState = state.get();
+ if (currentState != VSRState.FROZEN &&
+ currentState != VSRState.FLUSHING) {
+ throw new IllegalStateException("Cannot export VSR in state: " + currentState);
+ }
+
+ lock.readLock().lock();
+ try {
+ ArrowArray arrowArray = ArrowArray.allocateNew(allocator);
+ ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator);
+
+ // Export the VectorSchemaRoot to C Data Interface
+ Data.exportVectorSchemaRoot(allocator, vsr, null, arrowArray, arrowSchema);
+
+ return new ArrowExport(arrowArray, arrowSchema);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public ArrowExport exportSchema() {
+ lock.readLock().lock();
+ try {
+ ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator);
+
+ // Export the VectorSchemaRoot to C Data Interface
+ Data.exportSchema(allocator, vsr.getSchema(), null, arrowSchema);
+
+ return new ArrowExport(null, arrowSchema);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /**
+ * Checks if this VSR is immutable (frozen).
+ *
+ * @return true if VSR cannot be modified
+ */
+ public boolean isImmutable() {
+ VSRState currentState = state.get();
+ return currentState != VSRState.ACTIVE;
+ }
+
+
+ /**
+ * Gets the VSR ID.
+ *
+ * @return Unique identifier for this VSR
+ */
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * Gets the creation timestamp.
+ *
+ * @return Creation time in milliseconds
+ */
+ public long getCreatedTime() {
+ return createdTime;
+ }
+
+ /**
+ * Gets the associated BufferAllocator.
+ *
+ * @return BufferAllocator used by this VSR
+ */
+ public BufferAllocator getAllocator() {
+ return allocator;
+ }
+
+ /**
+ * Closes this VSR and releases all resources.
+ */
+ @Override
+ public void close() {
+ lock.writeLock().lock();
+ try {
+ if (state.get() != VSRState.CLOSED) {
+ state.set(VSRState.CLOSED);
+ vsr.close();
+ allocator.close();
+ }
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+
+ @Override
+ public String toString() {
+ return String.format("ManagedVSR{id='%s', state=%s, rows=%d, immutable=%s}",
+ id, state.get(), getRowCount(), isImmutable());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/VSRManager.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/VSRManager.java
new file mode 100644
index 0000000000000..602402e31001d
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/VSRManager.java
@@ -0,0 +1,278 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package com.parquet.parquetdataformat.vsr;
+
+import com.parquet.parquetdataformat.bridge.ArrowExport;
+import com.parquet.parquetdataformat.bridge.NativeParquetWriter;
+import com.parquet.parquetdataformat.memory.ArrowBufferPool;
+import com.parquet.parquetdataformat.writer.ParquetDocumentInput;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.index.engine.exec.FlushIn;
+import org.opensearch.index.engine.exec.WriteResult;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Manages VectorSchemaRoot lifecycle with integrated memory management and native call wrappers.
+ * Provides a high-level interface for Parquet document operations using managed VSR abstractions.
+ *
+ * This class orchestrates the following components:
+ *
+ * - {@link ManagedVSR} - Thread-safe VSR with state management
+ * - {@link VSRPool} - Resource pooling for VSRs
+ * - {@link com.parquet.parquetdataformat.bridge.RustBridge} - Direct JNI calls to Rust backend
+ *
+ */
+public class VSRManager implements Closeable {
+ private final AtomicReference managedVSR = new AtomicReference<>();
+ private Map fieldVectorMap;
+ private final Schema schema;
+ private final String fileName;
+ private final VSRPool vsrPool;
+ private NativeParquetWriter writer;
+
+ private static final Logger logger = LogManager.getLogger(VSRManager.class);
+
+
+ public VSRManager(String fileName, Schema schema, ArrowBufferPool arrowBufferPool) {
+ this.fileName = fileName;
+ this.schema = schema;
+
+ // Create VSR pool
+ this.vsrPool = new VSRPool("pool-" + fileName, schema, arrowBufferPool);
+
+ // Get active VSR from pool
+ this.managedVSR.set(vsrPool.getActiveVSR());
+ initializeFieldVectorMap();
+ // Initialize writer lazily to avoid crashes
+ initializeWriter();
+ }
+
+ private void initializeWriter() {
+ try {
+ try (ArrowExport export = managedVSR.get().exportSchema()) {
+ writer = new NativeParquetWriter(fileName, export.getSchemaAddress());
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to initialize Parquet writer: " + e.getMessage(), e);
+ }
+ }
+
+ public WriteResult addToManagedVSR(ParquetDocumentInput document) throws IOException {
+ ManagedVSR currentVSR = managedVSR.updateAndGet(vsr -> {
+ if (vsr == null) {
+ ManagedVSR newVSR = vsrPool.getActiveVSR();
+ if (newVSR != null) {
+ reinitializeFieldVectorMap();
+ }
+ return newVSR;
+ }
+ return vsr;
+ });
+
+ if (currentVSR == null) {
+ throw new IOException("No active VSR available");
+ }
+ if (currentVSR.getState() != VSRState.ACTIVE) {
+ throw new IOException("Cannot add document - VSR is not active: " + currentVSR.getState());
+ }
+
+ logger.debug("addToManagedVSR called for {}, current row count: {}", fileName, currentVSR.getRowCount());
+
+ try {
+ // Since ParquetDocumentInput now works directly with ManagedVSR,
+ // fields should already be populated in vectors via addField() calls.
+ // We just need to finalize the document by calling addToWriter()
+ // which will increment the row count.
+ WriteResult result = document.addToWriter();
+
+ logger.debug("After adding document to {}, row count: {}", fileName, currentVSR.getRowCount());
+
+ // Check for VSR rotation AFTER successful document processing
+ maybeRotateActiveVSR();
+
+ return result;
+ } catch (Exception e) {
+ logger.error("Error in addToManagedVSR for {}: {}", fileName, e.getMessage(), e);
+ throw new IOException("Failed to add document: " + e.getMessage(), e);
+ }
+ }
+
+ public String flush(FlushIn flushIn) throws IOException {
+ ManagedVSR currentVSR = managedVSR.get();
+ logger.info("Flush called for {}, row count: {}", fileName, currentVSR.getRowCount());
+ try {
+ // Only flush if we have data
+ if (currentVSR.getRowCount() == 0) {
+ logger.debug("No data to flush for {}, returning null", fileName);
+ return null;
+ }
+
+ // Transition VSR to FROZEN state before flushing
+ currentVSR.setState(VSRState.FROZEN);
+ logger.info("Flushing {} rows for {}", currentVSR.getRowCount(), fileName);
+
+ // Transition to FLUSHING state
+ currentVSR.setState(VSRState.FLUSHING);
+
+ // Write through native writer handle
+ try (ArrowExport export = currentVSR.exportToArrow()) {
+ writer.write(export.getArrayAddress(), export.getSchemaAddress());
+ writer.close();
+ }
+ logger.info("Successfully flushed data for {}", fileName);
+
+ return fileName;
+ } catch (Exception e) {
+ logger.error("Error in flush for {}: {}", fileName, e.getMessage(), e);
+ throw new IOException("Failed to flush data: " + e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void close() {
+ try {
+ if (writer != null) {
+ writer.flush();
+ writer.close();
+ }
+ vsrPool.close();
+ managedVSR.set(null);
+ } catch (Exception e) {
+ logger.error("Error during close for {}: {}", fileName, e.getMessage(), e);
+ }
+ }
+
+ private boolean checkFlushConditions() {
+ // TODO: Implement memory pressure-based flush conditions
+ return false;
+ }
+
+ /**
+ * Handles VSR rotation after successful document addition.
+ * Checks if rotation is needed and immediately processes any frozen VSR.
+ */
+ public void maybeRotateActiveVSR() throws IOException {
+ try {
+ // Check if rotation is needed and perform it if safe
+ boolean rotated = vsrPool.maybeRotateActiveVSR();
+
+ if (rotated) {
+ logger.debug("VSR rotation occurred after document addition for {}", fileName);
+
+ // Get the frozen VSR that was just created by rotation
+ ManagedVSR frozenVSR = vsrPool.getFrozenVSR();
+ if (frozenVSR != null) {
+ logger.debug("Processing frozen VSR: {} with {} rows for {}",
+ frozenVSR.getId(), frozenVSR.getRowCount(), fileName);
+
+ // Write the frozen VSR data immediately
+ frozenVSR.setState(VSRState.FLUSHING);
+ try (ArrowExport export = frozenVSR.exportToArrow()) {
+ writer.write(export.getArrayAddress(), export.getSchemaAddress());
+ }
+
+ logger.debug("Successfully wrote frozen VSR data for {}", fileName);
+
+ // Complete the VSR processing
+ vsrPool.completeVSR(frozenVSR);
+ vsrPool.unsetFrozenVSR();
+ } else {
+ logger.warn("Rotation occurred but no frozen VSR found for {}", fileName);
+ }
+
+ // Update to new active VSR atomically with field vector map
+ ManagedVSR oldVSR = managedVSR.get();
+ ManagedVSR newVSR = vsrPool.getActiveVSR();
+ if (newVSR == null) {
+ throw new IOException("No active VSR available after rotation");
+ }
+ updateVSRAndReinitialize(oldVSR, newVSR);
+
+ // Reinitialize field vector map with new VSR
+ reinitializeFieldVectorMap();
+
+ logger.debug("VSR rotation completed for {}, new active VSR: {}, row count: {}",
+ fileName, newVSR.getId(), newVSR.getRowCount());
+ }
+ } catch (IOException e) {
+ logger.error("Error during VSR rotation for {}: {}", fileName, e.getMessage(), e);
+ throw e;
+ }
+ }
+
+ /**
+ * Checks if VSR rotation is needed based on row count and memory pressure.
+ * If rotation occurs, updates the managed VSR reference and reinitializes field vectors.
+ *
+ * @deprecated Use handleVSRRotationAfterAddToManagedVSR() instead for safer rotation after document processing
+ */
+ @Deprecated
+ private void checkAndHandleVSRRotation() throws IOException {
+ // Get active VSR from pool - this will trigger rotation if needed
+ ManagedVSR currentActive = vsrPool.getActiveVSR();
+
+ // Check if we got a different VSR (rotation occurred)
+ ManagedVSR oldVSR = managedVSR.get();
+ if (currentActive != oldVSR) {
+ logger.debug("VSR rotation detected for {}, updating references", fileName);
+
+ // Update the managed VSR reference atomically with field vector map
+ updateVSRAndReinitialize(oldVSR, currentActive);
+
+ // Note: Writer initialization is not needed per VSR as it's per file
+ logger.debug("VSR rotation completed for {}, new row count: {}", fileName, currentActive.getRowCount());
+ }
+ }
+
+ /**
+ * Atomically updates managedVSR and reinitializes field vector map.
+ */
+ private void updateVSRAndReinitialize(ManagedVSR oldVSR, ManagedVSR newVSR) {
+ if (managedVSR.compareAndSet(oldVSR, newVSR)) {
+ reinitializeFieldVectorMap();
+ }
+ }
+
+ /**
+ * Reinitializes the field vector map with the current managed VSR.
+ * Called after VSR rotation to update vector references.
+ */
+ private void reinitializeFieldVectorMap() {
+ fieldVectorMap.clear();
+ initializeFieldVectorMap();
+ }
+
+ private void initializeFieldVectorMap() {
+ fieldVectorMap = new HashMap<>();
+ for (Field field : schema.getFields()) {
+ String fieldName = field.getName();
+ FieldVector fieldVector = managedVSR.get().getVector(fieldName);
+ // Vector is already properly typed from ManagedVSR.getVector()
+ fieldVectorMap.put(fieldName, fieldVector);
+ }
+ }
+
+ /**
+ * Gets the current active ManagedVSR for document input creation.
+ *
+ * @return The current managed VSR instance
+ */
+ public ManagedVSR getActiveManagedVSR() {
+ return managedVSR.get();
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/VSRPool.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/VSRPool.java
new file mode 100644
index 0000000000000..4c3317200b712
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/VSRPool.java
@@ -0,0 +1,332 @@
+package com.parquet.parquetdataformat.vsr;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import com.parquet.parquetdataformat.memory.ArrowBufferPool;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Manages VectorSchemaRoot lifecycle with ACTIVE and FROZEN states as specified
+ * in the Project Mustang design. Each ParquetWriter maintains a single ACTIVE VSR
+ * for writing and a single FROZEN VSR for Rust handoff.
+ */
+public class VSRPool {
+
+ private static final Logger logger = LogManager.getLogger(VSRPool.class);
+
+ private final Schema schema;
+ private final ArrowBufferPool bufferPool;
+ private final String poolId;
+
+ // VSR lifecycle management
+ private final AtomicReference activeVSR;
+ private final AtomicReference frozenVSR;
+ private final ConcurrentHashMap allVSRs;
+ private final AtomicInteger vsrCounter;
+
+ // Configuration
+ private final int maxRowsPerVSR;
+
+ public VSRPool(String poolId, Schema schema, ArrowBufferPool arrowBufferPool) {
+ this.poolId = poolId;
+ this.schema = schema;
+ this.bufferPool = arrowBufferPool;
+ this.activeVSR = new AtomicReference<>();
+ this.frozenVSR = new AtomicReference<>();
+ this.allVSRs = new ConcurrentHashMap<>();
+ this.vsrCounter = new AtomicInteger(0);
+
+ // Configuration - could be made configurable
+ this.maxRowsPerVSR = 50000; // Max rows before forcing freeze
+
+ // Initialize with first active VSR
+ initializeActiveVSR();
+ }
+
+ /**
+ * Gets the current active VSR for writing.
+ * Simply returns the current active VSR without any rotation logic.
+ *
+ * @return Active ManagedVSR for writing, or null if none exists
+ */
+ public ManagedVSR getActiveVSR() {
+ return activeVSR.get();
+ }
+
+ /**
+ * Checks if VSR rotation is needed and performs it if safe to do so.
+ * Throws IOException if rotation is needed but frozen slot is occupied.
+ *
+ * @return true if rotation occurred, false if no rotation was needed
+ * @throws IOException if rotation is needed but cannot be performed due to occupied frozen slot
+ */
+ public boolean maybeRotateActiveVSR() throws IOException {
+ ManagedVSR current = activeVSR.get();
+
+ // Check if rotation is needed
+ if (current == null || !shouldRotateVSR(current)) {
+ return false; // No rotation needed
+ }
+
+ // CRITICAL: Check if frozen slot is occupied before rotation
+ if (frozenVSR.get() != null) {
+ throw new IOException("Cannot rotate VSR: frozen slot is occupied. " +
+ "Previous frozen VSR has not been processed. This indicates a " +
+ "system bottleneck or processing failure.");
+ }
+
+ // Safe to rotate - perform the rotation
+ synchronized (this) {
+ // Double-check conditions under lock
+ current = activeVSR.get();
+ if (current == null || !shouldRotateVSR(current)) {
+ return false; // Conditions changed while acquiring lock
+ }
+
+ // Check frozen slot again under lock
+ if (frozenVSR.get() != null) {
+ throw new IOException("Cannot rotate VSR: frozen slot became occupied during rotation");
+ }
+
+ // Freeze current VSR if it exists and has data
+ if (current != null && current.getRowCount() > 0) {
+ freezeVSR(current);
+ }
+
+ // Create new active VSR
+ ManagedVSR newActive = createNewVSR();
+ activeVSR.set(newActive);
+
+ return true; // Rotation occurred
+ }
+ }
+
+ /**
+ * Freezes the current active VSR and creates a new active one.
+ * The frozen VSR replaces any existing frozen VSR.
+ *
+ * @deprecated Use maybeRotateActiveVSR() instead for safer rotation with checks
+ * @return Newly created active VSR
+ */
+ @Deprecated
+ public ManagedVSR rotateActiveVSR() {
+ synchronized (this) {
+ ManagedVSR current = activeVSR.get();
+
+ // Freeze current VSR if it exists and has data
+ if (current != null && current.getRowCount() > 0) {
+ freezeVSR(current);
+ }
+
+ // Create new active VSR
+ ManagedVSR newActive = createNewVSR();
+ activeVSR.set(newActive);
+
+ return newActive;
+ }
+ }
+
+ /**
+ * Gets the frozen VSR for Rust processing.
+ *
+ * @return Frozen VSR, or null if none available
+ */
+ public ManagedVSR getFrozenVSR() {
+ return frozenVSR.get();
+ }
+
+ public void unsetFrozenVSR() throws IOException {
+ if (frozenVSR.get() == null) {
+ throw new IOException("unsetFrozenVSR called when frozen VSR is not set");
+ }
+ if (!VSRState.CLOSED.equals(frozenVSR.get().getState())) {
+ throw new IOException("frozenVSR cannot be unset, state is " + frozenVSR.get().getState());
+ }
+ frozenVSR.set(null);
+ }
+
+ /**
+ * Takes the frozen VSR for processing and clears the frozen slot.
+ *
+ * @return Frozen VSR that was taken, or null if none available
+ */
+ public ManagedVSR takeFrozenVSR() {
+ return frozenVSR.getAndSet(null);
+ }
+
+ /**
+ * Marks a VSR as flushing (being processed by Rust).
+ *
+ * @param vsr VSR being processed
+ */
+ public void markFlushing(ManagedVSR vsr) {
+ vsr.setState(VSRState.FLUSHING);
+ }
+
+ /**
+ * Completes VSR processing and cleans up resources.
+ *
+ * @param vsr VSR that has been processed
+ */
+ public void completeVSR(ManagedVSR vsr) {
+ vsr.setState(VSRState.CLOSED);
+ vsr.close();
+ allVSRs.remove(vsr.getId());
+ }
+
+ /**
+ * Forces all VSRs to be frozen for immediate processing.
+ * Used during refresh or shutdown.
+ */
+ public void freezeAll() {
+ ManagedVSR current = activeVSR.getAndSet(null);
+ if (current != null && current.getRowCount() > 0) {
+ freezeVSR(current);
+ }
+ }
+
+ /**
+ * Gets statistics about the VSR pool.
+ *
+ * @return PoolStats with current state
+ */
+ public PoolStats getStats() {
+ ManagedVSR active = activeVSR.get();
+ ManagedVSR frozen = frozenVSR.get();
+ int frozenCount = frozen != null ? 1 : 0;
+
+ return new PoolStats(
+ poolId,
+ active != null ? active.getRowCount() : 0,
+ frozenCount,
+ allVSRs.size(),
+ allVSRs.values().stream().mapToLong(ManagedVSR::getRowCount).sum()
+ );
+ }
+
+ /**
+ * Closes the pool and cleans up all resources.
+ */
+ public void close() {
+ // Close active VSR
+ ManagedVSR active = activeVSR.getAndSet(null);
+ if (active != null) {
+ active.close();
+ }
+
+ // Close frozen VSR
+ ManagedVSR frozen = frozenVSR.getAndSet(null);
+ if (frozen != null) {
+ frozen.close();
+ }
+
+ // Close any remaining VSRs
+ allVSRs.values().forEach(ManagedVSR::close);
+ allVSRs.clear();
+ }
+
+ private void initializeActiveVSR() {
+ ManagedVSR initial = createNewVSR();
+ activeVSR.set(initial);
+ }
+
+ private ManagedVSR createNewVSR() {
+
+ String vsrId = poolId + "-vsr-" + vsrCounter.incrementAndGet();
+ BufferAllocator allocator = null;
+ VectorSchemaRoot vsr = null;
+
+ try {
+ allocator = bufferPool.createChildAllocator(vsrId);
+ vsr = VectorSchemaRoot.create(schema, allocator);
+
+ ManagedVSR managedVSR = new ManagedVSR(vsrId, vsr, allocator);
+ allVSRs.put(vsrId, managedVSR);
+
+ // Success: ManagedVSR now owns the resources
+ return managedVSR;
+ } catch (Exception e) {
+ // Clean up resources on failure since ManagedVSR couldn't take ownership
+ if (vsr != null) {
+ try {
+ vsr.close();
+ } catch (Exception closeEx) {
+ e.addSuppressed(closeEx);
+ }
+ }
+ if (allocator != null) {
+ try {
+ allocator.close();
+ } catch (Exception closeEx) {
+ e.addSuppressed(closeEx);
+ }
+ }
+ throw new RuntimeException("Failed to create new VSR", e);
+ }
+ }
+
+ private void freezeVSR(ManagedVSR vsr) {
+ vsr.setState(VSRState.FROZEN);
+
+ // CRITICAL FIX: Check if frozen slot is already occupied
+ ManagedVSR previousFrozen = frozenVSR.get();
+ if (previousFrozen != null) {
+ // NEVER blindly overwrite a frozen VSR - this would cause data loss
+ logger.error("Attempting to freeze VSR when frozen slot is occupied! " +
+ "Previous VSR: {} ({} rows), New VSR: {} ({} rows). " +
+ "This indicates a logic error - frozen VSR should be consumed before replacement.",
+ previousFrozen.getId(), previousFrozen.getRowCount(),
+ vsr.getId(), vsr.getRowCount());
+
+ // Return VSR to ACTIVE state to prevent state corruption
+ vsr.setState(VSRState.ACTIVE);
+ throw new IllegalStateException("Cannot freeze VSR: frozen slot is occupied by unprocessed VSR " +
+ previousFrozen.getId() + ". This would cause data loss.");
+ }
+
+ // Safe to set frozen VSR since slot is empty
+ boolean success = frozenVSR.compareAndSet(null, vsr);
+ if (!success) {
+ // Race condition: another thread set frozen VSR between our check and set
+ vsr.setState(VSRState.ACTIVE);
+ throw new IllegalStateException("Race condition detected: frozen slot was occupied during freeze operation");
+ }
+ }
+
+ private boolean shouldRotateVSR(ManagedVSR vsr) {
+ return vsr.getRowCount() >= maxRowsPerVSR;
+ }
+
+ /**
+ * Statistics for the VSR pool.
+ */
+ public static class PoolStats {
+ private final String poolId;
+ private final long activeRowCount;
+ private final int frozenVSRCount;
+ private final int totalVSRCount;
+ private final long totalRowCount;
+
+ public PoolStats(String poolId, long activeRowCount, int frozenVSRCount,
+ int totalVSRCount, long totalRowCount) {
+ this.poolId = poolId;
+ this.activeRowCount = activeRowCount;
+ this.frozenVSRCount = frozenVSRCount;
+ this.totalVSRCount = totalVSRCount;
+ this.totalRowCount = totalRowCount;
+ }
+
+ public String getPoolId() { return poolId; }
+ public long getActiveRowCount() { return activeRowCount; }
+ public int getFrozenVSRCount() { return frozenVSRCount; }
+ public int getTotalVSRCount() { return totalVSRCount; }
+ public long getTotalRowCount() { return totalRowCount; }
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/VSRState.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/VSRState.java
new file mode 100644
index 0000000000000..cd55f30ca24cc
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/vsr/VSRState.java
@@ -0,0 +1,28 @@
+package com.parquet.parquetdataformat.vsr;
+
+/**
+ * Represents the lifecycle states of a VectorSchemaRoot in the Project Mustang
+ * Parquet Writer Plugin architecture.
+ */
+public enum VSRState {
+ /**
+ * Currently accepting writes - the VSR is active and can be modified.
+ */
+ ACTIVE,
+
+ /**
+ * Read-only state - VSR is frozen and queued for flush to Rust.
+ * No further modifications are allowed in this state.
+ */
+ FROZEN,
+
+ /**
+ * Currently being processed by Rust - VSR is in the handoff process.
+ */
+ FLUSHING,
+
+ /**
+ * Completed and cleaned up - VSR processing is complete and resources freed.
+ */
+ CLOSED
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/writer/ParquetDocumentInput.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/writer/ParquetDocumentInput.java
new file mode 100644
index 0000000000000..41bb192f55ea3
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/writer/ParquetDocumentInput.java
@@ -0,0 +1,94 @@
+package com.parquet.parquetdataformat.writer;
+
+import com.parquet.parquetdataformat.fields.ArrowFieldRegistry;
+import com.parquet.parquetdataformat.fields.ParquetField;
+import org.apache.arrow.vector.BigIntVector;
+import org.opensearch.index.engine.exec.DocumentInput;
+import org.opensearch.index.engine.exec.WriteResult;
+import org.opensearch.index.engine.exec.composite.CompositeDataFormatWriter;
+import org.opensearch.index.mapper.MappedFieldType;
+import com.parquet.parquetdataformat.vsr.ManagedVSR;
+
+import java.io.IOException;
+
+/**
+ * Document input wrapper for Parquet-based document processing.
+ *
+ * This class serves as an adapter between OpenSearch's DocumentInput interface
+ * and the Arrow-based vector representation. It works directly with a {@link ManagedVSR}
+ * to populate field vectors and manage document lifecycle.
+ *
+ *
The implementation follows the builder pattern, allowing incremental construction
+ * of documents through field addition before finalizing the document for writing.
+ *
+ *
Key responsibilities:
+ *
+ * - Direct field vector population using OpenSearch's {@link MappedFieldType}
+ * - Document lifecycle management via ManagedVSR
+ * - Integration with the Arrow-based Parquet writer pipeline
+ *
+ *
+ * This implementation works directly with Arrow field vectors, eliminating the
+ * intermediate ParquetDocument representation for improved performance and memory efficiency.
+ */
+public class ParquetDocumentInput implements DocumentInput {
+ private final ManagedVSR managedVSR;
+
+ public ParquetDocumentInput(ManagedVSR managedVSR) {
+ this.managedVSR = managedVSR;
+ }
+
+ @Override
+ public void addRowIdField(String fieldName, long rowId) {
+ BigIntVector bigIntVector = (BigIntVector) managedVSR.getVector(CompositeDataFormatWriter.ROW_ID);
+ int rowCount = managedVSR.getRowCount();
+ bigIntVector.setSafe(rowCount, rowId);
+ }
+
+ @Override
+ public void addField(MappedFieldType fieldType, Object value) {
+ final String fieldTypeName = fieldType.typeName();
+ final ParquetField parquetField = ArrowFieldRegistry.getParquetField(fieldTypeName);
+
+ if (parquetField == null) {
+ throw new IllegalArgumentException(
+ String.format("Unsupported field type: %s. Field type is not registered in ArrowFieldRegistry.", fieldTypeName)
+ );
+ }
+
+ parquetField.createField(fieldType, managedVSR, value);
+ }
+
+ @Override
+ public void setPrimaryTerm(String fieldName, long primaryTerm) {
+ BigIntVector bigIntVector = (BigIntVector) managedVSR.getVector(fieldName);
+ int rowCount = managedVSR.getRowCount();
+ bigIntVector.setSafe(rowCount, primaryTerm);
+ }
+
+ @Override
+ public ManagedVSR getFinalInput() {
+ return managedVSR;
+ }
+
+ @Override
+ public WriteResult addToWriter() throws IOException {
+ // Complete the current document by incrementing row count
+ // This will internally call setValueCount on all field vectors
+ int currentRowCount = managedVSR.getRowCount();
+ managedVSR.setRowCount(currentRowCount + 1);
+
+ // TODO: Return appropriate WriteResult based on operation success
+ return new WriteResult(true, null, 1, 1, 1);
+ }
+
+ @Override
+ public void close() throws Exception {
+ // NOTE: ParquetDocumentInput does NOT own the ManagedVSR lifecycle
+ // The ManagedVSR is owned and managed by VSRManager/VSRPool
+ // VSRManager.close() -> vsrPool.completeVSR(managedVSR) handles cleanup
+ // ParquetDocumentInput only holds a reference for field population
+
+ // No cleanup needed here - VSRManager handles the ManagedVSR lifecycle
+ }
+}
diff --git a/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/writer/ParquetWriter.java b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/writer/ParquetWriter.java
new file mode 100644
index 0000000000000..84df70879e550
--- /dev/null
+++ b/modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/writer/ParquetWriter.java
@@ -0,0 +1,96 @@
+package com.parquet.parquetdataformat.writer;
+
+import com.parquet.parquetdataformat.memory.ArrowBufferPool;
+import com.parquet.parquetdataformat.vsr.VSRManager;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.index.engine.exec.FileInfos;
+import org.opensearch.index.engine.exec.FlushIn;
+import org.opensearch.index.engine.exec.WriteResult;
+import org.opensearch.index.engine.exec.Writer;
+import org.opensearch.index.engine.exec.WriterFileSet;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+import static com.parquet.parquetdataformat.engine.ParquetDataFormat.PARQUET_DATA_FORMAT;
+
+/**
+ * Parquet file writer implementation that integrates with OpenSearch's Writer interface.
+ *
+ * This writer provides a high-level interface for writing Parquet documents to disk
+ * using the underlying VSRManager for Arrow-based data management and native Rust
+ * backend for efficient Parquet file generation.
+ *
+ *
Key features:
+ *
+ * - Arrow schema-based document structure
+ * - Batch-oriented writing with memory management
+ * - Integration with OpenSearch indexing pipeline
+ * - Native Rust backend for high-performance Parquet operations
+ *
+ *
+ * The writer manages the complete lifecycle from document addition through
+ * flushing and cleanup, delegating the actual Arrow and Parquet operations
+ * to the {@link VSRManager}.
+ */
+public class ParquetWriter implements Writer {
+
+ private static final Logger logger = LogManager.getLogger(ParquetWriter.class);
+
+ private final String file;
+ private final Schema schema;
+ private final VSRManager vsrManager;
+ private final long writerGeneration;
+
+ public ParquetWriter(String file, Schema schema, long writerGeneration, ArrowBufferPool arrowBufferPool) {
+ this.file = file;
+ this.schema = schema;
+ this.vsrManager = new VSRManager(file, schema, arrowBufferPool);
+ this.writerGeneration = writerGeneration;
+ }
+
+ @Override
+ public WriteResult addDoc(ParquetDocumentInput d) throws IOException {
+ return vsrManager.addToManagedVSR(d);
+ }
+
+ @Override
+ public FileInfos flush(FlushIn flushIn) throws IOException {
+ String fileName = vsrManager.flush(flushIn);
+ // no data flushed
+ if (fileName == null) {
+ return FileInfos.empty();
+ }
+ Path file = Path.of(fileName);
+ WriterFileSet writerFileSet = WriterFileSet.builder()
+ .directory(file.getParent())
+ .writerGeneration(writerGeneration)
+ .addFile(file.getFileName().toString())
+ .build();
+ return FileInfos.builder().putWriterFileSet(PARQUET_DATA_FORMAT, writerFileSet).build();
+ }
+
+ @Override
+ public void sync() throws IOException {
+
+ }
+
+ @Override
+ public void close() {
+ vsrManager.close();
+ }
+
+ @Override
+ public ParquetDocumentInput newDocumentInput() {
+ try {
+ vsrManager.maybeRotateActiveVSR();
+ } catch (IOException e) {
+ logger.error("Failed to handle VSR rotation: {}", e.getMessage(), e);
+ }
+
+ // Get a new ManagedVSR from VSRManager for this document input
+ return new ParquetDocumentInput(vsrManager.getActiveManagedVSR());
+ }
+}
diff --git a/modules/parquet-data-format/src/main/resources/META-INF/services/org.opensearch.vectorized.execution.search.spi.DataSourceCodec b/modules/parquet-data-format/src/main/resources/META-INF/services/org.opensearch.vectorized.execution.search.spi.DataSourceCodec
new file mode 100644
index 0000000000000..7d1e56cc25536
--- /dev/null
+++ b/modules/parquet-data-format/src/main/resources/META-INF/services/org.opensearch.vectorized.execution.search.spi.DataSourceCodec
@@ -0,0 +1 @@
+com.parquet.parquetdataformat.engine.read.ParquetDataSourceCodec
diff --git a/modules/parquet-data-format/src/main/rust/Cargo.toml b/modules/parquet-data-format/src/main/rust/Cargo.toml
new file mode 100644
index 0000000000000..9591d62fd3d26
--- /dev/null
+++ b/modules/parquet-data-format/src/main/rust/Cargo.toml
@@ -0,0 +1,63 @@
+[package]
+name = "rust"
+version = "0.1.0"
+edition = "2024"
+
+[lib]
+name = "parquet_dataformat_jni"
+crate-type = ["cdylib", "lib"]
+
+[dependencies]
+
+# DataFusion dependencies
+datafusion = "49.0.0"
+datafusion-substrait = "49.0.0"
+arrow = { version = "54.0.0", features = ["ffi"] }
+
+arrow-array = "54.0.0"
+arrow-schema = "54.0.0"
+arrow-buffer = "54.0.0"
+
+# JNI dependencies
+jni = "0.21"
+
+# Async runtime
+tokio = { version = "1.0", features = ["full"] }
+futures = "0.3"
+futures-util = "0.3"
+
+# Serialization
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+# Error handling
+anyhow = "1.0"
+thiserror = "1.0"
+
+# Logging
+log = "0.4"
+
+# Parquet support
+parquet = "54.0.0"
+
+# Object store for file access
+object_store = "0.11"
+url = "2.0"
+
+# Substrait support
+substrait = "0.47"
+prost = "0.13"
+
+# Temporary directory support
+tempfile = "3.0"
+
+#jni = "0.21.1"
+#arrow = { version = "53.0.0", features = ["ffi"] }
+#parquet = "53.0.0"
+lazy_static = "1.4.0"
+dashmap = "7.0.0-rc2"
+chrono = "0.4"
+
+
+[build-dependencies]
+cbindgen = "0.27"
diff --git a/modules/parquet-data-format/src/main/rust/src/context.rs b/modules/parquet-data-format/src/main/rust/src/context.rs
new file mode 100644
index 0000000000000..022912ed84c48
--- /dev/null
+++ b/modules/parquet-data-format/src/main/rust/src/context.rs
@@ -0,0 +1,70 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+use datafusion::prelude::*;
+use datafusion::execution::context::SessionContext;
+use std::collections::HashMap;
+use std::sync::Arc;
+use anyhow::Result;
+
+/// Manages DataFusion session contexts
+pub struct SessionContextManager {
+ contexts: HashMap<*mut SessionContext, Arc>,
+ next_runtime_id: u64,
+}
+
+impl SessionContextManager {
+ pub fn new() -> Self {
+ Self {
+ contexts: HashMap::new(),
+ next_runtime_id: 1,
+ }
+ }
+
+ pub async fn register_directory(
+ &mut self,
+ table_name: &str,
+ directory_path: &str,
+ options: HashMap,
+ ) -> Result {
+ // Placeholder implementation - would register parquet directory as table
+ log::info!("Registering directory: {} at path: {} with options: {:?}",
+ table_name, directory_path, options);
+
+ let runtime_id = self.next_runtime_id;
+ self.next_runtime_id += 1;
+ Ok(runtime_id)
+ }
+
+ pub async fn create_session_context(
+ &mut self,
+ config: HashMap,
+ ) -> Result<*mut SessionContext> {
+ // Create actual DataFusion session context
+ let mut session_config = SessionConfig::new();
+
+ // Apply configuration options
+ if let Some(batch_size) = config.get("batch_size") {
+ if let Ok(size) = batch_size.parse::() {
+ session_config = session_config.with_batch_size(size);
+ }
+ }
+
+ let ctx = Arc::new(SessionContext::new_with_config(session_config));
+ let ctx_ptr = Arc::as_ptr(&ctx) as *mut SessionContext;
+
+ self.contexts.insert(ctx_ptr, ctx);
+
+ Ok(ctx_ptr)
+ }
+
+ pub async fn close_session_context(&mut self, ctx_ptr: *mut SessionContext) -> Result<()> {
+ self.contexts.remove(&ctx_ptr);
+ Ok(())
+ }
+
+ pub fn get_context(&self, ctx_ptr: *mut SessionContext) -> Option<&Arc> {
+ self.contexts.get(&ctx_ptr)
+ }
+}
diff --git a/modules/parquet-data-format/src/main/rust/src/lib.rs b/modules/parquet-data-format/src/main/rust/src/lib.rs
new file mode 100644
index 0000000000000..a6381876acfdf
--- /dev/null
+++ b/modules/parquet-data-format/src/main/rust/src/lib.rs
@@ -0,0 +1,252 @@
+use jni::objects::{JClass, JString};
+use jni::sys::{jint, jlong};
+use jni::{JNIEnv, JavaVM};
+use dashmap::DashMap;
+use arrow::record_batch::RecordBatch;
+use parquet::arrow::ArrowWriter;
+use std::fs::File;
+use std::sync::{Arc, Mutex};
+use lazy_static::lazy_static;
+use arrow::ffi::{FFI_ArrowSchema, FFI_ArrowArray};
+use parquet::basic::{Compression, ZstdLevel};
+use parquet::file::properties::WriterProperties;
+
+pub mod logger;
+pub mod parquet_merge;
+pub use parquet_merge::*;
+
+lazy_static! {
+ static ref WRITER_MANAGER: DashMap>>> = DashMap::new();
+ static ref FILE_MANAGER: DashMap = DashMap::new();
+}
+
+struct NativeParquetWriter;
+
+impl NativeParquetWriter {
+
+ fn create_writer(filename: String, schema_address: i64) -> Result<(), Box> {
+ logger::log_info(&format!("[RUST] create_writer called for file: {}, schema_address: {}", filename, schema_address));
+
+ let arrow_schema = unsafe { FFI_ArrowSchema::from_raw(schema_address as *mut _) };
+ let schema = Arc::new(arrow::datatypes::Schema::try_from(&arrow_schema)?);
+
+ logger::log_info(&format!("[RUST] Schema created with {} fields", schema.fields().len()));
+
+ for (i, field) in schema.fields().iter().enumerate() {
+ logger::log_debug(&format!("[RUST] Field {}: {} ({})", i, field.name(), field.data_type()));
+ }
+
+ let file = File::create(&filename)?;
+ let file_clone = file.try_clone()?;
+ FILE_MANAGER.insert(filename.clone(), file_clone);
+ let props = WriterProperties::builder()
+ .set_compression(Compression::ZSTD(ZstdLevel::try_new(3).unwrap()))
+ .build();
+ let writer = ArrowWriter::try_new(file, schema, Some(props))?;
+ WRITER_MANAGER.insert(filename, Arc::new(Mutex::new(writer)));
+ Ok(())
+ }
+
+ fn write_data(filename: String, array_address: i64, schema_address: i64) -> Result<(), Box> {
+ logger::log_info(&format!("[RUST] write_data called for file: {}, array_address: {}, schema_address: {}", filename, array_address, schema_address));
+
+ unsafe {
+ let arrow_schema = FFI_ArrowSchema::from_raw(schema_address as *mut _);
+ let arrow_array = FFI_ArrowArray::from_raw(array_address as *mut _);
+
+ match arrow::ffi::from_ffi(arrow_array, &arrow_schema) {
+ Ok(array_data) => {
+ logger::log_debug(&format!("[RUST] Successfully imported array_data, length: {}", array_data.len()));
+
+ let array: Arc = arrow::array::make_array(array_data);
+ logger::log_debug(&format!("[RUST] Array type: {:?}, length: {}", array.data_type(), array.len()));
+
+ if let Some(struct_array) = array.as_any().downcast_ref::() {
+ logger::log_debug(&format!("[RUST] Successfully cast to StructArray with {} columns", struct_array.num_columns()));
+
+ let schema = Arc::new(arrow::datatypes::Schema::new(
+ struct_array.fields().clone()
+ ));
+
+ let record_batch = RecordBatch::try_new(
+ schema.clone(),
+ struct_array.columns().to_vec(),
+ )?;
+
+ logger::log_info(&format!("[RUST] Created RecordBatch with {} rows and {} columns", record_batch.num_rows(), record_batch.num_columns()));
+
+ if let Some(writer_arc) = WRITER_MANAGER.get(&filename) {
+ logger::log_debug("[RUST] Writing RecordBatch to file");
+ let mut writer = writer_arc.lock().unwrap();
+ writer.write(&record_batch)?;
+ logger::log_info("[RUST] Successfully wrote RecordBatch");
+ } else {
+ logger::log_error(&format!("[RUST] ERROR: No writer found for file: {}", filename));
+ }
+ Ok(())
+ } else {
+ logger::log_error(&format!("[RUST] ERROR: Array is not a StructArray, type: {:?}", array.data_type()));
+ Err("Expected struct array from VectorSchemaRoot".into())
+ }
+ }
+ Err(e) => {
+ logger::log_error(&format!("[RUST] ERROR: Failed to import from FFI: {:?}", e));
+ Err(e.into())
+ }
+ }
+ }
+ }
+
+ fn close_writer(filename: String) -> Result<(), Box> {
+ logger::log_info(&format!("[RUST] close_writer called for file: {}", filename));
+
+ if let Some((_, writer_arc)) = WRITER_MANAGER.remove(&filename) {
+ match Arc::try_unwrap(writer_arc) {
+ Ok(mutex) => {
+ let mut writer = mutex.into_inner().unwrap();
+ match writer.close() {
+ Ok(_) => {
+ logger::log_info(&format!("[RUST] Successfully closed writer for file: {}", filename));
+ Ok(())
+ }
+ Err(e) => {
+ logger::log_error(&format!("[RUST] ERROR: Failed to close writer for file: {}", filename));
+ Err(e.into())
+ }
+ }
+ }
+ Err(_) => {
+ logger::log_error(&format!("[RUST] ERROR: Writer still in use for file: {}", filename));
+ Err("Writer still in use".into())
+ }
+ }
+ } else {
+ Ok(())
+ }
+ }
+
+ fn flush_to_disk(filename: String) -> Result<(), Box> {
+ logger::log_info(&format!("[RUST] fsync_file called for file: {}", filename));
+
+ if let Some(file) = FILE_MANAGER.get_mut(&filename) {
+ match file.sync_all() {
+ Ok(_) => {
+ logger::log_info(&format!("[RUST] Successfully fsynced file: {}", filename));
+ Ok(())
+ }
+ Err(e) => {
+ logger::log_error(&format!("[RUST] ERROR: Failed to fsync file: {}", filename));
+ Err(e.into())
+ }
+ }
+ } else {
+ logger::log_error(&format!("[RUST] ERROR: File not found for fsync: {}", filename));
+ Err("File not found".into())
+ }
+ }
+
+ fn get_filtered_writer_memory_usage(path_prefix: String) -> Result> {
+ logger::log_debug(&format!("[RUST] get_filtered_writer_memory_usage called with prefix: {}", path_prefix));
+
+ let mut total_memory = 0;
+ let mut writer_count = 0;
+
+ for entry in WRITER_MANAGER.iter() {
+ let filename = entry.key();
+ let writer_arc = entry.value();
+
+ // Filter writers by path prefix
+ if filename.starts_with(&path_prefix) {
+ if let Ok(writer) = writer_arc.lock() {
+ let memory_usage = writer.memory_size();
+ total_memory += memory_usage;
+ writer_count += 1;
+
+ logger::log_debug(&format!("[RUST] Filtered Writer {}: {} bytes", filename, memory_usage));
+ }
+ }
+ }
+
+ logger::log_debug(&format!("[RUST] Total memory usage across {} filtered ArrowWriters (prefix: {}): {} bytes", writer_count, path_prefix, total_memory));
+
+ Ok(total_memory)
+ }
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_com_parquet_parquetdataformat_bridge_RustBridge_initLogger(
+ env: JNIEnv,
+ _class: JClass,
+) {
+ if let Ok(jvm) = env.get_java_vm() {
+ logger::init_logger(jvm);
+ }
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_com_parquet_parquetdataformat_bridge_RustBridge_createWriter(
+ mut env: JNIEnv,
+ _class: JClass,
+ file: JString,
+ schema_address: jlong
+) -> jint {
+ let filename: String = env.get_string(&file).expect("Couldn't get java string!").into();
+ match NativeParquetWriter::create_writer(filename, schema_address as i64) {
+ Ok(_) => 0,
+ Err(_) => -1,
+ }
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_com_parquet_parquetdataformat_bridge_RustBridge_write(
+ mut env: JNIEnv,
+ _class: JClass,
+ file: JString,
+ array_address: jlong,
+ schema_address: jlong
+) -> jint {
+ let filename: String = env.get_string(&file).expect("Couldn't get java string!").into();
+ match NativeParquetWriter::write_data(filename, array_address as i64, schema_address as i64) {
+ Ok(_) => 0,
+ Err(_) => -1,
+ }
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_com_parquet_parquetdataformat_bridge_RustBridge_closeWriter(
+ mut env: JNIEnv,
+ _class: JClass,
+ file: JString
+) -> jint {
+ let filename: String = env.get_string(&file).expect("Couldn't get java string!").into();
+ match NativeParquetWriter::close_writer(filename) {
+ Ok(_) => 0,
+ Err(_) => -1,
+ }
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_com_parquet_parquetdataformat_bridge_RustBridge_flushToDisk(
+ mut env: JNIEnv,
+ _class: JClass,
+ file: JString
+) -> jint {
+ let filename: String = env.get_string(&file).expect("Couldn't get java string!").into();
+ match NativeParquetWriter::flush_to_disk(filename) {
+ Ok(_) => 0,
+ Err(_) => -1,
+ }
+}
+
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_com_parquet_parquetdataformat_bridge_RustBridge_getFilteredNativeBytesUsed(
+ mut env: JNIEnv,
+ _class: JClass,
+ path_prefix: JString
+) -> jlong {
+ let prefix: String = env.get_string(&path_prefix).expect("Couldn't get java string!").into();
+ match NativeParquetWriter::get_filtered_writer_memory_usage(prefix) {
+ Ok(memory_usage) => memory_usage as jlong,
+ Err(_) => 0,
+ }
+}
diff --git a/modules/parquet-data-format/src/main/rust/src/logger.rs b/modules/parquet-data-format/src/main/rust/src/logger.rs
new file mode 100644
index 0000000000000..313cae61806a1
--- /dev/null
+++ b/modules/parquet-data-format/src/main/rust/src/logger.rs
@@ -0,0 +1,103 @@
+use jni::{JNIEnv, JavaVM};
+use std::sync::OnceLock;
+
+static JAVA_VM: OnceLock = OnceLock::new();
+
+/// Initialize the logger with the JVM instance
+pub fn init_logger(jvm: JavaVM) {
+ JAVA_VM.set(jvm).ok();
+}
+
+/// Log an info message through JNI callback to Java
+pub fn log_info(message: &str) {
+ if let Some(jvm) = JAVA_VM.get() {
+ if let Ok(mut env) = jvm.attach_current_thread() {
+ call_java_logger(&mut env, "logInfo", message);
+ }
+ }
+}
+
+/// Log a warning message through JNI callback to Java
+pub fn log_warn(message: &str) {
+ if let Some(jvm) = JAVA_VM.get() {
+ if let Ok(mut env) = jvm.attach_current_thread() {
+ call_java_logger(&mut env, "logWarn", message);
+ }
+ }
+}
+
+/// Log an error message through JNI callback to Java
+pub fn log_error(message: &str) {
+ if let Some(jvm) = JAVA_VM.get() {
+ if let Ok(mut env) = jvm.attach_current_thread() {
+ call_java_logger(&mut env, "logError", message);
+ }
+ }
+}
+
+/// Log a debug message through JNI callback to Java
+pub fn log_debug(message: &str) {
+ if let Some(jvm) = JAVA_VM.get() {
+ if let Ok(mut env) = jvm.attach_current_thread() {
+ call_java_logger(&mut env, "logDebug", message);
+ }
+ }
+}
+
+/// Internal function to call the Java logger method
+fn call_java_logger(env: &mut JNIEnv, method_name: &str, message: &str) {
+ let result = (|| -> Result<(), Box> {
+ // Find the RustLoggerBridge class
+ let class = env.find_class("com/parquet/parquetdataformat/bridge/RustLoggerBridge")?;
+
+ // Convert Rust string to Java string
+ let java_message = env.new_string(message)?;
+
+ // Call the static method
+ env.call_static_method(
+ class,
+ method_name,
+ "(Ljava/lang/String;)V",
+ &[(&java_message).into()],
+ )?;
+
+ Ok(())
+ })();
+
+ // If logging fails, fall back to println as last resort
+ if result.is_err() {
+ println!("[RUST_LOG_FALLBACK] {}: {}", method_name, message);
+ }
+}
+
+/// Macro for easy info logging
+#[macro_export]
+macro_rules! rust_log_info {
+ ($($arg:tt)*) => {
+ $crate::logger::log_info(&format!($($arg)*))
+ };
+}
+
+/// Macro for easy warning logging
+#[macro_export]
+macro_rules! rust_log_warn {
+ ($($arg:tt)*) => {
+ $crate::logger::log_warn(&format!($($arg)*))
+ };
+}
+
+/// Macro for easy error logging
+#[macro_export]
+macro_rules! rust_log_error {
+ ($($arg:tt)*) => {
+ $crate::logger::log_error(&format!($($arg)*))
+ };
+}
+
+/// Macro for easy debug logging
+#[macro_export]
+macro_rules! rust_log_debug {
+ ($($arg:tt)*) => {
+ $crate::logger::log_debug(&format!($($arg)*))
+ };
+}
diff --git a/modules/parquet-data-format/src/main/rust/src/parquet_merge.rs b/modules/parquet-data-format/src/main/rust/src/parquet_merge.rs
new file mode 100644
index 0000000000000..b4f368bd49375
--- /dev/null
+++ b/modules/parquet-data-format/src/main/rust/src/parquet_merge.rs
@@ -0,0 +1,288 @@
+use jni::JNIEnv;
+use jni::objects::{JClass, JObject, JString};
+use jni::sys::jint;
+use std::fs::File;
+use std::error::Error;
+use std::any::Any;
+use std::sync::Arc;
+use std::panic::AssertUnwindSafe;
+use parquet::basic::Compression;
+use parquet::file::properties::WriterProperties;
+use arrow::array::{Int64Array, ArrayRef};
+use arrow::datatypes::SchemaRef;
+use arrow::record_batch::RecordBatch;
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+use parquet::arrow::arrow_writer::ArrowWriter;
+
+use crate::{rust_log_info, rust_log_error};
+
+// Constants
+const READER_BATCH_SIZE: usize = 8192;
+const WRITER_BATCH_SIZE: usize = 8192;
+const ROW_ID_COLUMN_NAME: &str = "___row_id";
+
+// Custom error types
+#[derive(Debug)]
+pub enum ParquetMergeError {
+ EmptyInput,
+ InvalidFile(String),
+ SchemaReadError(String),
+ WriterCreationError(String),
+ BatchProcessingError(String),
+}
+
+impl std::fmt::Display for ParquetMergeError {
+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+ match self {
+ ParquetMergeError::EmptyInput => write!(f, "No input files provided"),
+ ParquetMergeError::InvalidFile(path) => write!(f, "Invalid file: {}", path),
+ ParquetMergeError::SchemaReadError(msg) => write!(f, "Schema read error: {}", msg),
+ ParquetMergeError::WriterCreationError(msg) => write!(f, "Writer creation error: {}", msg),
+ ParquetMergeError::BatchProcessingError(msg) => write!(f, "Batch processing error: {}", msg),
+ }
+ }
+}
+
+impl Error for ParquetMergeError {}
+
+// Statistics tracking
+struct ProcessingStats {
+ files_processed: usize,
+ total_rows: usize,
+ total_batches: usize,
+}
+
+// JNI Entry Point
+#[unsafe(no_mangle)]
+pub extern "system" fn Java_com_parquet_parquetdataformat_bridge_RustBridge_mergeParquetFilesInRust(
+ mut env: JNIEnv,
+ _class: JClass,
+ input_files: JObject,
+ output_file: JString,
+) -> jint {
+ let result = catch_unwind(|| {
+ let input_files_vec = convert_java_list_to_vec(&mut env, input_files)
+ .map_err(|e| format!("Failed to convert Java list: {}", e))?;
+
+ let output_path: String = env
+ .get_string(&output_file)
+ .map_err(|e| format!("Failed to get output file string: {}", e))?
+ .into();
+
+ log_info(&format!("Starting merge of {} files to {}", input_files_vec.len(), output_path));
+
+ process_parquet_files(&input_files_vec, &output_path)?;
+
+ log_info("Merge completed successfully");
+ Ok(())
+ });
+
+ match result {
+ Ok(Ok(_)) => 0,
+ Ok(Err(e)) => {
+ let error_msg = format!("Error processing Parquet files: {}", e);
+ log_error(&error_msg);
+ let _ = env.throw_new("java/lang/RuntimeException", &error_msg);
+ -1
+ }
+ Err(e) => {
+ let error_msg = format!("Rust panic occurred: {:?}", e);
+ log_error(&error_msg);
+ let _ = env.throw_new("java/lang/RuntimeException", &error_msg);
+ -1
+ }
+ }
+}
+
+// Main processing function
+pub fn process_parquet_files(input_files: &[String], output_path: &str) -> Result<(), Box> {
+ // Validate input
+ validate_input(input_files)?;
+
+ // Read schema from first file
+ let schema = read_schema_from_file(&input_files[0])?;
+ log_info(&format!("Schema read successfully: {:?}", schema));
+
+ // Create writer
+ let mut writer = create_writer(output_path, schema.clone())?;
+
+ // Process files
+ let stats = process_files(input_files, &schema, &mut writer)?;
+
+ // Close writer
+ writer.close()
+ .map_err(|e| ParquetMergeError::WriterCreationError(format!("Failed to close writer: {}", e)))?;
+
+ log_info(&format!(
+ "Processing complete: {} files, {} rows, {} batches",
+ stats.files_processed, stats.total_rows, stats.total_batches
+ ));
+
+ Ok(())
+}
+
+// Validation functions
+fn validate_input(input_files: &[String]) -> Result<(), Box> {
+ if input_files.is_empty() {
+ return Err(Box::new(ParquetMergeError::EmptyInput));
+ }
+
+ for path in input_files {
+ if !std::path::Path::new(path).exists() {
+ return Err(Box::new(ParquetMergeError::InvalidFile(path.clone())));
+ }
+ }
+
+ Ok(())
+}
+
+// Schema reading
+fn read_schema_from_file(file_path: &str) -> Result> {
+ let file = File::open(file_path)
+ .map_err(|e| ParquetMergeError::InvalidFile(format!("{}: {}", file_path, e)))?;
+
+ let builder = ParquetRecordBatchReaderBuilder::try_new(file)
+ .map_err(|e| ParquetMergeError::SchemaReadError(format!("Failed to read schema: {}", e)))?;
+
+ Ok(builder.schema().clone())
+}
+
+// Writer creation
+fn create_writer(output_path: &str, schema: SchemaRef) -> Result, Box> {
+ let props = WriterProperties::builder()
+ .set_write_batch_size(WRITER_BATCH_SIZE)
+ .set_compression(Compression::ZSTD(Default::default()))
+ .build();
+
+ let out_file = File::create(output_path)
+ .map_err(|e| ParquetMergeError::WriterCreationError(format!("Failed to create output file: {}", e)))?;
+
+ ArrowWriter::try_new(out_file, schema, Some(props))
+ .map_err(|e| ParquetMergeError::WriterCreationError(format!("Failed to create writer: {}", e)).into())
+}
+
+// File processing
+fn process_files(
+ input_files: &[String],
+ schema: &SchemaRef,
+ writer: &mut ArrowWriter,
+) -> Result> {
+ let mut current_row_id: i64 = 0;
+ let mut stats = ProcessingStats {
+ files_processed: 0,
+ total_rows: 0,
+ total_batches: 0,
+ };
+
+ for path in input_files {
+ log_info(&format!("Processing file: {}", path));
+
+ let file = File::open(path)
+ .map_err(|e| ParquetMergeError::InvalidFile(format!("{}: {}", path, e)))?;
+
+ let reader = ParquetRecordBatchReaderBuilder::try_new(file)
+ .map_err(|e| ParquetMergeError::BatchProcessingError(format!("Failed to create reader: {}", e)))?
+ .with_batch_size(READER_BATCH_SIZE)
+ .build()
+ .map_err(|e| ParquetMergeError::BatchProcessingError(format!("Failed to build reader: {}", e)))?;
+
+ let mut file_rows = 0;
+ let mut file_batches = 0;
+
+ for batch_result in reader {
+ let original_batch = batch_result
+ .map_err(|e| ParquetMergeError::BatchProcessingError(format!("Failed to read batch: {}", e)))?;
+
+ let batch_rows = original_batch.num_rows();
+
+ let new_batch = update_row_ids(&original_batch, current_row_id, schema)?;
+
+ writer.write(&new_batch)
+ .map_err(|e| ParquetMergeError::BatchProcessingError(format!("Failed to write batch: {}", e)))?;
+
+ current_row_id += batch_rows as i64;
+ file_rows += batch_rows;
+ file_batches += 1;
+ }
+
+ stats.files_processed += 1;
+ stats.total_rows += file_rows;
+ stats.total_batches += file_batches;
+
+ log_info(&format!("File processed: {} rows, {} batches", file_rows, file_batches));
+ }
+
+ Ok(stats)
+}
+
+// Row ID update logic
+pub fn update_row_ids(
+ original_batch: &RecordBatch,
+ start_id: i64,
+ schema: &SchemaRef,
+) -> Result> {
+ let row_count = original_batch.num_rows();
+
+ // Create new row IDs
+ let row_ids: Int64Array = (start_id..start_id + row_count as i64)
+ .collect::>()
+ .into();
+
+ // Build new columns array
+ let mut columns: Vec = Vec::with_capacity(original_batch.num_columns());
+
+ for (i, column) in original_batch.columns().iter().enumerate() {
+ let field_name = schema.field(i).name();
+ if field_name == ROW_ID_COLUMN_NAME {
+ columns.push(Arc::new(row_ids.clone()));
+ } else {
+ columns.push(column.clone());
+ }
+ }
+
+ RecordBatch::try_new(schema.clone(), columns)
+ .map_err(|e| ParquetMergeError::BatchProcessingError(format!("Failed to create batch: {}", e)).into())
+}
+
+// JNI helper functions
+fn convert_java_list_to_vec(env: &mut JNIEnv, list: JObject) -> Result, Box> {
+ let iterator = env.call_method(&list, "iterator", "()Ljava/util/Iterator;", &[])?
+ .l()?;
+
+ let mut result = Vec::new();
+ while env.call_method(&iterator, "hasNext", "()Z", &[])?.z()? {
+ let element = env.call_method(&iterator, "next", "()Ljava/lang/Object;", &[])?
+ .l()?;
+ let path_string = env.call_method(&element, "toString", "()Ljava/lang/String;", &[])?
+ .l()?;
+ let jstring = JString::from(path_string);
+ let string = env.get_string(&jstring)?;
+ result.push(string.to_str()?.to_string());
+ }
+
+ Ok(result)
+}
+
+fn catch_unwind Result<(), Box>>(
+ f: F
+) -> Result>, Box> {
+ std::panic::catch_unwind(AssertUnwindSafe(f))
+}
+
+// Logging functions
+fn log_info(message: &str) {
+ rust_log_info!("{}", message);
+}
+
+fn log_error(message: &str) {
+ rust_log_error!("{}", message);
+}
+
+// Close function
+// #[no_mangle]
+// pub extern "system" fn Java_org_opensearch_arrow_bridge_ArrowRustBridge_close(
+// _env: JNIEnv,
+// _class: JClass,
+// ) {
+// log_info("Closing ArrowRustBridge");
+// }
diff --git a/modules/parquet-data-format/src/main/rust/tests/parquet_merge_tests.rs b/modules/parquet-data-format/src/main/rust/tests/parquet_merge_tests.rs
new file mode 100644
index 0000000000000..46ac89421f6d0
--- /dev/null
+++ b/modules/parquet-data-format/src/main/rust/tests/parquet_merge_tests.rs
@@ -0,0 +1,105 @@
+use parquet_dataformat_jni::process_parquet_files;
+use arrow::array::{Int64Array, StringArray};
+use arrow::record_batch::RecordBatch;
+use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+use std::fs::File;
+use std::path::PathBuf;
+
+/// Helper to get test file from resources
+fn test_file(name: &str) -> String {
+ let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+ path.push("../../test/resources/parquetTestFiles");
+ path.push(name);
+ path.to_string_lossy().to_string()
+}
+
+/// Helper to read a Parquet file into record batches
+fn read_batches(path: &str) -> Vec {
+ let file = File::open(path).unwrap();
+ let reader = ParquetRecordBatchReaderBuilder::try_new(file)
+ .unwrap()
+ .build()
+ .unwrap();
+
+ reader.map(|r| r.unwrap()).collect()
+}
+
+#[test]
+fn test_process_parquet_files_empty_input() {
+ let output_path = std::env::temp_dir().join("test_output_empty.parquet");
+ let result = process_parquet_files(&[], output_path.to_str().unwrap());
+ assert!(result.is_err());
+ assert_eq!(result.unwrap_err().to_string(), "No input files provided");
+}
+
+#[test]
+fn test_process_parquet_files_nonexistent_file() {
+ let output_path = std::env::temp_dir().join("test_output_nonexistent.parquet");
+ let result = process_parquet_files(&["/nonexistent/file.parquet".to_string()], output_path.to_str().unwrap());
+ assert!(result.is_err());
+}
+
+#[test]
+fn test_process_single_file() {
+ let input_path = test_file("small_file1.parquet");
+ let output_path = std::env::temp_dir().join("test_output_single.parquet");
+
+ process_parquet_files(&[input_path.clone()], output_path.to_str().unwrap()).unwrap();
+
+ let batches = read_batches(output_path.to_str().unwrap());
+ assert!(!batches.is_empty());
+
+ // Verify ___row_id increments
+ for (batch_index, batch) in batches.iter().enumerate() {
+ let row_id_idx = batch.schema().fields().iter().position(|f| f.name() == "___row_id").unwrap();
+ let row_id_column = batch.column(row_id_idx).as_any().downcast_ref::().unwrap();
+
+ for i in 0..batch.num_rows() {
+ assert_eq!(row_id_column.value(i), (batch_index * batch.num_rows() + i) as i64);
+ }
+ }
+
+ std::fs::remove_file(output_path).ok();
+}
+
+#[test]
+fn test_merge_files_with_complete_data_verification() {
+ let input1 = test_file("small_file1.parquet");
+ let input2 = test_file("small_file2.parquet");
+ let output_path = std::env::temp_dir().join("test_output_complete_merge.parquet");
+
+ process_parquet_files(&[input1, input2], output_path.to_str().unwrap()).unwrap();
+
+ let batches = read_batches(output_path.to_str().unwrap());
+ let mut all_row_ids = vec![];
+ let mut all_names = vec![];
+ let mut all_ages = vec![];
+ let mut all_cities = vec![];
+
+ for batch in batches {
+ let schema = batch.schema();
+ let row_id_idx = schema.fields().iter().position(|f| f.name() == "___row_id").unwrap();
+ let name_idx = schema.fields().iter().position(|f| f.name() == "Name").unwrap();
+ let age_idx = schema.fields().iter().position(|f| f.name() == "Age").unwrap();
+ let city_idx = schema.fields().iter().position(|f| f.name() == "City").unwrap();
+
+ let row_id_col = batch.column(row_id_idx).as_any().downcast_ref::().unwrap();
+ let name_col = batch.column(name_idx).as_any().downcast_ref::().unwrap();
+ let age_col = batch.column(age_idx).as_any().downcast_ref::().unwrap();
+ let city_col = batch.column(city_idx).as_any().downcast_ref::().unwrap();
+
+ for i in 0..batch.num_rows() {
+ all_row_ids.push(row_id_col.value(i));
+ all_names.push(name_col.value(i).to_string());
+ all_ages.push(age_col.value(i));
+ all_cities.push(city_col.value(i).to_string());
+ }
+ }
+
+ assert_eq!(all_row_ids, vec![0, 1, 2, 3]);
+ assert_eq!(all_names, vec!["John", "Jane", "Shailesh", "Singh"]);
+ assert_eq!(all_ages, vec![30, 25, 23, 6]);
+ assert_eq!(all_cities, vec!["New York", "London", "Delhi", "Bangalore"]);
+
+ std::fs::remove_file(output_path).ok();
+}
diff --git a/modules/parquet-data-format/src/test/java/com/parquet/parquetdataformat/ParquetDataFormatPluginIT.java b/modules/parquet-data-format/src/test/java/com/parquet/parquetdataformat/ParquetDataFormatPluginIT.java
new file mode 100644
index 0000000000000..f4c123b8a96f4
--- /dev/null
+++ b/modules/parquet-data-format/src/test/java/com/parquet/parquetdataformat/ParquetDataFormatPluginIT.java
@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package com.parquet.parquetdataformat;
+
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import org.apache.hc.core5.http.ParseException;
+import org.apache.hc.core5.http.io.entity.EntityUtils;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.hamcrest.Matchers.containsString;
+
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE)
+public class ParquetDataFormatPluginIT extends OpenSearchIntegTestCase {
+
+ @Override
+ protected Collection> nodePlugins() {
+ return Collections.singletonList(ParquetDataFormatPlugin.class);
+ }
+
+ public void testPluginInstalled() throws IOException, ParseException {
+ Response response = getRestClient().performRequest(new Request("GET", "/_cat/plugins"));
+ String body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+
+ logger.info("response body: {}", body);
+ assertThat(body, containsString("parquet"));
+ }
+}
diff --git a/modules/parquet-data-format/src/test/java/com/parquet/parquetdataformat/ParquetDataFormatTests.java b/modules/parquet-data-format/src/test/java/com/parquet/parquetdataformat/ParquetDataFormatTests.java
new file mode 100644
index 0000000000000..b52466249d727
--- /dev/null
+++ b/modules/parquet-data-format/src/test/java/com/parquet/parquetdataformat/ParquetDataFormatTests.java
@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package com.parquet.parquetdataformat;
+
+import com.parquet.parquetdataformat.bridge.RustBridge;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+
+public class ParquetDataFormatTests extends OpenSearchTestCase {
+
+ public void testIngestion() throws IOException {
+ // Test only basic functionality without Arrow operations
+ try {
+ // Create plugin but don't call complex operations
+ ParquetDataFormatPlugin plugin = new ParquetDataFormatPlugin();
+ plugin.indexDataToParquetEngine();
+
+ } catch (UnsatisfiedLinkError e) {
+ fail("Native library not loaded properly: " + e.getMessage());
+ } catch (Exception e) {
+ fail("Test failed: " + e.getMessage());
+ }
+ }
+}
diff --git a/modules/parquet-data-format/src/test/resources/parquetTestFiles/large_file1.parquet b/modules/parquet-data-format/src/test/resources/parquetTestFiles/large_file1.parquet
new file mode 100644
index 0000000000000..bff2d5d2a8c1b
Binary files /dev/null and b/modules/parquet-data-format/src/test/resources/parquetTestFiles/large_file1.parquet differ
diff --git a/modules/parquet-data-format/src/test/resources/parquetTestFiles/large_file2.parquet b/modules/parquet-data-format/src/test/resources/parquetTestFiles/large_file2.parquet
new file mode 100644
index 0000000000000..fc5b4a5dcd45b
Binary files /dev/null and b/modules/parquet-data-format/src/test/resources/parquetTestFiles/large_file2.parquet differ
diff --git a/modules/parquet-data-format/src/test/resources/parquetTestFiles/small_file1.parquet b/modules/parquet-data-format/src/test/resources/parquetTestFiles/small_file1.parquet
new file mode 100644
index 0000000000000..1bbd388e1bc66
Binary files /dev/null and b/modules/parquet-data-format/src/test/resources/parquetTestFiles/small_file1.parquet differ
diff --git a/modules/parquet-data-format/src/test/resources/parquetTestFiles/small_file2.parquet b/modules/parquet-data-format/src/test/resources/parquetTestFiles/small_file2.parquet
new file mode 100644
index 0000000000000..b4d831b241ae5
Binary files /dev/null and b/modules/parquet-data-format/src/test/resources/parquetTestFiles/small_file2.parquet differ
diff --git a/modules/parquet-data-format/src/yamlRestTest/java/org.opensearch/parquetdataformat/ParquetDataFormatClientYamlTestSuiteIT.java b/modules/parquet-data-format/src/yamlRestTest/java/org.opensearch/parquetdataformat/ParquetDataFormatClientYamlTestSuiteIT.java
new file mode 100644
index 0000000000000..324c6ce3debd1
--- /dev/null
+++ b/modules/parquet-data-format/src/yamlRestTest/java/org.opensearch/parquetdataformat/ParquetDataFormatClientYamlTestSuiteIT.java
@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package org.opensearch.parquetdataformat;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.opensearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase;
+
+
+public class ParquetDataFormatClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase {
+
+ public ParquetDataFormatClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+ super(testCandidate);
+ }
+
+ @ParametersFactory
+ public static Iterable parameters() throws Exception {
+ return OpenSearchClientYamlSuiteTestCase.createParameters();
+ }
+}
diff --git a/modules/parquet-data-format/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml b/modules/parquet-data-format/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml
new file mode 100644
index 0000000000000..0399b16c51642
--- /dev/null
+++ b/modules/parquet-data-format/src/yamlRestTest/resources/rest-api-spec/test/10_basic.yml
@@ -0,0 +1,8 @@
+"Test that the plugin is loaded in OpenSearch":
+ - do:
+ cat.plugins:
+ local: true
+ h: component
+
+ - match:
+ $body: /^rename\n$/
diff --git a/plugins/engine-datafusion/.cargo/config.toml b/plugins/engine-datafusion/.cargo/config.toml
new file mode 100644
index 0000000000000..00b0f674f4037
--- /dev/null
+++ b/plugins/engine-datafusion/.cargo/config.toml
@@ -0,0 +1,2 @@
+[build]
+rustflags = ["--cfg", "tokio_unstable", "-C", "force-frame-pointers=yes", "-C", "symbol-mangling-version=v0"]
\ No newline at end of file
diff --git a/plugins/engine-datafusion/.gitignore b/plugins/engine-datafusion/.gitignore
new file mode 100644
index 0000000000000..cb03c41334f19
--- /dev/null
+++ b/plugins/engine-datafusion/.gitignore
@@ -0,0 +1,41 @@
+# Gradle
+.gradle/
+build/
+
+# Java
+*.class
+*.jar
+*.war
+*.ear
+hs_err_pid*
+
+# IDE
+.idea/
+*.iml
+*.ipr
+*.iws
+.vscode/
+.settings/
+.project
+.classpath
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Rust
+target/
+Cargo.lock
+
+jni/target/
+jni/Cargo.lock
+
+# Native libraries
+src/main/resources/native/
+
+# Logs
+*.log
+
+# Temporary files
+*.tmp
+*.temp
diff --git a/plugins/engine-datafusion/Cargo.toml b/plugins/engine-datafusion/Cargo.toml
new file mode 100644
index 0000000000000..c2015752579b7
--- /dev/null
+++ b/plugins/engine-datafusion/Cargo.toml
@@ -0,0 +1,86 @@
+[workspace]
+resolver = "2"
+members = [
+ "jni"
+]
+
+[workspace.dependencies]
+# DataFusion dependencies
+datafusion = "50.0.0"
+datafusion-expr = "50.0.0"
+datafusion-datasource = "50.0.0"
+arrow-json = "56.2"
+arrow = { version = "56.2", features = ["ffi", "ipc_compression"] }
+#arrow = "55.2.0"
+arrow-array = "56.2.0"
+arrow-schema = "56.2.0"
+arrow-buffer = "56.2.0"
+downcast-rs = "1.2"
+
+
+# JNI dependencies
+jni = "0.21"
+
+# Substrait support
+datafusion-substrait = "50.0.0"
+prost = "0.13"
+
+
+# Async runtime
+tokio = { version = "1.0", features = ["full"] }
+futures = "0.3"
+#tokio = { version = "1.0", features = ["rt", "rt-multi-thread", "macros"] }
+tokio-metrics = "0.4"
+
+# Serialization
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+# Error handling
+anyhow = "1.0"
+thiserror = "1.0"
+
+# Logging
+log = "0.4"
+# Parquet support
+parquet = "54.0.0"
+
+# Object store for file access
+object_store = "=0.12.3"
+url = "2.0"
+
+# Substrait support
+substrait = "0.47"
+
+# Temporary directory support
+tempfile = "3.0"
+chrono = "0.4.41"
+
+async-trait = "0.1.89"
+itertools = "0.14.0"
+rstest = "0.26.1"
+regex = "1.11.2"
+#
+#[build-dependencies]
+#cbindgen = "0.27"
+
+once_cell = "1.21.3"
+tokio-stream = "0.1.17"
+parking_lot = "0.12.5"
+tracing = "0.1.41"
+
+[profile.release]
+lto = true
+codegen-units = 1
+panic = "abort"
+incremental = true # Enable incremental compilation
+debug = "line-tables-only"
+strip = false
+
+[profile.dev]
+opt-level = 1 # Some optimization for reasonable performance
+lto = false # Disable LTO for faster builds
+codegen-units = 1 # More parallel compilation
+incremental = true # Enable incremental compilation
+debug = "full"
+strip = false
diff --git a/plugins/engine-datafusion/README.md b/plugins/engine-datafusion/README.md
new file mode 100644
index 0000000000000..5176d819558bf
--- /dev/null
+++ b/plugins/engine-datafusion/README.md
@@ -0,0 +1,133 @@
+
+## Prerequisites
+
+1. Checkout branch `substrait-plan` for OpenSearch SQL Plugin - https://github.com/vinaykpud/sql/tree/substrait-plan OR https://github.com/bharath-techie/sql/tree/substrait-plan
+
+2. Publish OpenSearch to maven local
+```
+./gradlew publishToMavenLocal
+```
+3. Publish SQL plugin to maven local
+```
+./gradlew publishToMavenLocal
+```
+4. Run opensearch with following parameters
+```
+ ./gradlew run --preserve-data -PremotePlugins="['org.opensearch.plugin:opensearch-job-scheduler:3.3.0.0-SNAPSHOT', 'org.opensearch.plugin:opensearch-sql-plugin:3.3.0.0-SNAPSHOT']" -PinstalledPlugins="['engine-datafusion']" --debug-jvm
+```
+
+
+## Steps to test indexing + search e2e
+
+TODO : need to remove hardcoded index name `index-7`
+
+1. Delete previous index if any
+```
+curl --location --request DELETE 'localhost:9200/index-7'
+```
+
+2. Create index with name : `index-7`
+```
+curl --location --request PUT 'http://localhost:9200/index-7' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+ "settings": {
+ "number_of_shards": 1,
+ "number_of_replicas": 0,
+ "refresh_interval": -1
+ },
+ "mappings": {
+ "properties": {
+ "id": {
+ "type": "keyword"
+ },
+ "name": {
+ "type": "keyword"
+ },
+ "age": {
+ "type": "integer"
+ },
+ "salary": {
+ "type": "long"
+ },
+ "score": {
+ "type": "double"
+ },
+ "active": {
+ "type": "boolean"
+ },
+ "created_date": {
+ "type": "date"
+ }
+ }
+ }
+}'
+```
+3. Index docs
+```
+curl --location --request POST 'http://localhost:9200/_bulk' \
+--header 'Content-Type: application/json' \
+--data-raw '{"index":{"_index":"index-7"}}
+{"id":"1","name":"Alice","age":30,"salary":75000,"score":95.5,"active":true,"created_date":"2024-01-15"}
+{"index":{"_index":"index-7"}}
+{"id":"2","name":"Bob","age":25,"salary":60000,"score":88.3,"active":true,"created_date":"2024-02-20"}
+{"index":{"_index":"index-7"}}
+{"id":"3","name":"Charlie","age":35,"salary":90000,"score":92.7,"active":false,"created_date":"2024-03-10"}
+{"index":{"_index":"index-7"}}
+{"id":"4","name":"Diana","age":28,"salary":70000,"score":89.1,"active":true,"created_date":"2024-04-05"}
+{"index":{"_index":"index-7"}}
+{"id":"5","name":"Bob","age":30,"salary":55000,"score":81.1,"active":true,"created_date":"2024-04-05"}
+{"index":{"_index":"index-7"}}
+{"id":"5","name":"Diana","age":35,"salary":65000,"score":71.1,"active":true,"created_date":"2024-02-05"}
+'
+'
+```
+4. Refresh the index
+```
+curl localhost:9200/index-7/_refresh
+```
+5. Query
+```
+curl --location --request POST 'http://localhost:9200/_plugins/_ppl' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+ "query": "source=index-7 | stats count(), min(age) as min, max(age) as max, avg(age) as avg"
+}'
+
+
+curl --location --request POST 'http://localhost:9200/_plugins/_ppl' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+ "query": "source=index-7 | stats count() as c by name | sort c"
+}'
+
+curl --location --request POST 'http://localhost:9200/_plugins/_ppl' --header 'Content-Type: application/json' --data-raw '{
+ "query": "source=index-7 | stats count(), sum(age) as c by name | sort c"
+}'
+
+curl --location --request POST 'http://localhost:9200/_plugins/_ppl' --header 'Content-Type: application/json' --data-raw '{
+ "query": "source=index-7 | where name = \"Bob\" | stats sum(age)"
+}'
+
+
+curl --location --request POST 'http://localhost:9200/_plugins/_ppl' --header 'Content-Type: application/json' --data-raw '{
+ "query": "source=index-7 | stats sum(age) as s by name | sort s"
+}'
+
+curl --location --request POST 'http://localhost:9200/_plugins/_ppl' --header 'Content-Type: application/json' --data-raw '{
+ "query": "source=index-7 | stats sum(age) as s by name | sort name"
+}'
+
+curl --location --request POST 'http://localhost:9200/_plugins/_ppl' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+ "query": "source=index-7 | stats count() as c by name"
+}'
+```
+
+## Steps to Run Unit Tests for Search Flow
+
+Run the following command in **OpenSearch** to execute tests
+```
+./gradlew :plugins:engine-datafusion:test --tests "org.opensearch.datafusion.DataFusionReaderManagerTests"
+```
diff --git a/plugins/engine-datafusion/build.gradle b/plugins/engine-datafusion/build.gradle
new file mode 100644
index 0000000000000..bb26ebd449612
--- /dev/null
+++ b/plugins/engine-datafusion/build.gradle
@@ -0,0 +1,224 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+apply plugin: 'java'
+apply plugin: 'idea'
+apply plugin: 'opensearch.internal-cluster-test'
+apply plugin: 'opensearch.yaml-rest-test'
+apply plugin: 'opensearch.pluginzip'
+
+def pluginName = 'engine-datafusion'
+def pluginDescription = 'OpenSearch plugin providing access to DataFusion via JNI'
+def projectPath = 'org.opensearch'
+def pathToPlugin = 'datafusion.DataFusionPlugin'
+def pluginClassName = 'DataFusionPlugin'
+def buildType = project.hasProperty('rustDebug') ? 'debug' : 'release'
+
+opensearchplugin {
+ name = pluginName
+ description = pluginDescription
+ classname = "${projectPath}.${pathToPlugin}"
+ licenseFile = rootProject.file('LICENSE.txt')
+ noticeFile = rootProject.file('NOTICE.txt')
+}
+
+
+dependencies {
+ api project(':libs:opensearch-vectorized-exec-spi')
+ implementation "org.apache.logging.log4j:log4j-api:${versions.log4j}"
+ implementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
+
+ // Bundle Jackson in the plugin JAR using 'api' like other OpenSearch plugins
+ api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+ api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+ api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+
+ // Apache Arrow dependencies for memory management
+ implementation "org.apache.arrow:arrow-memory-core:17.0.0"
+ implementation "org.apache.arrow:arrow-memory-unsafe:17.0.0"
+ implementation "org.apache.arrow:arrow-vector:17.0.0"
+ implementation "org.apache.arrow:arrow-c-data:17.0.0"
+ implementation "org.apache.arrow:arrow-format:17.0.0"
+ // SLF4J API for Arrow logging compatibility
+ implementation "org.slf4j:slf4j-api:${versions.slf4j}"
+ // CheckerFramework annotations required by Arrow 17.0.0
+ implementation "org.checkerframework:checker-qual:3.42.0"
+ // FlatBuffers dependency required by Arrow 17.0.0
+ implementation "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}"
+
+ testImplementation "junit:junit:${versions.junit}"
+ testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
+ testImplementation "org.mockito:mockito-core:${versions.mockito}"
+ testImplementation project(":modules:parquet-data-format")
+ // Add CSV plugin for testing
+ // testImplementation project(':plugins:dataformat-csv')
+}
+
+// Task to build the Rust JNI library
+task buildRustLibrary(type: Exec) {
+ description = 'Build the Rust JNI library using Cargo'
+ group = 'build'
+
+ workingDir file('jni')
+
+ // Determine the target directory and library name based on OS
+ def osName = System.getProperty('os.name').toLowerCase()
+ def libPrefix = osName.contains('windows') ? '' : 'lib'
+ def libExtension = osName.contains('windows') ? '.dll' : (osName.contains('mac') ? '.dylib' : '.so')
+
+ def targetDir = "target/${buildType}"
+
+ // Find cargo executable - try common locations
+ def cargoExecutable = 'cargo'
+ def possibleCargoPaths = [
+ System.getenv('HOME') + '/.cargo/bin/cargo',
+ '/usr/local/bin/cargo',
+ 'cargo'
+ ]
+
+ for (String path : possibleCargoPaths) {
+ if (new File(path).exists()) {
+ cargoExecutable = path
+ break
+ }
+ }
+
+ def cargoArgs = [cargoExecutable, 'build']
+ if (buildType == 'release') {
+ cargoArgs.add('--release')
+ }
+
+ if (osName.contains('windows')) {
+ commandLine cargoArgs
+ } else {
+ commandLine cargoArgs
+ }
+
+ // Set environment variables for cross-compilation if needed
+ environment 'CARGO_TARGET_DIR', file('jni/target').absolutePath
+
+ inputs.files fileTree('jni/src')
+ inputs.file 'jni/Cargo.toml'
+ outputs.files file("jni/${targetDir}/${libPrefix}opensearch_datafusion_jni${libExtension}")
+ System.out.println("Building Rust library in ${buildType} mode");
+}
+
+// Task to copy the native library to resources
+task copyNativeLibrary(type: Copy, dependsOn: buildRustLibrary) {
+ description = 'Copy the native library to Java resources'
+ group = 'build'
+
+ def osName = System.getProperty('os.name').toLowerCase()
+ def libPrefix = osName.contains('windows') ? '' : 'lib'
+ def libExtension = osName.contains('windows') ? '.dll' : (osName.contains('mac') ? '.dylib' : '.so')
+
+ from file("jni/target/${buildType}/${libPrefix}opensearch_datafusion_jni${libExtension}")
+ into file('src/main/resources/native')
+
+ // Rename to a standard name for Java to load
+ rename { filename ->
+ "libopensearch_datafusion_jni${libExtension}"
+ }
+
+ // Remove executable permissions to comply with OpenSearch file permission checks
+ filePermissions {
+ unix(0644)
+ }
+}
+
+// Ensure native library is built before Java compilation
+compileJava.dependsOn copyNativeLibrary
+
+// Ensure processResources depends on copyNativeLibrary
+processResources.dependsOn copyNativeLibrary
+sourcesJar.dependsOn copyNativeLibrary
+
+// Ensure filepermissions task depends on copyNativeLibrary
+tasks.named('filepermissions').configure {
+ dependsOn copyNativeLibrary
+}
+
+// Ensure sourcesJar depends on copyNativeLibrary since it includes resources
+sourcesJar.dependsOn copyNativeLibrary
+
+// Ensure filepermissions task depends on copyNativeLibrary
+tasks.named("filepermissions").configure {
+ dependsOn copyNativeLibrary
+}
+
+// Ensure forbiddenPatterns task depends on copyNativeLibrary
+tasks.named("forbiddenPatterns").configure {
+ dependsOn copyNativeLibrary
+ // Exclude native library files from pattern checking since they are binary
+ exclude '**/native/**'
+}
+
+// Ensure spotlessJava task has proper dependency ordering
+tasks.named("spotlessJava").configure {
+ mustRunAfter copyNativeLibrary
+}
+
+// Clean task should also clean Rust artifacts
+clean {
+ delete file('jni/target')
+ delete file('src/main/resources/native')
+}
+
+test {
+ // Set system property to help tests find the native library
+ jvmArgs += ["--add-opens", "java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED"]
+
+ systemProperty 'java.library.path', file('src/main/resources/native').absolutePath
+}
+
+yamlRestTest {
+ systemProperty 'tests.security.manager', 'false'
+ // Disable yamlRestTest since this plugin doesn't have REST API endpoints
+ enabled = false
+}
+
+tasks.named("dependencyLicenses").configure {
+ mapping from: /jackson-.*/, to: 'jackson'
+ mapping from: /arrow-.*/, to: 'arrow'
+ mapping from: /slf4j-.*/, to: 'slf4j-api'
+ mapping from: /checker-qual.*/, to: 'checker-qual'
+ mapping from: /flatbuffers-.*/, to: 'flatbuffers-java'
+}
+
+// Configure third party audit to handle Apache Arrow dependencies
+tasks.named('thirdPartyAudit').configure {
+ ignoreMissingClasses(
+ // Apache Commons Codec (missing dependency)
+ 'org.apache.commons.codec.binary.Hex'
+ )
+ ignoreViolations(
+ // Apache Arrow internal classes that use Unsafe operations
+ 'org.apache.arrow.memory.ArrowBuf',
+ 'org.apache.arrow.memory.unsafe.UnsafeAllocationManager',
+ 'org.apache.arrow.memory.util.ByteFunctionHelpers',
+ 'org.apache.arrow.memory.util.MemoryUtil',
+ 'org.apache.arrow.memory.util.MemoryUtil$1',
+ 'org.apache.arrow.memory.util.hash.MurmurHasher',
+ 'org.apache.arrow.memory.util.hash.SimpleHasher',
+ 'org.apache.arrow.vector.BaseFixedWidthVector',
+ 'org.apache.arrow.vector.BitVectorHelper',
+ 'org.apache.arrow.vector.Decimal256Vector',
+ 'org.apache.arrow.vector.DecimalVector',
+ 'org.apache.arrow.vector.util.DecimalUtility',
+ 'org.apache.arrow.vector.util.VectorAppender'
+ )
+}
+
+// Configure Javadoc to skip package documentation requirements ie package-info.java
+missingJavadoc {
+ javadocMissingIgnore = [
+ 'org.opensearch.datafusion',
+ 'org.opensearch.datafusion.action',
+ 'org.opensearch.datafusion.core'
+ ]
+}
diff --git a/plugins/engine-datafusion/jni/.cargo/config.toml b/plugins/engine-datafusion/jni/.cargo/config.toml
new file mode 100644
index 0000000000000..00b0f674f4037
--- /dev/null
+++ b/plugins/engine-datafusion/jni/.cargo/config.toml
@@ -0,0 +1,2 @@
+[build]
+rustflags = ["--cfg", "tokio_unstable", "-C", "force-frame-pointers=yes", "-C", "symbol-mangling-version=v0"]
\ No newline at end of file
diff --git a/plugins/engine-datafusion/jni/Cargo.toml b/plugins/engine-datafusion/jni/Cargo.toml
new file mode 100644
index 0000000000000..ce62c628a5013
--- /dev/null
+++ b/plugins/engine-datafusion/jni/Cargo.toml
@@ -0,0 +1,94 @@
+[package]
+name = "opensearch-datafusion-jni"
+version = "0.1.0"
+edition = "2021"
+description = "JNI bindings for DataFusion integration with OpenSearch"
+license = "Apache-2.0"
+
+[lib]
+name = "opensearch_datafusion_jni"
+crate-type = ["cdylib"]
+
+[dependencies]
+# DataFusion dependencies
+datafusion = { workspace = true }
+datafusion-expr = { workspace = true }
+datafusion-datasource = { workspace = true }
+arrow-json = { workspace = true }
+arrow = { workspace = true }
+#arrow = "55.2.0"
+arrow-array = { workspace = true }
+arrow-schema = { workspace = true }
+arrow-buffer = { workspace = true }
+
+
+# JNI dependencies
+jni = { workspace = true }
+
+# Substrait support
+datafusion-substrait = { workspace = true }
+prost = { workspace = true }
+
+
+# Async runtime
+tokio = { workspace = true }
+futures = { workspace = true }
+#tokio = { version = "1.0", features = ["rt", "rt-multi-thread", "macros"] }
+tokio-metrics = { workspace = true }
+
+# Serialization
+serde = { workspace = true }
+serde_json = { workspace = true }
+
+# Error handling
+anyhow = { workspace = true }
+thiserror = { workspace = true }
+
+# Logging
+log ={ workspace = true }
+# Parquet support
+parquet = { workspace = true }
+
+# System info
+num_cpus = "1.16"
+
+
+# Object store for file access
+object_store = { workspace = true }
+url = { workspace = true }
+
+# Substrait support
+substrait = { workspace = true }
+
+# Temporary directory support
+tempfile ={ workspace = true }
+chrono = { workspace = true }
+
+async-trait = { workspace = true }
+itertools = { workspace = true }
+rstest = { workspace = true }
+regex = { workspace = true }
+
+once_cell = { workspace = true }
+tokio-stream = { workspace = true }
+parking_lot = { workspace = true }
+tracing = { workspace = true }
+
+[build-dependencies]
+cbindgen = "0.27"
+
+[profile.release]
+lto = true
+codegen-units = 1
+panic = "abort"
+incremental = true
+debug = "line-tables-only" #TODO : remove this
+strip = false
+
+[profile.dev]
+opt-level = 1 # Some optimization for reasonable performance
+lto = false # Disable LTO for faster builds
+codegen-units = 16 # More parallel compilation
+incremental = true # Enable incremental compilation
+debug = "full"
+strip = false
diff --git a/plugins/engine-datafusion/jni/src/cache.rs b/plugins/engine-datafusion/jni/src/cache.rs
new file mode 100644
index 0000000000000..e1ca93ba6b34a
--- /dev/null
+++ b/plugins/engine-datafusion/jni/src/cache.rs
@@ -0,0 +1,186 @@
+use std::sync::{Arc, Mutex};
+use jni::JNIEnv;
+
+use datafusion::execution::cache::cache_manager::{FileMetadataCache};
+use datafusion::execution::cache::cache_unit::{DefaultFilesMetadataCache};
+use datafusion::execution::cache::CacheAccessor;
+use object_store::ObjectMeta;
+
+pub const ALL_CACHE_TYPES: &[&str] = &[CACHE_TYPE_METADATA, CACHE_TYPE_STATS];
+
+// Cache type constants
+pub const CACHE_TYPE_METADATA: &str = "METADATA";
+pub const CACHE_TYPE_STATS: &str = "STATISTICS";
+
+// Helper function to handle cache errors
+fn handle_cache_error(env: &mut JNIEnv, operation: &str, error: &str) {
+ let msg = format!("Cache {} failed: {}", operation, error);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("java/lang/DataFusionException", &msg);
+}
+
+// Helper function to log cache operations
+fn log_cache_error(operation: &str, error: &str) {
+ eprintln!("[CACHE ERROR] {} operation failed: {}", operation, error);
+}
+
+// Note: MutexFileMetadataCache wrapper has been removed as DefaultFilesMetadataCache
+// is already thread-safe with its own internal Mutex.
+// The double-locking was causing race conditions and crashes.
+
+// Note: create_cache function has been removed. Cache creation is now handled through CacheManagerConfig only.
+// metadata_cache_put, metadata_cache_remove, and metadata_cache_get functions have been moved to CustomCacheManager as internal methods
+
+// Wrapper to make Mutex implement FileMetadataCache
+pub struct MutexFileMetadataCache {
+ pub inner: Mutex,
+}
+
+impl MutexFileMetadataCache {
+ pub fn new(cache: DefaultFilesMetadataCache) -> Self {
+ Self {
+ inner: Mutex::new(cache),
+ }
+ }
+
+ pub fn clear(&self) {
+ if let Ok(mut cache) = self.inner.lock() {
+ cache.clear();
+ }
+ }
+
+ pub fn update_cache_limit(&self, new_limit: usize) {
+ if let Ok(mut cache) = self.inner.lock() {
+ cache.update_cache_limit(new_limit);
+ }
+ }
+
+ pub fn cache_limit(&self) -> usize {
+ if let Ok(cache) = self.inner.lock() {
+ cache.cache_limit()
+ } else {
+ 0
+ }
+ }
+}
+
+// Implement CacheAccessor which is required by FileMetadataCache
+impl CacheAccessor> for MutexFileMetadataCache {
+ type Extra = ObjectMeta;
+
+ fn get(&self, k: &ObjectMeta) -> Option> {
+ match self.inner.lock() {
+ Ok(cache) => cache.get(k),
+ Err(e) => {
+ log_cache_error("get", &e.to_string());
+ None
+ }
+ }
+ }
+
+ fn get_with_extra(&self, k: &ObjectMeta, extra: &Self::Extra) -> Option> {
+ match self.inner.lock() {
+ Ok(cache) => cache.get_with_extra(k, extra),
+ Err(e) => {
+ log_cache_error("get_with_extra", &e.to_string());
+ None
+ }
+ }
+ }
+
+ fn put(&self, k: &ObjectMeta, v: Arc) -> Option> {
+ match self.inner.lock() {
+ Ok(mut cache) => cache.put(k, v),
+ Err(e) => {
+ log_cache_error("put", &e.to_string());
+ None
+ }
+ }
+ }
+
+ fn put_with_extra(&self, k: &ObjectMeta, v: Arc, e: &Self::Extra) -> Option> {
+ match self.inner.lock() {
+ Ok(mut cache) => cache.put_with_extra(k, v, e),
+ Err(err) => {
+ log_cache_error("put_with_extra", &err.to_string());
+ None
+ }
+ }
+ }
+
+ fn remove(&mut self, k: &ObjectMeta) -> Option> {
+ match self.inner.lock() {
+ Ok(mut cache) => cache.remove(k),
+ Err(e) => {
+ log_cache_error("remove", &e.to_string());
+ None
+ }
+ }
+ }
+
+ fn contains_key(&self, k: &ObjectMeta) -> bool {
+ match self.inner.lock() {
+ Ok(cache) => cache.contains_key(k),
+ Err(e) => {
+ log_cache_error("contains_key", &e.to_string());
+ false
+ }
+ }
+ }
+
+ fn len(&self) -> usize {
+ match self.inner.lock() {
+ Ok(cache) => cache.len(),
+ Err(e) => {
+ log_cache_error("len", &e.to_string());
+ 0
+ }
+ }
+ }
+
+ fn clear(&self) {
+ match self.inner.lock() {
+ Ok(mut cache) => cache.clear(),
+ Err(e) => log_cache_error("clear", &e.to_string()),
+ }
+ }
+
+ fn name(&self) -> String {
+ match self.inner.lock() {
+ Ok(cache) => cache.name(),
+ Err(e) => {
+ log_cache_error("name", &e.to_string());
+ "cache_error".to_string()
+ }
+ }
+ }
+}
+
+impl FileMetadataCache for MutexFileMetadataCache {
+ fn cache_limit(&self) -> usize {
+ match self.inner.lock() {
+ Ok(cache) => cache.cache_limit(),
+ Err(e) => {
+ log_cache_error("cache_limit", &e.to_string());
+ 0
+ }
+ }
+ }
+
+ fn update_cache_limit(&self, limit: usize) {
+ match self.inner.lock() {
+ Ok(mut cache) => cache.update_cache_limit(limit),
+ Err(e) => log_cache_error("update_cache_limit", &e.to_string()),
+ }
+ }
+
+ fn list_entries(&self) -> std::collections::HashMap {
+ match self.inner.lock() {
+ Ok(cache) => cache.list_entries(),
+ Err(e) => {
+ log_cache_error("list_entries", &e.to_string());
+ std::collections::HashMap::new()
+ }
+ }
+ }
+}
diff --git a/plugins/engine-datafusion/jni/src/cache_jni.rs b/plugins/engine-datafusion/jni/src/cache_jni.rs
new file mode 100644
index 0000000000000..a3db9920fbdee
--- /dev/null
+++ b/plugins/engine-datafusion/jni/src/cache_jni.rs
@@ -0,0 +1,453 @@
+use jni::objects::{JClass, JObject, JObjectArray, JString};
+use jni::sys::{jlong, jstring};
+use jni::{JNIEnv};
+use crate::custom_cache_manager::CustomCacheManager;
+use crate::util::{parse_string_arr};
+use crate::cache;
+use crate::DataFusionRuntime;
+use datafusion::execution::cache::cache_unit::DefaultFilesMetadataCache;
+use std::sync::Arc;
+
+/// Create a CustomCacheManager instance
+#[no_mangle]
+pub extern "system" fn Java_org_opensearch_datafusion_jni_NativeBridge_createCustomCacheManager(
+ mut env: JNIEnv,
+ _class: JClass,
+) -> jlong {
+ let manager = CustomCacheManager::new();
+ Box::into_raw(Box::new(manager)) as jlong
+}
+
+/// Destroy a CustomCacheManager instance
+#[no_mangle]
+pub extern "system" fn Java_org_opensearch_datafusion_jni_NativeBridge_destroyCustomCacheManager(
+ mut env: JNIEnv,
+ _class: JClass,
+ cache_manager_ptr: jlong,
+) {
+ if cache_manager_ptr != 0 {
+ let _ = unsafe { Box::from_raw(cache_manager_ptr as *mut CustomCacheManager) };
+ println!("[CACHE INFO] CustomCacheManager destroyed");
+ }
+}
+
+/// Generic cache creation method that handles all cache types
+#[no_mangle]
+pub extern "system" fn Java_org_opensearch_datafusion_jni_NativeBridge_createCache(
+ mut env: JNIEnv,
+ _class: JClass,
+ cache_manager_ptr: jlong,
+ cache_type: JString,
+ size_limit: jlong,
+ eviction_type: JString,
+) -> jlong {
+ if cache_manager_ptr == 0 {
+ let _ = env.throw_new("java/lang/DataFusionException", "CustomCacheManager pointer is null");
+ return 0;
+ }
+
+ let cache_type_str: String = match env.get_string(&cache_type) {
+ Ok(s) => s.into(),
+ Err(e) => {
+ let msg = format!("Failed to convert cache_type string: {}", e);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("java/lang/DataFusionException", &msg);
+ return 0;
+ }
+ };
+
+ let eviction_type_str: String = match env.get_string(&eviction_type) {
+ Ok(s) => s.into(),
+ Err(e) => {
+ let msg = format!("Failed to convert eviction_type string: {}", e);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("java/lang/DataFusionException", &msg);
+ return 0;
+ }
+ };
+
+ println!("[CACHE INFO] Creating cache: type={}, size_limit={}, eviction_type={}",
+ cache_type_str, size_limit, eviction_type_str);
+
+ let manager = unsafe { &mut *(cache_manager_ptr as *mut CustomCacheManager) };
+
+ match cache_type_str.as_str() {
+ cache::CACHE_TYPE_METADATA => {
+ let inner_cache = DefaultFilesMetadataCache::new(size_limit as usize);
+ let metadata_cache = Arc::new(cache::MutexFileMetadataCache::new(inner_cache));
+ manager.set_file_metadata_cache(metadata_cache);
+ println!("[CACHE INFO] Successfully created {} cache in CustomCacheManager", cache_type_str);
+ }
+ cache::CACHE_TYPE_STATS => {
+ let msg = "Stats cache not yet implemented";
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("java/lang/DataFusionException", msg);
+ return 0;
+ }
+ _ => {
+ let msg = format!("Invalid cache type: {}", cache_type_str);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("java/lang/DataFusionException", &msg);
+ return 0;
+ }
+ }
+
+ 0
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_opensearch_datafusion_jni_NativeBridge_cacheManagerAddFiles(
+ mut env: JNIEnv,
+ _class: JClass,
+ runtime_env_ptr: jlong,
+ files: JObjectArray,
+) {
+ if runtime_env_ptr == 0 {
+ let _ = env.throw_new("java/lang/NullPointerException", "Cache manager pointer is null");
+ return;
+ }
+
+ let runtime_env = unsafe { &*(runtime_env_ptr as *const DataFusionRuntime) };
+
+ match &runtime_env.custom_cache_manager {
+ Some(manager) => {
+ let file_paths: Vec = match parse_string_arr(&mut env, files) {
+ Ok(paths) => paths,
+ Err(e) => {
+ let msg = format!("Failed to parse file paths array: {}", e);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ return;
+ }
+ };
+
+ match manager.add_files(&file_paths) {
+ Ok(results) => {
+ let mut failed_files = Vec::new();
+ for (file_path, success) in results {
+ if !success {
+ failed_files.push(file_path);
+ }
+ }
+
+ if !failed_files.is_empty() {
+ let msg = format!("Failed to add {} files to cache: {:?}", failed_files.len(), failed_files);
+ eprintln!("[CACHE ERROR] {}", msg);
+ }
+ }
+ Err(e) => {
+ let msg = format!("Failed to add files to cache: {}", e);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ }
+ }
+ }
+ None => {
+ let msg = "No custom cache manager available";
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", msg);
+ }
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_opensearch_datafusion_jni_NativeBridge_cacheManagerRemoveFiles(
+ mut env: JNIEnv,
+ _class: JClass,
+ runtime_env_ptr: jlong,
+ files: JObjectArray,
+) {
+ if runtime_env_ptr == 0 {
+ let _ = env.throw_new("java/lang/NullPointerException", "Cache manager pointer is null");
+ return;
+ }
+
+ let runtime_env = unsafe { &*(runtime_env_ptr as *const DataFusionRuntime) };
+
+ let file_paths: Vec = match parse_string_arr(&mut env, files) {
+ Ok(paths) => paths,
+ Err(e) => {
+ let msg = format!("Failed to parse file paths array: {}", e);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ return;
+ }
+ };
+
+ match &runtime_env.custom_cache_manager {
+ Some(manager) => {
+ match manager.remove_files(&file_paths) {
+ Ok(results) => {
+ let mut failed_files = Vec::new();
+ for (file_path, removed) in results {
+ if !removed {
+ failed_files.push(file_path);
+ }
+ }
+
+ if !failed_files.is_empty() {
+ let msg = format!("Failed to remove {} files from cache: {:?}", failed_files.len(), failed_files);
+ eprintln!("[CACHE ERROR] {}", msg);
+ }
+ }
+ Err(e) => {
+ let msg = format!("Failed to remove files from cache: {}", e);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ }
+ }
+ }
+ None => {
+ let msg = "No custom cache manager available";
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", msg);
+ }
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_opensearch_datafusion_jni_NativeBridge_cacheManagerClear(
+ mut env: JNIEnv,
+ _class: JClass,
+ runtime_env_ptr: jlong,
+) {
+ if runtime_env_ptr == 0 {
+ let _ = env.throw_new("java/lang/NullPointerException", "Cache manager pointer is null");
+ return;
+ }
+
+ let runtime_env = unsafe { &*(runtime_env_ptr as *const DataFusionRuntime) };
+
+ match &runtime_env.custom_cache_manager {
+ Some(manager) => {
+ manager.clear_all();
+ println!("[CACHE INFO] Successfully cleared all caches");
+ }
+ None => {
+ let msg = "No custom cache manager available";
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", msg);
+ }
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_opensearch_datafusion_jni_NativeBridge_cacheManagerUpdateSizeLimitForCacheType(
+ mut env: JNIEnv,
+ _class: JClass,
+ runtime_env_ptr: jlong,
+ cache_type: JString,
+ new_size_limit: jlong,
+) -> bool {
+ if runtime_env_ptr == 0 {
+ let _ = env.throw_new("java/lang/NullPointerException", "Cache manager pointer is null");
+ return false;
+ }
+
+ let runtime_env = unsafe { &*(runtime_env_ptr as *const DataFusionRuntime) };
+
+ let cache_type: String = match env.get_string(&cache_type) {
+ Ok(s) => s.into(),
+ Err(e) => {
+ let msg = format!("Failed to convert cache type string: {}", e);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ return false;
+ }
+ };
+
+ match &runtime_env.custom_cache_manager {
+ Some(manager) => {
+ match cache_type.as_str() {
+ cache::CACHE_TYPE_METADATA => {
+ manager.update_metadata_cache_limit(new_size_limit as usize);
+ true
+ }
+ _ => {
+ let msg = format!("Unknown cache type: {}", cache_type);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ false
+ }
+ }
+ }
+ None => {
+ let msg = "No custom cache manager available";
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", msg);
+ false
+ }
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_opensearch_datafusion_jni_NativeBridge_cacheManagerGetMemoryConsumedForCacheType(
+ mut env: JNIEnv,
+ _class: JClass,
+ runtime_env_ptr: jlong,
+ cache_type: JString,
+) -> jlong {
+ if runtime_env_ptr == 0 {
+ let _ = env.throw_new("java/lang/NullPointerException", "Cache manager pointer is null");
+ return 0;
+ }
+
+ let runtime_env = unsafe { &*(runtime_env_ptr as *const DataFusionRuntime) };
+
+ let cache_type: String = match env.get_string(&cache_type) {
+ Ok(s) => s.into(),
+ Err(e) => {
+ let msg = format!("Failed to convert cache type string: {}", e);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ return 0;
+ }
+ };
+
+ match &runtime_env.custom_cache_manager {
+ Some(manager) => {
+ match cache_type.as_str() {
+ cache::CACHE_TYPE_METADATA => {
+ manager.get_total_memory_consumed() as jlong
+ }
+ _ => {
+ let msg = format!("Unknown cache type: {}", cache_type);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ 0
+ }
+ }
+ }
+ None => {
+ let msg = "No custom cache manager available";
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", msg);
+ 0
+ }
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_opensearch_datafusion_jni_NativeBridge_cacheManagerGetTotalMemoryConsumed(
+ mut env: JNIEnv,
+ _class: JClass,
+ runtime_env_ptr: jlong,
+) -> jlong {
+ if runtime_env_ptr == 0 {
+ let _ = env.throw_new("java/lang/NullPointerException", "Cache manager pointer is null");
+ return 0;
+ }
+
+ let runtime_env = unsafe { &*(runtime_env_ptr as *const DataFusionRuntime) };
+
+ match &runtime_env.custom_cache_manager {
+ Some(manager) => {
+ manager.get_total_memory_consumed() as jlong
+ }
+ None => {
+ 0
+ }
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_opensearch_datafusion_jni_NativeBridge_cacheManagerClearByCacheType(
+ mut env: JNIEnv,
+ _class: JClass,
+ runtime_env_ptr: jlong,
+ cache_type: JString,
+) {
+ if runtime_env_ptr == 0 {
+ let _ = env.throw_new("java/lang/NullPointerException", "Cache manager pointer is null");
+ return;
+ }
+
+ let runtime_env = unsafe { &*(runtime_env_ptr as *const DataFusionRuntime) };
+
+ let cache_type: String = match env.get_string(&cache_type) {
+ Ok(s) => s.into(),
+ Err(e) => {
+ let msg = format!("Failed to convert cache type string: {}", e);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ return;
+ }
+ };
+
+ match &runtime_env.custom_cache_manager {
+ Some(manager) => {
+ match manager.clear_cache_type(&cache_type) {
+ Ok(_) => {
+ println!("[CACHE INFO] Cache Type: {} cleared", cache_type);
+ }
+ Err(e) => {
+ eprintln!("[CACHE ERROR] {}", e);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &e);
+ }
+ }
+ }
+ None => {
+ let msg = "No custom cache manager available";
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", msg);
+ }
+ }
+}
+
+#[no_mangle]
+pub extern "system" fn Java_org_opensearch_datafusion_jni_NativeBridge_cacheManagerGetItemByCacheType(
+ mut env: JNIEnv,
+ _class: JClass,
+ runtime_env_ptr: jlong,
+ cache_type: JString,
+ file_path: JString,
+) -> bool {
+ if runtime_env_ptr == 0 {
+ let _ = env.throw_new("java/lang/NullPointerException", "Cache manager pointer is null");
+ return false;
+ }
+
+ let runtime_env = unsafe { &*(runtime_env_ptr as *const DataFusionRuntime) };
+
+ let cache_type: String = match env.get_string(&cache_type) {
+ Ok(s) => s.into(),
+ Err(e) => {
+ let msg = format!("Failed to convert cache type string: {}", e);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ return false;
+ }
+ };
+
+ let file_path: String = match env.get_string(&file_path) {
+ Ok(s) => s.into(),
+ Err(e) => {
+ let msg = format!("Failed to convert file path string: {}", e);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ return false;
+ }
+ };
+
+ match &runtime_env.custom_cache_manager {
+ Some(manager) => {
+ match cache_type.as_str() {
+ cache::CACHE_TYPE_METADATA => {
+ manager.contains_file(&file_path)
+ }
+ _ => {
+ let msg = format!("Unknown cache type: {}", cache_type);
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", &msg);
+ false
+ }
+ }
+ }
+ None => {
+ let msg = "No custom cache manager available";
+ eprintln!("[CACHE ERROR] {}", msg);
+ let _ = env.throw_new("org/opensearch/datafusion/DataFusionException", msg);
+ false
+ }
+ }
+}
diff --git a/plugins/engine-datafusion/jni/src/cross_rt_stream.rs b/plugins/engine-datafusion/jni/src/cross_rt_stream.rs
new file mode 100644
index 0000000000000..cd7c91cb3945b
--- /dev/null
+++ b/plugins/engine-datafusion/jni/src/cross_rt_stream.rs
@@ -0,0 +1,266 @@
+use std::{
+ future::Future,
+ pin::Pin,
+ sync::Arc,
+ task::{Context, Poll},
+};
+use crate::executor::{DedicatedExecutor, JobError};
+use datafusion::arrow::datatypes::SchemaRef;
+use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::error::DataFusionError;
+use datafusion::physical_plan::SendableRecordBatchStream;
+use futures::{future::BoxFuture, FutureExt, Stream, StreamExt, ready};
+use tokio::sync::mpsc::{Sender, channel};
+use tokio_stream::wrappers::ReceiverStream;
+
+// Copy of - https://github.com/influxdata/influxdb3_core/blob/main/iox_query/src/exec/cross_rt_stream.rs
+/// A stream adapter that bridges data from one Tokio runtime to another.
+///
+/// This is useful when you need to execute a DataFusion stream on a dedicated
+/// executor (e.g., for CPU-intensive work) but consume the results on a different
+/// runtime (e.g., the main I/O runtime).
+///
+///
+/// The stream uses a channel-based approach:
+/// - A "driver" future runs on the source runtime, pulling data from the source stream
+/// - Data is sent through an MPSC channel to the consumer runtime
+/// - The `CrossRtStream` polls both the driver and the receiver to ensure proper cleanup
+///
+///
+/// We need to poll both the driver and the inner receiver because:
+/// 1. The inner stream tells us when data is available or the channel is closed
+/// 2. The driver tells us when the background task has fully completed
+/// 3. We only return `Poll::Ready(None)` when BOTH are done to ensure proper cleanup
+pub struct CrossRtStream {
+ /// The background task that drives the source stream and sends data through the channel.
+ /// This future runs on the dedicated executor and handles:
+ /// - Polling the source stream
+ /// - Sending results through the channel
+ /// - Error handling and conversion
+ driver: BoxFuture<'static, ()>,
+
+ /// Tracks whether the driver future has completed.
+ /// We need to poll the driver to completion even after the channel closes
+ /// to ensure proper cleanup and avoid leaked resources.
+ driver_ready: bool,
+
+ /// The receiving end of the channel, wrapped in a stream adapter.
+ /// This receives `RecordBatch` results from the driver running on another runtime.
+ inner: ReceiverStream>,
+
+ /// Tracks whether the inner stream has ended (channel closed or exhausted).
+ /// Once true, we only need to wait for the driver to complete before returning `Poll::Ready(None)`.
+ inner_done: bool,
+
+ /// The Arrow schema for the record batches in this stream.
+ /// Cached here so it can be returned synchronously without runtime interaction.
+ schema: SchemaRef,
+}
+
+impl CrossRtStream {
+ /// Creates a new `CrossRtStream` with a custom driver function.
+ ///
+ /// # Arguments
+ ///
+ /// * `f` - A function that receives a channel sender and returns a future.
+ /// This future will be the driver that sends data through the channel.
+ /// * `schema` - The Arrow schema for the record batches
+ ///
+ /// # Type Parameters
+ ///
+ /// * `F` - The function type that creates the driver future
+ /// * `Fut` - The future type returned by `F`, must be `Send + 'static`
+ fn new_with_tx(f: F, schema: SchemaRef) -> Self
+ where
+ F: FnOnce(Sender>) -> Fut,
+ Fut: Future