diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 65309af..890c04c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -28,7 +28,7 @@ jobs: uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main with: macos_exclude_xcode_versions: "[{\"xcode_version\": \"16.0\"}, {\"xcode_version\": \"16.1\"}, {\"xcode_version\": \"16.2\"}, {\"xcode_version\": \"16.3\"}]" - linux_exclude_swift_versions: "[{\"swift_version\": \"5.9\"}, {\"swift_version\": \"5.10\"}, {\"swift_version\": \"6.0\"}, {\"swift_version\": \"6.1\"}, {\"swift_version\": \"nightly-main\"}, {\"swift_version\": \"nightly-6.1\"}]" + linux_exclude_swift_versions: "[{\"swift_version\": \"5.9\"}, {\"swift_version\": \"5.10\"}, {\"swift_version\": \"6.0\"}, {\"swift_version\": \"6.1\"}, {\"swift_version\": \"nightly-main\"}, {\"swift_version\": \"nightly-6.1\"}, {\"swift_version\": \"nightly-6.3\"}]" windows_exclude_swift_versions: "[{\"swift_version\": \"5.9\"}, {\"swift_version\": \"6.0\"}, {\"swift_version\": \"6.1\"}, {\"swift_version\": \"nightly\"}, {\"swift_version\": \"nightly-6.1\"}]" enable_macos_checks: false enable_windows_checks: false diff --git a/.swift-format-ignore b/.swift-format-ignore deleted file mode 100644 index 5bacb88..0000000 --- a/.swift-format-ignore +++ /dev/null @@ -1 +0,0 @@ -Sources/Arrow/Generated/** diff --git a/Package.swift b/Package.swift index d92f2d0..a487c86 100644 --- a/Package.swift +++ b/Package.swift @@ -29,7 +29,7 @@ let package = Package( dependencies: [ .package( url: "https://github.com/google/flatbuffers.git", - exact: "25.2.10" + exact: "25.12.19" ), .package( url: "https://github.com/apple/swift-atomics.git", diff --git a/README.md b/README.md index 11ba67e..3b7493c 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ The in-memory contiguous buffers allow constant-time random access to large, str ## Project status: -IPC serialization / deserialization has been tested against the Arrow integration testing JSON files, using the following strategy: +IPC deserialization has been tested against the Arrow integration testing JSON files (gold test files), using the following strategy: -1. Read the [Arrow cpp21 generated files](https://github.com/apache/arrow-testing/tree/master/data/arrow-ipc-stream/integration/cpp-21.0.0) into memory. +1. Read the Arrow integration test files into memory. 2. Encode the results to Codable & Equatable structs that can read and write the [test data format.](https://arrow.apache.org/docs/format/Integration.html#json-test-data-format). 3. Read the test JSON into the same Codable & Equatable struct and compare with the deserialized results,using Swift equality. This @@ -125,7 +125,7 @@ typealias ArrowArrayUtf8 = ArrowArrayVariable< FixedWidthBufferIPC, VariableLengthBufferIPC > -`` +``` ## Relationship to Arrow-Swift diff --git a/Scripts/flatc/Dockerfile b/Scripts/flatc/Dockerfile index e1e81fd..40017f5 100644 --- a/Scripts/flatc/Dockerfile +++ b/Scripts/flatc/Dockerfile @@ -10,7 +10,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* # Build FlatBuffers from source (ARM-compatible) -ENV FLATBUFFERS_VERSION=25.9.23 +ENV FLATBUFFERS_VERSION=25.12.19 RUN git clone --depth 1 --branch v${FLATBUFFERS_VERSION} https://github.com/google/flatbuffers.git && \ cd flatbuffers && \ cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release . && \ diff --git a/Scripts/flatc/compile-flatbuffers.sh b/Scripts/flatc/compile-flatbuffers.sh old mode 100644 new mode 100755 index 74fc902..f664840 --- a/Scripts/flatc/compile-flatbuffers.sh +++ b/Scripts/flatc/compile-flatbuffers.sh @@ -1,2 +1,2 @@ #!/bin/bash -container run -v "$(pwd)":/src flatc File.fbs +container run -v "$(pwd)":/src flatc File.fbs Message.fbs Schema.fbs SparseTensor.fbs Tensor.fbs diff --git a/Scripts/readArrowIPC.py b/Scripts/readArrowIPC.py index 83e59ce..d0ad20d 100644 --- a/Scripts/readArrowIPC.py +++ b/Scripts/readArrowIPC.py @@ -1,7 +1,8 @@ # Useful for debugging IPC writing issues. -import pyarrow as pa import sys +import pyarrow as pa + print(f"PyArrow version: {pa.__version__}") try: @@ -9,11 +10,11 @@ reader = pa.ipc.open_file(f) print(f"Schema: {reader.schema}") print(f"Num batches: {reader.num_record_batches}") - + for i in range(reader.num_record_batches): batch = reader.get_batch(i) print(f"Batch {i}: {batch.num_rows} rows, {batch.num_columns} columns") - + print("✓ File read successfully") except Exception as e: print(f"✗ Error: {e}") diff --git a/Sources/Arrow/Array/Array.swift b/Sources/Arrow/Array/Array.swift index 4aac340..b139fce 100644 --- a/Sources/Arrow/Array/Array.swift +++ b/Sources/Arrow/Array/Array.swift @@ -54,7 +54,7 @@ extension ArrowArrayBinaryView: StringArrayProtocol where ItemType == String {} /// A type which provides access to arrays of `Data`, with opaque offset types. /// /// The underlying array may have fixed or variable-length items. -protocol BinaryArrayProtocol: AnyArrowArrayProtocol { +public protocol BinaryArrayProtocol: AnyArrowArrayProtocol { subscript(index: Int) -> Data? { get } } extension ArrowArrayFixedSizeBinary: BinaryArrayProtocol {} diff --git a/Sources/Arrow/Array/ArrowColumn.swift b/Sources/Arrow/Array/ArrowColumn.swift index 63e8529..10c3296 100644 --- a/Sources/Arrow/Array/ArrowColumn.swift +++ b/Sources/Arrow/Array/ArrowColumn.swift @@ -14,7 +14,7 @@ // limitations under the License. public class ArrowColumn { - let dataHolder: any ChunkedArrayProtocol + public let dataHolder: any ChunkedArrayProtocol public let field: ArrowField public var length: Int { self.dataHolder.length } public var nullCount: Int { self.dataHolder.nullCount } diff --git a/Sources/Arrow/ArrowError.swift b/Sources/Arrow/ArrowError.swift index 9a28491..0ac39a1 100644 --- a/Sources/Arrow/ArrowError.swift +++ b/Sources/Arrow/ArrowError.swift @@ -21,7 +21,7 @@ public struct ArrowError: Error { case outOfBounds(index: Int64) case arrayHasNoElements case unknownError(String) - case notImplemented + case notImplemented(String) case ioError(String) case invalid(String) } diff --git a/Sources/Arrow/ArrowType.swift b/Sources/Arrow/ArrowType.swift index 04726fd..b2f9078 100644 --- a/Sources/Arrow/ArrowType.swift +++ b/Sources/Arrow/ArrowType.swift @@ -645,7 +645,7 @@ extension ArrowType { @inlinable public var isVariable: Bool { switch self { - case .binary, .utf8: true + case .binary, .utf8, .largeBinary, .largeUtf8: true default: false } } @@ -963,7 +963,7 @@ extension ArrowType { case .list(let field): return "+l" + (try field.type.cDataFormatId) default: - throw .init(.notImplemented) + throw .init(.notImplemented("cData not implmented for \(self).")) } } } @@ -1033,6 +1033,6 @@ extension ArrowType { } else if from == "u" { return .utf8 } - throw .init(.notImplemented) + throw .init(.notImplemented("cData not implmented for \(self).")) } } diff --git a/Sources/Arrow/Buffer/BinaryView.swift b/Sources/Arrow/Buffer/BinaryView.swift index cdb6709..75dd337 100644 --- a/Sources/Arrow/Buffer/BinaryView.swift +++ b/Sources/Arrow/Buffer/BinaryView.swift @@ -91,7 +91,7 @@ return view } - /// Create a referenced view (for length > 12) - trivial! + /// Create a referenced view (for length > 12) . public static func referenced( length: Int32, prefix: UInt32, @@ -107,7 +107,6 @@ view.words[1] = Int32(bitPattern: prefix) view.words[2] = bufferIndex view.words[3] = offset - return view } } diff --git a/Sources/Arrow/Extensions.swift b/Sources/Arrow/Extensions.swift new file mode 100644 index 0000000..b8cfd6e --- /dev/null +++ b/Sources/Arrow/Extensions.swift @@ -0,0 +1,39 @@ +// Copyright 2026 The Columnar Swift Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +extension ArrowField { + + /// Return the GeoArrow extension type if set, else returns nil. + public var geoArrowType: GeoArrowType? { + metadata["ARROW:extension:name"].flatMap { GeoArrowType(rawValue: $0) } + } +} + +/// The GeoArrow types. +/// +/// Use these in conjunction with a column's `ArrowType` to interpret a geometry column. There are +/// multiple possible memory layouts for each type. +public enum GeoArrowType: String { + case point = "geoarrow.point" + case linestring = "geoarrow.linestring" + case polygon = "geoarrow.polygon" + case multipoint = "geoarrow.multipoint" + case multilinestring = "geoarrow.multilinestring" + case multipolygon = "geoarrow.multipolygon" + case geometry = "geoarrow.geometry" + case geometrycollection = "geoarrow.geometrycollection" + case box = "geoarrow.box" + case wkb = "geoarrow.wkb" + case wkt = "geoarrow.wkt" +} diff --git a/Sources/Arrow/Generated/File_generated.swift b/Sources/Arrow/Generated/File_generated.swift deleted file mode 100644 index 28994c6..0000000 --- a/Sources/Arrow/Generated/File_generated.swift +++ /dev/null @@ -1,152 +0,0 @@ -// swift-format-ignore-file -// Copyright 2025 The Apache Software Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import FlatBuffers - -public struct org_apache_arrow_flatbuf_Block: NativeStruct, Verifiable, FlatbuffersInitializable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - - /// Index to the start of the RecordBlock (note this is past the Message header) - private var _offset: Int64 - /// Length of the metadata - private var _metaDataLength: Int32 - private let padding0__: UInt32 = 0 - /// Length of the data (this is aligned so there can be a gap between this and - /// the metadata). - private var _bodyLength: Int64 - - public init(_ bb: ByteBuffer, o: Int32) { - let _accessor = Struct(bb: bb, position: o) - _offset = _accessor.readBuffer(of: Int64.self, at: 0) - _metaDataLength = _accessor.readBuffer(of: Int32.self, at: 8) - _bodyLength = _accessor.readBuffer(of: Int64.self, at: 16) - } - - public init(offset: Int64, metaDataLength: Int32, bodyLength: Int64) { - _offset = offset - _metaDataLength = metaDataLength - _bodyLength = bodyLength - } - - public init() { - _offset = 0 - _metaDataLength = 0 - _bodyLength = 0 - } - - /// Index to the start of the RecordBlock (note this is past the Message header) - public var offset: Int64 { _offset } - /// Length of the metadata - public var metaDataLength: Int32 { _metaDataLength } - /// Length of the data (this is aligned so there can be a gap between this and - /// the metadata). - public var bodyLength: Int64 { _bodyLength } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - try verifier.inBuffer(position: position, of: org_apache_arrow_flatbuf_Block.self) - } -} - -public struct org_apache_arrow_flatbuf_Block_Mutable: FlatBufferObject { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Struct - - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Struct(bb: bb, position: o) } - - public var offset: Int64 { return _accessor.readBuffer(of: Int64.self, at: 0) } - public var metaDataLength: Int32 { return _accessor.readBuffer(of: Int32.self, at: 8) } - public var bodyLength: Int64 { return _accessor.readBuffer(of: Int64.self, at: 16) } -} - -/// ---------------------------------------------------------------------- -/// Arrow File metadata -/// -public struct org_apache_arrow_flatbuf_Footer: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case version = 4 - case schema = 6 - case dictionaries = 8 - case recordBatches = 10 - case customMetadata = 12 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var version: org_apache_arrow_flatbuf_MetadataVersion { let o = _accessor.offset(VTOFFSET.version.v); return o == 0 ? .v1 : org_apache_arrow_flatbuf_MetadataVersion(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .v1 } - public var schema: org_apache_arrow_flatbuf_Schema? { let o = _accessor.offset(VTOFFSET.schema.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Schema(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } - public var hasDictionaries: Bool { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? false : true } - public var dictionariesCount: Int32 { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func dictionaries(at index: Int32) -> org_apache_arrow_flatbuf_Block? { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Block.self, offset: _accessor.vector(at: o) + index * 24) } - public func mutableDictionaries(at index: Int32) -> org_apache_arrow_flatbuf_Block_Mutable? { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Block_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 24) } - public var hasRecordBatches: Bool { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? false : true } - public var recordBatchesCount: Int32 { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func recordBatches(at index: Int32) -> org_apache_arrow_flatbuf_Block? { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Block.self, offset: _accessor.vector(at: o) + index * 24) } - public func mutableRecordBatches(at index: Int32) -> org_apache_arrow_flatbuf_Block_Mutable? { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Block_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 24) } - /// User-defined metadata - public var hasCustomMetadata: Bool { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? false : true } - public var customMetadataCount: Int32 { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func customMetadata(at index: Int32) -> org_apache_arrow_flatbuf_KeyValue? { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? nil : org_apache_arrow_flatbuf_KeyValue(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } - public static func startFooter(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) } - public static func add(version: org_apache_arrow_flatbuf_MetadataVersion, _ fbb: inout FlatBufferBuilder) { fbb.add(element: version.rawValue, def: 0, at: VTOFFSET.version.p) } - public static func add(schema: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: schema, at: VTOFFSET.schema.p) } - public static func addVectorOf(dictionaries: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: dictionaries, at: VTOFFSET.dictionaries.p) } - public static func startVectorOfDictionaries(_ size: Int, in builder: inout FlatBufferBuilder) { - builder.startVector(size * MemoryLayout.size, elementSize: MemoryLayout.alignment) - } - public static func addVectorOf(recordBatches: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: recordBatches, at: VTOFFSET.recordBatches.p) } - public static func startVectorOfRecordBatches(_ size: Int, in builder: inout FlatBufferBuilder) { - builder.startVector(size * MemoryLayout.size, elementSize: MemoryLayout.alignment) - } - public static func addVectorOf(customMetadata: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: customMetadata, at: VTOFFSET.customMetadata.p) } - public static func endFooter(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createFooter( - _ fbb: inout FlatBufferBuilder, - version: org_apache_arrow_flatbuf_MetadataVersion = .v1, - schemaOffset schema: Offset = Offset(), - dictionariesVectorOffset dictionaries: Offset = Offset(), - recordBatchesVectorOffset recordBatches: Offset = Offset(), - customMetadataVectorOffset customMetadata: Offset = Offset() - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Footer.startFooter(&fbb) - org_apache_arrow_flatbuf_Footer.add(version: version, &fbb) - org_apache_arrow_flatbuf_Footer.add(schema: schema, &fbb) - org_apache_arrow_flatbuf_Footer.addVectorOf(dictionaries: dictionaries, &fbb) - org_apache_arrow_flatbuf_Footer.addVectorOf(recordBatches: recordBatches, &fbb) - org_apache_arrow_flatbuf_Footer.addVectorOf(customMetadata: customMetadata, &fbb) - return org_apache_arrow_flatbuf_Footer.endFooter(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.version.p, fieldName: "version", required: false, type: org_apache_arrow_flatbuf_MetadataVersion.self) - try _v.visit(field: VTOFFSET.schema.p, fieldName: "schema", required: false, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.dictionaries.p, fieldName: "dictionaries", required: false, type: ForwardOffset>.self) - try _v.visit(field: VTOFFSET.recordBatches.p, fieldName: "recordBatches", required: false, type: ForwardOffset>.self) - try _v.visit(field: VTOFFSET.customMetadata.p, fieldName: "customMetadata", required: false, type: ForwardOffset, org_apache_arrow_flatbuf_KeyValue>>.self) - _v.finish() - } -} - diff --git a/Sources/Arrow/Generated/FlatBuffersTypes.swift b/Sources/Arrow/Generated/FlatBuffersTypes.swift deleted file mode 100644 index bf184ab..0000000 --- a/Sources/Arrow/Generated/FlatBuffersTypes.swift +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2025 Columnar-Swift contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -typealias FFooter = org_apache_arrow_flatbuf_Footer -typealias FMessage = org_apache_arrow_flatbuf_Message -typealias FBlock = org_apache_arrow_flatbuf_Block -typealias FField = org_apache_arrow_flatbuf_Field -typealias FSchema = org_apache_arrow_flatbuf_Schema -typealias FBuffer = org_apache_arrow_flatbuf_Buffer -typealias FFieldNode = org_apache_arrow_flatbuf_FieldNode -typealias FRecordBatch = org_apache_arrow_flatbuf_RecordBatch -typealias FType = org_apache_arrow_flatbuf_Type_ -typealias FFloatingPoint = org_apache_arrow_flatbuf_FloatingPoint -typealias FInt = org_apache_arrow_flatbuf_Int -typealias FBool = org_apache_arrow_flatbuf_Bool -typealias FDate = org_apache_arrow_flatbuf_Date -typealias FTime = org_apache_arrow_flatbuf_Time -typealias FTimestamp = org_apache_arrow_flatbuf_Timestamp -typealias FTimeUnit = org_apache_arrow_flatbuf_TimeUnit -typealias FStruct = org_apache_arrow_flatbuf_Struct_ - -typealias FUtf8 = org_apache_arrow_flatbuf_Utf8 -typealias FBinary = org_apache_arrow_flatbuf_Binary -typealias FMessageHeader = org_apache_arrow_flatbuf_MessageHeader diff --git a/Sources/Arrow/Generated/Message_generated.swift b/Sources/Arrow/Generated/Message_generated.swift deleted file mode 100644 index f18f546..0000000 --- a/Sources/Arrow/Generated/Message_generated.swift +++ /dev/null @@ -1,431 +0,0 @@ -// swift-format-ignore-file -// Copyright 2025 The Apache Software Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - - -import FlatBuffers - -public enum org_apache_arrow_flatbuf_CompressionType: Int8, Enum, Verifiable { - public typealias T = Int8 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int8 { return self.rawValue } - case lz4Frame = 0 - case zstd = 1 - - public static var max: org_apache_arrow_flatbuf_CompressionType { return .zstd } - public static var min: org_apache_arrow_flatbuf_CompressionType { return .lz4Frame } -} - - -/// Provided for forward compatibility in case we need to support different -/// strategies for compressing the IPC message body (like whole-body -/// compression rather than buffer-level) in the future -public enum org_apache_arrow_flatbuf_BodyCompressionMethod: Int8, Enum, Verifiable { - public typealias T = Int8 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int8 { return self.rawValue } - /// Each constituent buffer is first compressed with the indicated - /// compressor, and then written with the uncompressed length in the first 8 - /// bytes as a 64-bit little-endian signed integer followed by the compressed - /// buffer bytes (and then padding as required by the protocol). The - /// uncompressed length may be set to -1 to indicate that the data that - /// follows is not compressed, which can be useful for cases where - /// compression does not yield appreciable savings. - case buffer = 0 - - public static var max: org_apache_arrow_flatbuf_BodyCompressionMethod { return .buffer } - public static var min: org_apache_arrow_flatbuf_BodyCompressionMethod { return .buffer } -} - - -/// ---------------------------------------------------------------------- -/// The root Message type -/// This union enables us to easily send different message types without -/// redundant storage, and in the future we can easily add new message types. -/// -/// Arrow implementations do not need to implement all of the message types, -/// which may include experimental metadata types. For maximum compatibility, -/// it is best to send data using RecordBatch -public enum org_apache_arrow_flatbuf_MessageHeader: UInt8, UnionEnum { - public typealias T = UInt8 - - public init?(value: T) { - self.init(rawValue: value) - } - - public static var byteSize: Int { return MemoryLayout.size } - public var value: UInt8 { return self.rawValue } - case none_ = 0 - case schema = 1 - case dictionarybatch = 2 - case recordbatch = 3 - case tensor = 4 - case sparsetensor = 5 - - public static var max: org_apache_arrow_flatbuf_MessageHeader { return .sparsetensor } - public static var min: org_apache_arrow_flatbuf_MessageHeader { return .none_ } -} - - -/// ---------------------------------------------------------------------- -/// Data structures for describing a table row batch (a collection of -/// equal-length Arrow arrays) -/// Metadata about a field at some level of a nested type tree (but not -/// its children). -/// -/// For example, a List with values `[[1, 2, 3], null, [4], [5, 6], null]` -/// would have {length: 5, null_count: 2} for its List node, and {length: 6, -/// null_count: 0} for its Int16 node, as separate FieldNode structs -public struct org_apache_arrow_flatbuf_FieldNode: NativeStruct, Verifiable, FlatbuffersInitializable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - - /// The number of value slots in the Arrow array at this level of a nested - /// tree - private var _length: Int64 - /// The number of observed nulls. Fields with null_count == 0 may choose not - /// to write their physical validity bitmap out as a materialized buffer, - /// instead setting the length of the bitmap buffer to 0. - private var _nullCount: Int64 - - public init(_ bb: ByteBuffer, o: Int32) { - let _accessor = Struct(bb: bb, position: o) - _length = _accessor.readBuffer(of: Int64.self, at: 0) - _nullCount = _accessor.readBuffer(of: Int64.self, at: 8) - } - - public init(length: Int64, nullCount: Int64) { - _length = length - _nullCount = nullCount - } - - public init() { - _length = 0 - _nullCount = 0 - } - - /// The number of value slots in the Arrow array at this level of a nested - /// tree - public var length: Int64 { _length } - /// The number of observed nulls. Fields with null_count == 0 may choose not - /// to write their physical validity bitmap out as a materialized buffer, - /// instead setting the length of the bitmap buffer to 0. - public var nullCount: Int64 { _nullCount } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - try verifier.inBuffer(position: position, of: org_apache_arrow_flatbuf_FieldNode.self) - } -} - -/// ---------------------------------------------------------------------- -/// Data structures for describing a table row batch (a collection of -/// equal-length Arrow arrays) -/// Metadata about a field at some level of a nested type tree (but not -/// its children). -/// -/// For example, a List with values `[[1, 2, 3], null, [4], [5, 6], null]` -/// would have {length: 5, null_count: 2} for its List node, and {length: 6, -/// null_count: 0} for its Int16 node, as separate FieldNode structs -public struct org_apache_arrow_flatbuf_FieldNode_Mutable: FlatBufferObject { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Struct - - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Struct(bb: bb, position: o) } - - public var length: Int64 { return _accessor.readBuffer(of: Int64.self, at: 0) } - public var nullCount: Int64 { return _accessor.readBuffer(of: Int64.self, at: 8) } -} - -/// Optional compression for the memory buffers constituting IPC message -/// bodies. Intended for use with RecordBatch but could be used for other -/// message types -public struct org_apache_arrow_flatbuf_BodyCompression: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case codec = 4 - case method = 6 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// Compressor library. - /// For LZ4_FRAME, each compressed buffer must consist of a single frame. - public var codec: org_apache_arrow_flatbuf_CompressionType { let o = _accessor.offset(VTOFFSET.codec.v); return o == 0 ? .lz4Frame : org_apache_arrow_flatbuf_CompressionType(rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .lz4Frame } - /// Indicates the way the record batch body was compressed - public var method: org_apache_arrow_flatbuf_BodyCompressionMethod { let o = _accessor.offset(VTOFFSET.method.v); return o == 0 ? .buffer : org_apache_arrow_flatbuf_BodyCompressionMethod(rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .buffer } - public static func startBodyCompression(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) } - public static func add(codec: org_apache_arrow_flatbuf_CompressionType, _ fbb: inout FlatBufferBuilder) { fbb.add(element: codec.rawValue, def: 0, at: VTOFFSET.codec.p) } - public static func add(method: org_apache_arrow_flatbuf_BodyCompressionMethod, _ fbb: inout FlatBufferBuilder) { fbb.add(element: method.rawValue, def: 0, at: VTOFFSET.method.p) } - public static func endBodyCompression(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createBodyCompression( - _ fbb: inout FlatBufferBuilder, - codec: org_apache_arrow_flatbuf_CompressionType = .lz4Frame, - method: org_apache_arrow_flatbuf_BodyCompressionMethod = .buffer - ) -> Offset { - let __start = org_apache_arrow_flatbuf_BodyCompression.startBodyCompression(&fbb) - org_apache_arrow_flatbuf_BodyCompression.add(codec: codec, &fbb) - org_apache_arrow_flatbuf_BodyCompression.add(method: method, &fbb) - return org_apache_arrow_flatbuf_BodyCompression.endBodyCompression(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.codec.p, fieldName: "codec", required: false, type: org_apache_arrow_flatbuf_CompressionType.self) - try _v.visit(field: VTOFFSET.method.p, fieldName: "method", required: false, type: org_apache_arrow_flatbuf_BodyCompressionMethod.self) - _v.finish() - } -} - -/// A data header describing the shared memory layout of a "record" or "row" -/// batch. Some systems call this a "row batch" internally and others a "record -/// batch". -public struct org_apache_arrow_flatbuf_RecordBatch: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case length = 4 - case nodes = 6 - case buffers = 8 - case compression = 10 - case variadicBufferCounts = 12 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// number of records / rows. The arrays in the batch should all have this - /// length - public var length: Int64 { let o = _accessor.offset(VTOFFSET.length.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) } - /// Nodes correspond to the pre-ordered flattened logical schema - public var hasNodes: Bool { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? false : true } - public var nodesCount: Int32 { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func nodes(at index: Int32) -> org_apache_arrow_flatbuf_FieldNode? { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_FieldNode.self, offset: _accessor.vector(at: o) + index * 16) } - public func mutableNodes(at index: Int32) -> org_apache_arrow_flatbuf_FieldNode_Mutable? { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? nil : org_apache_arrow_flatbuf_FieldNode_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) } - /// Buffers correspond to the pre-ordered flattened buffer tree - /// - /// The number of buffers appended to this list depends on the schema. For - /// example, most primitive arrays will have 2 buffers, 1 for the validity - /// bitmap and 1 for the values. For struct arrays, there will only be a - /// single buffer for the validity (nulls) bitmap - public var hasBuffers: Bool { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? false : true } - public var buffersCount: Int32 { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func buffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer? { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Buffer.self, offset: _accessor.vector(at: o) + index * 16) } - public func mutableBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer_Mutable? { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) } - /// Optional compression of the message body - public var compression: org_apache_arrow_flatbuf_BodyCompression? { let o = _accessor.offset(VTOFFSET.compression.v); return o == 0 ? nil : org_apache_arrow_flatbuf_BodyCompression(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } - /// Some types such as Utf8View are represented using a variable number of buffers. - /// For each such Field in the pre-ordered flattened logical schema, there will be - /// an entry in variadicBufferCounts to indicate the number of number of variadic - /// buffers which belong to that Field in the current RecordBatch. - /// - /// For example, the schema - /// col1: Struct - /// col2: Utf8View - /// contains two Fields with variadic buffers so variadicBufferCounts will have - /// two entries, the first counting the variadic buffers of `col1.beta` and the - /// second counting `col2`'s. - /// - /// This field may be omitted if and only if the schema contains no Fields with - /// a variable number of buffers, such as BinaryView and Utf8View. - public var hasVariadicBufferCounts: Bool { let o = _accessor.offset(VTOFFSET.variadicBufferCounts.v); return o == 0 ? false : true } - public var variadicBufferCountsCount: Int32 { let o = _accessor.offset(VTOFFSET.variadicBufferCounts.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func variadicBufferCounts(at index: Int32) -> Int64 { let o = _accessor.offset(VTOFFSET.variadicBufferCounts.v); return o == 0 ? 0 : _accessor.directRead(of: Int64.self, offset: _accessor.vector(at: o) + index * 8) } - public var variadicBufferCounts: [Int64] { return _accessor.getVector(at: VTOFFSET.variadicBufferCounts.v) ?? [] } - public static func startRecordBatch(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) } - public static func add(length: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: length, def: 0, at: VTOFFSET.length.p) } - public static func addVectorOf(nodes: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: nodes, at: VTOFFSET.nodes.p) } - public static func startVectorOfNodes(_ size: Int, in builder: inout FlatBufferBuilder) { - builder.startVector(size * MemoryLayout.size, elementSize: MemoryLayout.alignment) - } - public static func addVectorOf(buffers: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: buffers, at: VTOFFSET.buffers.p) } - public static func startVectorOfBuffers(_ size: Int, in builder: inout FlatBufferBuilder) { - builder.startVector(size * MemoryLayout.size, elementSize: MemoryLayout.alignment) - } - public static func add(compression: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: compression, at: VTOFFSET.compression.p) } - public static func addVectorOf(variadicBufferCounts: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: variadicBufferCounts, at: VTOFFSET.variadicBufferCounts.p) } - public static func endRecordBatch(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createRecordBatch( - _ fbb: inout FlatBufferBuilder, - length: Int64 = 0, - nodesVectorOffset nodes: Offset = Offset(), - buffersVectorOffset buffers: Offset = Offset(), - compressionOffset compression: Offset = Offset(), - variadicBufferCountsVectorOffset variadicBufferCounts: Offset = Offset() - ) -> Offset { - let __start = org_apache_arrow_flatbuf_RecordBatch.startRecordBatch(&fbb) - org_apache_arrow_flatbuf_RecordBatch.add(length: length, &fbb) - org_apache_arrow_flatbuf_RecordBatch.addVectorOf(nodes: nodes, &fbb) - org_apache_arrow_flatbuf_RecordBatch.addVectorOf(buffers: buffers, &fbb) - org_apache_arrow_flatbuf_RecordBatch.add(compression: compression, &fbb) - org_apache_arrow_flatbuf_RecordBatch.addVectorOf(variadicBufferCounts: variadicBufferCounts, &fbb) - return org_apache_arrow_flatbuf_RecordBatch.endRecordBatch(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.length.p, fieldName: "length", required: false, type: Int64.self) - try _v.visit(field: VTOFFSET.nodes.p, fieldName: "nodes", required: false, type: ForwardOffset>.self) - try _v.visit(field: VTOFFSET.buffers.p, fieldName: "buffers", required: false, type: ForwardOffset>.self) - try _v.visit(field: VTOFFSET.compression.p, fieldName: "compression", required: false, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.variadicBufferCounts.p, fieldName: "variadicBufferCounts", required: false, type: ForwardOffset>.self) - _v.finish() - } -} - -/// For sending dictionary encoding information. Any Field can be -/// dictionary-encoded, but in this case none of its children may be -/// dictionary-encoded. -/// There is one vector / column per dictionary, but that vector / column -/// may be spread across multiple dictionary batches by using the isDelta -/// flag -public struct org_apache_arrow_flatbuf_DictionaryBatch: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case id = 4 - case data = 6 - case isDelta = 8 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var id: Int64 { let o = _accessor.offset(VTOFFSET.id.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) } - public var data: org_apache_arrow_flatbuf_RecordBatch? { let o = _accessor.offset(VTOFFSET.data.v); return o == 0 ? nil : org_apache_arrow_flatbuf_RecordBatch(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } - /// If isDelta is true the values in the dictionary are to be appended to a - /// dictionary with the indicated id. If isDelta is false this dictionary - /// should replace the existing dictionary. - public var isDelta: Bool { let o = _accessor.offset(VTOFFSET.isDelta.v); return o == 0 ? false : _accessor.readBuffer(of: Bool.self, at: o) } - public static func startDictionaryBatch(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 3) } - public static func add(id: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: id, def: 0, at: VTOFFSET.id.p) } - public static func add(data: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: data, at: VTOFFSET.data.p) } - public static func add(isDelta: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: isDelta, def: false, - at: VTOFFSET.isDelta.p) } - public static func endDictionaryBatch(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createDictionaryBatch( - _ fbb: inout FlatBufferBuilder, - id: Int64 = 0, - dataOffset data: Offset = Offset(), - isDelta: Bool = false - ) -> Offset { - let __start = org_apache_arrow_flatbuf_DictionaryBatch.startDictionaryBatch(&fbb) - org_apache_arrow_flatbuf_DictionaryBatch.add(id: id, &fbb) - org_apache_arrow_flatbuf_DictionaryBatch.add(data: data, &fbb) - org_apache_arrow_flatbuf_DictionaryBatch.add(isDelta: isDelta, &fbb) - return org_apache_arrow_flatbuf_DictionaryBatch.endDictionaryBatch(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.id.p, fieldName: "id", required: false, type: Int64.self) - try _v.visit(field: VTOFFSET.data.p, fieldName: "data", required: false, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.isDelta.p, fieldName: "isDelta", required: false, type: Bool.self) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_Message: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case version = 4 - case headerType = 6 - case header = 8 - case bodyLength = 10 - case customMetadata = 12 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var version: org_apache_arrow_flatbuf_MetadataVersion { let o = _accessor.offset(VTOFFSET.version.v); return o == 0 ? .v1 : org_apache_arrow_flatbuf_MetadataVersion(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .v1 } - public var headerType: org_apache_arrow_flatbuf_MessageHeader { let o = _accessor.offset(VTOFFSET.headerType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_MessageHeader(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ } - public func header(type: T.Type) -> T? { let o = _accessor.offset(VTOFFSET.header.v); return o == 0 ? nil : _accessor.union(o) } - public var bodyLength: Int64 { let o = _accessor.offset(VTOFFSET.bodyLength.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) } - public var hasCustomMetadata: Bool { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? false : true } - public var customMetadataCount: Int32 { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func customMetadata(at index: Int32) -> org_apache_arrow_flatbuf_KeyValue? { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? nil : org_apache_arrow_flatbuf_KeyValue(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } - public static func startMessage(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) } - public static func add(version: org_apache_arrow_flatbuf_MetadataVersion, _ fbb: inout FlatBufferBuilder) { fbb.add(element: version.rawValue, def: 0, at: VTOFFSET.version.p) } - public static func add(headerType: org_apache_arrow_flatbuf_MessageHeader, _ fbb: inout FlatBufferBuilder) { fbb.add(element: headerType.rawValue, def: 0, at: VTOFFSET.headerType.p) } - public static func add(header: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: header, at: VTOFFSET.header.p) } - public static func add(bodyLength: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: bodyLength, def: 0, at: VTOFFSET.bodyLength.p) } - public static func addVectorOf(customMetadata: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: customMetadata, at: VTOFFSET.customMetadata.p) } - public static func endMessage(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createMessage( - _ fbb: inout FlatBufferBuilder, - version: org_apache_arrow_flatbuf_MetadataVersion = .v1, - headerType: org_apache_arrow_flatbuf_MessageHeader = .none_, - headerOffset header: Offset = Offset(), - bodyLength: Int64 = 0, - customMetadataVectorOffset customMetadata: Offset = Offset() - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Message.startMessage(&fbb) - org_apache_arrow_flatbuf_Message.add(version: version, &fbb) - org_apache_arrow_flatbuf_Message.add(headerType: headerType, &fbb) - org_apache_arrow_flatbuf_Message.add(header: header, &fbb) - org_apache_arrow_flatbuf_Message.add(bodyLength: bodyLength, &fbb) - org_apache_arrow_flatbuf_Message.addVectorOf(customMetadata: customMetadata, &fbb) - return org_apache_arrow_flatbuf_Message.endMessage(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.version.p, fieldName: "version", required: false, type: org_apache_arrow_flatbuf_MetadataVersion.self) - try _v.visit(unionKey: VTOFFSET.headerType.p, unionField: VTOFFSET.header.p, unionKeyName: "headerType", fieldName: "header", required: false, completion: { (verifier, key: org_apache_arrow_flatbuf_MessageHeader, pos) in - switch key { - case .none_: - break // NOTE - SWIFT doesnt support none - case .schema: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Schema.self) - case .dictionarybatch: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_DictionaryBatch.self) - case .recordbatch: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_RecordBatch.self) - case .tensor: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Tensor.self) - case .sparsetensor: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_SparseTensor.self) - } - }) - try _v.visit(field: VTOFFSET.bodyLength.p, fieldName: "bodyLength", required: false, type: Int64.self) - try _v.visit(field: VTOFFSET.customMetadata.p, fieldName: "customMetadata", required: false, type: ForwardOffset, org_apache_arrow_flatbuf_KeyValue>>.self) - _v.finish() - } -} - diff --git a/Sources/Arrow/Generated/Schema_generated.swift b/Sources/Arrow/Generated/Schema_generated.swift deleted file mode 100644 index 31af7a2..0000000 --- a/Sources/Arrow/Generated/Schema_generated.swift +++ /dev/null @@ -1,1525 +0,0 @@ -// swift-format-ignore-file -// Copyright 2025 The Apache Software Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - - -import FlatBuffers - -public enum org_apache_arrow_flatbuf_MetadataVersion: Int16, Enum, Verifiable { - public typealias T = Int16 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int16 { return self.rawValue } - /// 0.1.0 (October 2016). - case v1 = 0 - /// 0.2.0 (February 2017). Non-backwards compatible with V1. - case v2 = 1 - /// 0.3.0 -> 0.7.1 (May - December 2017). Non-backwards compatible with V2. - case v3 = 2 - /// >= 0.8.0 (December 2017). Non-backwards compatible with V3. - case v4 = 3 - /// >= 1.0.0 (July 2020). Backwards compatible with V4 (V5 readers can read V4 - /// metadata and IPC messages). Implementations are recommended to provide a - /// V4 compatibility mode with V5 format changes disabled. - /// - /// Incompatible changes between V4 and V5: - /// - Union buffer layout has changed. In V5, Unions don't have a validity - /// bitmap buffer. - case v5 = 4 - - public static var max: org_apache_arrow_flatbuf_MetadataVersion { return .v5 } - public static var min: org_apache_arrow_flatbuf_MetadataVersion { return .v1 } -} - - -/// Represents Arrow Features that might not have full support -/// within implementations. This is intended to be used in -/// two scenarios: -/// 1. A mechanism for readers of Arrow Streams -/// and files to understand that the stream or file makes -/// use of a feature that isn't supported or unknown to -/// the implementation (and therefore can meet the Arrow -/// forward compatibility guarantees). -/// 2. A means of negotiating between a client and server -/// what features a stream is allowed to use. The enums -/// values here are intended to represent higher level -/// features, additional details may be negotiated -/// with key-value pairs specific to the protocol. -/// -/// Enums added to this list should be assigned power-of-two values -/// to facilitate exchanging and comparing bitmaps for supported -/// features. -public enum org_apache_arrow_flatbuf_Feature: Int64, Enum, Verifiable { - public typealias T = Int64 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int64 { return self.rawValue } - /// Needed to make flatbuffers happy. - case unused = 0 - /// The stream makes use of multiple full dictionaries with the - /// same ID and assumes clients implement dictionary replacement - /// correctly. - case dictionaryReplacement = 1 - /// The stream makes use of compressed bodies as described - /// in Message.fbs. - case compressedBody = 2 - - public static var max: org_apache_arrow_flatbuf_Feature { return .compressedBody } - public static var min: org_apache_arrow_flatbuf_Feature { return .unused } -} - - -public enum org_apache_arrow_flatbuf_UnionMode: Int16, Enum, Verifiable { - public typealias T = Int16 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int16 { return self.rawValue } - case sparse = 0 - case dense = 1 - - public static var max: org_apache_arrow_flatbuf_UnionMode { return .dense } - public static var min: org_apache_arrow_flatbuf_UnionMode { return .sparse } -} - - -public enum org_apache_arrow_flatbuf_Precision: Int16, Enum, Verifiable { - public typealias T = Int16 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int16 { return self.rawValue } - case half = 0 - case single = 1 - case double = 2 - - public static var max: org_apache_arrow_flatbuf_Precision { return .double } - public static var min: org_apache_arrow_flatbuf_Precision { return .half } -} - - -public enum org_apache_arrow_flatbuf_DateUnit: Int16, Enum, Verifiable { - public typealias T = Int16 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int16 { return self.rawValue } - case day = 0 - case millisecond = 1 - - public static var max: org_apache_arrow_flatbuf_DateUnit { return .millisecond } - public static var min: org_apache_arrow_flatbuf_DateUnit { return .day } -} - - -public enum org_apache_arrow_flatbuf_TimeUnit: Int16, Enum, Verifiable { - public typealias T = Int16 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int16 { return self.rawValue } - case second = 0 - case millisecond = 1 - case microsecond = 2 - case nanosecond = 3 - - public static var max: org_apache_arrow_flatbuf_TimeUnit { return .nanosecond } - public static var min: org_apache_arrow_flatbuf_TimeUnit { return .second } -} - - -public enum org_apache_arrow_flatbuf_IntervalUnit: Int16, Enum, Verifiable { - public typealias T = Int16 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int16 { return self.rawValue } - case yearMonth = 0 - case dayTime = 1 - case monthDayNano = 2 - - public static var max: org_apache_arrow_flatbuf_IntervalUnit { return .monthDayNano } - public static var min: org_apache_arrow_flatbuf_IntervalUnit { return .yearMonth } -} - - -/// ---------------------------------------------------------------------- -/// Top-level Type value, enabling extensible type-specific metadata. We can -/// add new logical types to Type without breaking backwards compatibility -public enum org_apache_arrow_flatbuf_Type_: UInt8, UnionEnum { - public typealias T = UInt8 - - public init?(value: T) { - self.init(rawValue: value) - } - - public static var byteSize: Int { return MemoryLayout.size } - public var value: UInt8 { return self.rawValue } - case none_ = 0 - case null = 1 - case int = 2 - case floatingpoint = 3 - case binary = 4 - case utf8 = 5 - case bool = 6 - case decimal = 7 - case date = 8 - case time = 9 - case timestamp = 10 - case interval = 11 - case list = 12 - case struct_ = 13 - case union = 14 - case fixedsizebinary = 15 - case fixedsizelist = 16 - case map = 17 - case duration = 18 - case largebinary = 19 - case largeutf8 = 20 - case largelist = 21 - case runendencoded = 22 - case binaryview = 23 - case utf8view = 24 - case listview = 25 - case largelistview = 26 - - public static var max: org_apache_arrow_flatbuf_Type_ { return .largelistview } - public static var min: org_apache_arrow_flatbuf_Type_ { return .none_ } -} - - -/// ---------------------------------------------------------------------- -/// Dictionary encoding metadata -/// Maintained for forwards compatibility, in the future -/// Dictionaries might be explicit maps between integers and values -/// allowing for non-contiguous index values -public enum org_apache_arrow_flatbuf_DictionaryKind: Int16, Enum, Verifiable { - public typealias T = Int16 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int16 { return self.rawValue } - case densearray = 0 - - public static var max: org_apache_arrow_flatbuf_DictionaryKind { return .densearray } - public static var min: org_apache_arrow_flatbuf_DictionaryKind { return .densearray } -} - - -/// ---------------------------------------------------------------------- -/// Endianness of the platform producing the data -public enum org_apache_arrow_flatbuf_Endianness: Int16, Enum, Verifiable { - public typealias T = Int16 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int16 { return self.rawValue } - case little = 0 - case big = 1 - - public static var max: org_apache_arrow_flatbuf_Endianness { return .big } - public static var min: org_apache_arrow_flatbuf_Endianness { return .little } -} - - -/// ---------------------------------------------------------------------- -/// A Buffer represents a single contiguous memory segment -public struct org_apache_arrow_flatbuf_Buffer: NativeStruct, Verifiable, FlatbuffersInitializable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - - /// The relative offset into the shared memory page where the bytes for this - /// buffer starts - private var _offset: Int64 - /// The absolute length (in bytes) of the memory buffer. The memory is found - /// from offset (inclusive) to offset + length (non-inclusive). When building - /// messages using the encapsulated IPC message, padding bytes may be written - /// after a buffer, but such padding bytes do not need to be accounted for in - /// the size here. - private var _length: Int64 - - public init(_ bb: ByteBuffer, o: Int32) { - let _accessor = Struct(bb: bb, position: o) - _offset = _accessor.readBuffer(of: Int64.self, at: 0) - _length = _accessor.readBuffer(of: Int64.self, at: 8) - } - - public init(offset: Int64, length: Int64) { - _offset = offset - _length = length - } - - public init() { - _offset = 0 - _length = 0 - } - - /// The relative offset into the shared memory page where the bytes for this - /// buffer starts - public var offset: Int64 { _offset } - /// The absolute length (in bytes) of the memory buffer. The memory is found - /// from offset (inclusive) to offset + length (non-inclusive). When building - /// messages using the encapsulated IPC message, padding bytes may be written - /// after a buffer, but such padding bytes do not need to be accounted for in - /// the size here. - public var length: Int64 { _length } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - try verifier.inBuffer(position: position, of: org_apache_arrow_flatbuf_Buffer.self) - } -} - -/// ---------------------------------------------------------------------- -/// A Buffer represents a single contiguous memory segment -public struct org_apache_arrow_flatbuf_Buffer_Mutable: FlatBufferObject { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Struct - - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Struct(bb: bb, position: o) } - - public var offset: Int64 { return _accessor.readBuffer(of: Int64.self, at: 0) } - public var length: Int64 { return _accessor.readBuffer(of: Int64.self, at: 8) } -} - -/// These are stored in the flatbuffer in the Type union below -public struct org_apache_arrow_flatbuf_Null: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startNull(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endNull(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -/// A Struct_ in the flatbuffer metadata is the same as an Arrow Struct -/// (according to the physical memory layout). We used Struct_ here as -/// Struct is a reserved word in Flatbuffers -public struct org_apache_arrow_flatbuf_Struct_: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startStruct_(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endStruct_(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_List: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startList(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endList(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -/// Same as List, but with 64-bit offsets, allowing to represent -/// extremely large data values. -public struct org_apache_arrow_flatbuf_LargeList: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startLargeList(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endLargeList(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -/// Represents the same logical types that List can, but contains offsets and -/// sizes allowing for writes in any order and sharing of child values among -/// list values. -public struct org_apache_arrow_flatbuf_ListView: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startListView(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endListView(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -/// Same as ListView, but with 64-bit offsets and sizes, allowing to represent -/// extremely large data values. -public struct org_apache_arrow_flatbuf_LargeListView: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startLargeListView(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endLargeListView(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_FixedSizeList: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case listSize = 4 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// Number of list items per value - public var listSize: Int32 { let o = _accessor.offset(VTOFFSET.listSize.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int32.self, at: o) } - public static func startFixedSizeList(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 1) } - public static func add(listSize: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: listSize, def: 0, at: VTOFFSET.listSize.p) } - public static func endFixedSizeList(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createFixedSizeList( - _ fbb: inout FlatBufferBuilder, - listSize: Int32 = 0 - ) -> Offset { - let __start = org_apache_arrow_flatbuf_FixedSizeList.startFixedSizeList(&fbb) - org_apache_arrow_flatbuf_FixedSizeList.add(listSize: listSize, &fbb) - return org_apache_arrow_flatbuf_FixedSizeList.endFixedSizeList(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.listSize.p, fieldName: "listSize", required: false, type: Int32.self) - _v.finish() - } -} - -/// A Map is a logical nested type that is represented as -/// -/// List> -/// -/// In this layout, the keys and values are each respectively contiguous. We do -/// not constrain the key and value types, so the application is responsible -/// for ensuring that the keys are hashable and unique. Whether the keys are sorted -/// may be set in the metadata for this field. -/// -/// In a field with Map type, the field has a child Struct field, which then -/// has two children: key type and the second the value type. The names of the -/// child fields may be respectively "entries", "key", and "value", but this is -/// not enforced. -/// -/// Map -/// ```text -/// - child[0] entries: Struct -/// - child[0] key: K -/// - child[1] value: V -/// ``` -/// Neither the "entries" field nor the "key" field may be nullable. -/// -/// The metadata is structured so that Arrow systems without special handling -/// for Map can make Map an alias for List. The "layout" attribute for the Map -/// field must have the same contents as a List. -public struct org_apache_arrow_flatbuf_Map: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case keysSorted = 4 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// Set to true if the keys within each value are sorted - public var keysSorted: Bool { let o = _accessor.offset(VTOFFSET.keysSorted.v); return o == 0 ? false : _accessor.readBuffer(of: Bool.self, at: o) } - public static func startMap(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 1) } - public static func add(keysSorted: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: keysSorted, def: false, - at: VTOFFSET.keysSorted.p) } - public static func endMap(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createMap( - _ fbb: inout FlatBufferBuilder, - keysSorted: Bool = false - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Map.startMap(&fbb) - org_apache_arrow_flatbuf_Map.add(keysSorted: keysSorted, &fbb) - return org_apache_arrow_flatbuf_Map.endMap(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.keysSorted.p, fieldName: "keysSorted", required: false, type: Bool.self) - _v.finish() - } -} - -/// A union is a complex type with children in Field -/// By default ids in the type vector refer to the offsets in the children -/// optionally typeIds provides an indirection between the child offset and the type id -/// for each child `typeIds[offset]` is the id used in the type vector -public struct org_apache_arrow_flatbuf_Union: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case mode = 4 - case typeIds = 6 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var mode: org_apache_arrow_flatbuf_UnionMode { let o = _accessor.offset(VTOFFSET.mode.v); return o == 0 ? .sparse : org_apache_arrow_flatbuf_UnionMode(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .sparse } - public var hasTypeIds: Bool { let o = _accessor.offset(VTOFFSET.typeIds.v); return o == 0 ? false : true } - public var typeIdsCount: Int32 { let o = _accessor.offset(VTOFFSET.typeIds.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func typeIds(at index: Int32) -> Int32 { let o = _accessor.offset(VTOFFSET.typeIds.v); return o == 0 ? 0 : _accessor.directRead(of: Int32.self, offset: _accessor.vector(at: o) + index * 4) } - public var typeIds: [Int32] { return _accessor.getVector(at: VTOFFSET.typeIds.v) ?? [] } - public static func startUnion(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) } - public static func add(mode: org_apache_arrow_flatbuf_UnionMode, _ fbb: inout FlatBufferBuilder) { fbb.add(element: mode.rawValue, def: 0, at: VTOFFSET.mode.p) } - public static func addVectorOf(typeIds: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: typeIds, at: VTOFFSET.typeIds.p) } - public static func endUnion(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createUnion( - _ fbb: inout FlatBufferBuilder, - mode: org_apache_arrow_flatbuf_UnionMode = .sparse, - typeIdsVectorOffset typeIds: Offset = Offset() - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Union.startUnion(&fbb) - org_apache_arrow_flatbuf_Union.add(mode: mode, &fbb) - org_apache_arrow_flatbuf_Union.addVectorOf(typeIds: typeIds, &fbb) - return org_apache_arrow_flatbuf_Union.endUnion(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.mode.p, fieldName: "mode", required: false, type: org_apache_arrow_flatbuf_UnionMode.self) - try _v.visit(field: VTOFFSET.typeIds.p, fieldName: "typeIds", required: false, type: ForwardOffset>.self) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_Int: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case bitWidth = 4 - case isSigned = 6 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var bitWidth: Int32 { let o = _accessor.offset(VTOFFSET.bitWidth.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int32.self, at: o) } - public var isSigned: Bool { let o = _accessor.offset(VTOFFSET.isSigned.v); return o == 0 ? false : _accessor.readBuffer(of: Bool.self, at: o) } - public static func startInt(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) } - public static func add(bitWidth: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: bitWidth, def: 0, at: VTOFFSET.bitWidth.p) } - public static func add(isSigned: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: isSigned, def: false, - at: VTOFFSET.isSigned.p) } - public static func endInt(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createInt( - _ fbb: inout FlatBufferBuilder, - bitWidth: Int32 = 0, - isSigned: Bool = false - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Int.startInt(&fbb) - org_apache_arrow_flatbuf_Int.add(bitWidth: bitWidth, &fbb) - org_apache_arrow_flatbuf_Int.add(isSigned: isSigned, &fbb) - return org_apache_arrow_flatbuf_Int.endInt(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.bitWidth.p, fieldName: "bitWidth", required: false, type: Int32.self) - try _v.visit(field: VTOFFSET.isSigned.p, fieldName: "isSigned", required: false, type: Bool.self) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_FloatingPoint: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case precision = 4 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var precision: org_apache_arrow_flatbuf_Precision { let o = _accessor.offset(VTOFFSET.precision.v); return o == 0 ? .half : org_apache_arrow_flatbuf_Precision(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .half } - public static func startFloatingPoint(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 1) } - public static func add(precision: org_apache_arrow_flatbuf_Precision, _ fbb: inout FlatBufferBuilder) { fbb.add(element: precision.rawValue, def: 0, at: VTOFFSET.precision.p) } - public static func endFloatingPoint(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createFloatingPoint( - _ fbb: inout FlatBufferBuilder, - precision: org_apache_arrow_flatbuf_Precision = .half - ) -> Offset { - let __start = org_apache_arrow_flatbuf_FloatingPoint.startFloatingPoint(&fbb) - org_apache_arrow_flatbuf_FloatingPoint.add(precision: precision, &fbb) - return org_apache_arrow_flatbuf_FloatingPoint.endFloatingPoint(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.precision.p, fieldName: "precision", required: false, type: org_apache_arrow_flatbuf_Precision.self) - _v.finish() - } -} - -/// Unicode with UTF-8 encoding -public struct org_apache_arrow_flatbuf_Utf8: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startUtf8(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endUtf8(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -/// Opaque binary data -public struct org_apache_arrow_flatbuf_Binary: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startBinary(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endBinary(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -/// Same as Utf8, but with 64-bit offsets, allowing to represent -/// extremely large data values. -public struct org_apache_arrow_flatbuf_LargeUtf8: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startLargeUtf8(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endLargeUtf8(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -/// Same as Binary, but with 64-bit offsets, allowing to represent -/// extremely large data values. -public struct org_apache_arrow_flatbuf_LargeBinary: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startLargeBinary(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endLargeBinary(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -/// Logically the same as Utf8, but the internal representation uses a view -/// struct that contains the string length and either the string's entire data -/// inline (for small strings) or an inlined prefix, an index of another buffer, -/// and an offset pointing to a slice in that buffer (for non-small strings). -/// -/// Since it uses a variable number of data buffers, each Field with this type -/// must have a corresponding entry in `variadicBufferCounts`. -public struct org_apache_arrow_flatbuf_Utf8View: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startUtf8View(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endUtf8View(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -/// Logically the same as Binary, but the internal representation uses a view -/// struct that contains the string length and either the string's entire data -/// inline (for small strings) or an inlined prefix, an index of another buffer, -/// and an offset pointing to a slice in that buffer (for non-small strings). -/// -/// Since it uses a variable number of data buffers, each Field with this type -/// must have a corresponding entry in `variadicBufferCounts`. -public struct org_apache_arrow_flatbuf_BinaryView: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startBinaryView(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endBinaryView(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_FixedSizeBinary: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case byteWidth = 4 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// Number of bytes per value - public var byteWidth: Int32 { let o = _accessor.offset(VTOFFSET.byteWidth.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int32.self, at: o) } - public static func startFixedSizeBinary(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 1) } - public static func add(byteWidth: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: byteWidth, def: 0, at: VTOFFSET.byteWidth.p) } - public static func endFixedSizeBinary(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createFixedSizeBinary( - _ fbb: inout FlatBufferBuilder, - byteWidth: Int32 = 0 - ) -> Offset { - let __start = org_apache_arrow_flatbuf_FixedSizeBinary.startFixedSizeBinary(&fbb) - org_apache_arrow_flatbuf_FixedSizeBinary.add(byteWidth: byteWidth, &fbb) - return org_apache_arrow_flatbuf_FixedSizeBinary.endFixedSizeBinary(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.byteWidth.p, fieldName: "byteWidth", required: false, type: Int32.self) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_Bool: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startBool(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endBool(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -/// Contains two child arrays, run_ends and values. -/// The run_ends child array must be a 16/32/64-bit integer array -/// which encodes the indices at which the run with the value in -/// each corresponding index in the values child array ends. -/// Like list/struct types, the value array can be of any type. -public struct org_apache_arrow_flatbuf_RunEndEncoded: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - public static func startRunEndEncoded(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 0) } - public static func endRunEndEncoded(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - _v.finish() - } -} - -/// Exact decimal value represented as an integer value in two's -/// complement. Currently 32-bit (4-byte), 64-bit (8-byte), -/// 128-bit (16-byte) and 256-bit (32-byte) integers are used. -/// The representation uses the endianness indicated in the Schema. -public struct org_apache_arrow_flatbuf_Decimal: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case precision = 4 - case scale = 6 - case bitWidth = 8 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// Total number of decimal digits - public var precision: Int32 { let o = _accessor.offset(VTOFFSET.precision.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int32.self, at: o) } - /// Number of digits after the decimal point "." - public var scale: Int32 { let o = _accessor.offset(VTOFFSET.scale.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int32.self, at: o) } - /// Number of bits per value. The accepted widths are 32, 64, 128 and 256. - /// We use bitWidth for consistency with Int::bitWidth. - public var bitWidth: Int32 { let o = _accessor.offset(VTOFFSET.bitWidth.v); return o == 0 ? 128 : _accessor.readBuffer(of: Int32.self, at: o) } - public static func startDecimal(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 3) } - public static func add(precision: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: precision, def: 0, at: VTOFFSET.precision.p) } - public static func add(scale: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: scale, def: 0, at: VTOFFSET.scale.p) } - public static func add(bitWidth: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: bitWidth, def: 128, at: VTOFFSET.bitWidth.p) } - public static func endDecimal(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createDecimal( - _ fbb: inout FlatBufferBuilder, - precision: Int32 = 0, - scale: Int32 = 0, - bitWidth: Int32 = 128 - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Decimal.startDecimal(&fbb) - org_apache_arrow_flatbuf_Decimal.add(precision: precision, &fbb) - org_apache_arrow_flatbuf_Decimal.add(scale: scale, &fbb) - org_apache_arrow_flatbuf_Decimal.add(bitWidth: bitWidth, &fbb) - return org_apache_arrow_flatbuf_Decimal.endDecimal(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.precision.p, fieldName: "precision", required: false, type: Int32.self) - try _v.visit(field: VTOFFSET.scale.p, fieldName: "scale", required: false, type: Int32.self) - try _v.visit(field: VTOFFSET.bitWidth.p, fieldName: "bitWidth", required: false, type: Int32.self) - _v.finish() - } -} - -/// Date is either a 32-bit or 64-bit signed integer type representing an -/// elapsed time since UNIX epoch (1970-01-01), stored in either of two units: -/// -/// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no -/// leap seconds), where the values are evenly divisible by 86400000 -/// * Days (32 bits) since the UNIX epoch -public struct org_apache_arrow_flatbuf_Date: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case unit = 4 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var unit: org_apache_arrow_flatbuf_DateUnit { let o = _accessor.offset(VTOFFSET.unit.v); return o == 0 ? .millisecond : org_apache_arrow_flatbuf_DateUnit(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .millisecond } - public static func startDate(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 1) } - public static func add(unit: org_apache_arrow_flatbuf_DateUnit, _ fbb: inout FlatBufferBuilder) { fbb.add(element: unit.rawValue, def: 1, at: VTOFFSET.unit.p) } - public static func endDate(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createDate( - _ fbb: inout FlatBufferBuilder, - unit: org_apache_arrow_flatbuf_DateUnit = .millisecond - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Date.startDate(&fbb) - org_apache_arrow_flatbuf_Date.add(unit: unit, &fbb) - return org_apache_arrow_flatbuf_Date.endDate(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.unit.p, fieldName: "unit", required: false, type: org_apache_arrow_flatbuf_DateUnit.self) - _v.finish() - } -} - -/// Time is either a 32-bit or 64-bit signed integer type representing an -/// elapsed time since midnight, stored in either of four units: seconds, -/// milliseconds, microseconds or nanoseconds. -/// -/// The integer `bitWidth` depends on the `unit` and must be one of the following: -/// * SECOND and MILLISECOND: 32 bits -/// * MICROSECOND and NANOSECOND: 64 bits -/// -/// The allowed values are between 0 (inclusive) and 86400 (=24*60*60) seconds -/// (exclusive), adjusted for the time unit (for example, up to 86400000 -/// exclusive for the MILLISECOND unit). -/// This definition doesn't allow for leap seconds. Time values from -/// measurements with leap seconds will need to be corrected when ingesting -/// into Arrow (for example by replacing the value 86400 with 86399). -public struct org_apache_arrow_flatbuf_Time: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case unit = 4 - case bitWidth = 6 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var unit: org_apache_arrow_flatbuf_TimeUnit { let o = _accessor.offset(VTOFFSET.unit.v); return o == 0 ? .millisecond : org_apache_arrow_flatbuf_TimeUnit(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .millisecond } - public var bitWidth: Int32 { let o = _accessor.offset(VTOFFSET.bitWidth.v); return o == 0 ? 32 : _accessor.readBuffer(of: Int32.self, at: o) } - public static func startTime(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) } - public static func add(unit: org_apache_arrow_flatbuf_TimeUnit, _ fbb: inout FlatBufferBuilder) { fbb.add(element: unit.rawValue, def: 1, at: VTOFFSET.unit.p) } - public static func add(bitWidth: Int32, _ fbb: inout FlatBufferBuilder) { fbb.add(element: bitWidth, def: 32, at: VTOFFSET.bitWidth.p) } - public static func endTime(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createTime( - _ fbb: inout FlatBufferBuilder, - unit: org_apache_arrow_flatbuf_TimeUnit = .millisecond, - bitWidth: Int32 = 32 - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Time.startTime(&fbb) - org_apache_arrow_flatbuf_Time.add(unit: unit, &fbb) - org_apache_arrow_flatbuf_Time.add(bitWidth: bitWidth, &fbb) - return org_apache_arrow_flatbuf_Time.endTime(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.unit.p, fieldName: "unit", required: false, type: org_apache_arrow_flatbuf_TimeUnit.self) - try _v.visit(field: VTOFFSET.bitWidth.p, fieldName: "bitWidth", required: false, type: Int32.self) - _v.finish() - } -} - -/// Timestamp is a 64-bit signed integer representing an elapsed time since a -/// fixed epoch, stored in either of four units: seconds, milliseconds, -/// microseconds or nanoseconds, and is optionally annotated with a timezone. -/// -/// Timestamp values do not include any leap seconds (in other words, all -/// days are considered 86400 seconds long). -/// -/// Timestamps with a non-empty timezone -/// ------------------------------------ -/// -/// If a Timestamp column has a non-empty timezone value, its epoch is -/// 1970-01-01 00:00:00 (January 1st 1970, midnight) in the *UTC* timezone -/// (the Unix epoch), regardless of the Timestamp's own timezone. -/// -/// Therefore, timestamp values with a non-empty timezone correspond to -/// physical points in time together with some additional information about -/// how the data was obtained and/or how to display it (the timezone). -/// -/// For example, the timestamp value 0 with the timezone string "Europe/Paris" -/// corresponds to "January 1st 1970, 00h00" in the UTC timezone, but the -/// application may prefer to display it as "January 1st 1970, 01h00" in -/// the Europe/Paris timezone (which is the same physical point in time). -/// -/// One consequence is that timestamp values with a non-empty timezone -/// can be compared and ordered directly, since they all share the same -/// well-known point of reference (the Unix epoch). -/// -/// Timestamps with an unset / empty timezone -/// ----------------------------------------- -/// -/// If a Timestamp column has no timezone value, its epoch is -/// 1970-01-01 00:00:00 (January 1st 1970, midnight) in an *unknown* timezone. -/// -/// Therefore, timestamp values without a timezone cannot be meaningfully -/// interpreted as physical points in time, but only as calendar / clock -/// indications ("wall clock time") in an unspecified timezone. -/// -/// For example, the timestamp value 0 with an empty timezone string -/// corresponds to "January 1st 1970, 00h00" in an unknown timezone: there -/// is not enough information to interpret it as a well-defined physical -/// point in time. -/// -/// One consequence is that timestamp values without a timezone cannot -/// be reliably compared or ordered, since they may have different points of -/// reference. In particular, it is *not* possible to interpret an unset -/// or empty timezone as the same as "UTC". -/// -/// Conversion between timezones -/// ---------------------------- -/// -/// If a Timestamp column has a non-empty timezone, changing the timezone -/// to a different non-empty value is a metadata-only operation: -/// the timestamp values need not change as their point of reference remains -/// the same (the Unix epoch). -/// -/// However, if a Timestamp column has no timezone value, changing it to a -/// non-empty value requires to think about the desired semantics. -/// One possibility is to assume that the original timestamp values are -/// relative to the epoch of the timezone being set; timestamp values should -/// then adjusted to the Unix epoch (for example, changing the timezone from -/// empty to "Europe/Paris" would require converting the timestamp values -/// from "Europe/Paris" to "UTC", which seems counter-intuitive but is -/// nevertheless correct). -/// -/// Guidelines for encoding data from external libraries -/// ---------------------------------------------------- -/// -/// Date & time libraries often have multiple different data types for temporal -/// data. In order to ease interoperability between different implementations the -/// Arrow project has some recommendations for encoding these types into a Timestamp -/// column. -/// -/// An "instant" represents a physical point in time that has no relevant timezone -/// (for example, astronomical data). To encode an instant, use a Timestamp with -/// the timezone string set to "UTC", and make sure the Timestamp values -/// are relative to the UTC epoch (January 1st 1970, midnight). -/// -/// A "zoned date-time" represents a physical point in time annotated with an -/// informative timezone (for example, the timezone in which the data was -/// recorded). To encode a zoned date-time, use a Timestamp with the timezone -/// string set to the name of the timezone, and make sure the Timestamp values -/// are relative to the UTC epoch (January 1st 1970, midnight). -/// -/// (There is some ambiguity between an instant and a zoned date-time with the -/// UTC timezone. Both of these are stored the same in Arrow. Typically, -/// this distinction does not matter. If it does, then an application should -/// use custom metadata or an extension type to distinguish between the two cases.) -/// -/// An "offset date-time" represents a physical point in time combined with an -/// explicit offset from UTC. To encode an offset date-time, use a Timestamp -/// with the timezone string set to the numeric timezone offset string -/// (e.g. "+03:00"), and make sure the Timestamp values are relative to -/// the UTC epoch (January 1st 1970, midnight). -/// -/// A "naive date-time" (also called "local date-time" in some libraries) -/// represents a wall clock time combined with a calendar date, but with -/// no indication of how to map this information to a physical point in time. -/// Naive date-times must be handled with care because of this missing -/// information, and also because daylight saving time (DST) may make -/// some values ambiguous or nonexistent. A naive date-time may be -/// stored as a struct with Date and Time fields. However, it may also be -/// encoded into a Timestamp column with an empty timezone. The timestamp -/// values should be computed "as if" the timezone of the date-time values -/// was UTC; for example, the naive date-time "January 1st 1970, 00h00" would -/// be encoded as timestamp value 0. -public struct org_apache_arrow_flatbuf_Timestamp: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case unit = 4 - case timezone = 6 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var unit: org_apache_arrow_flatbuf_TimeUnit { let o = _accessor.offset(VTOFFSET.unit.v); return o == 0 ? .second : org_apache_arrow_flatbuf_TimeUnit(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .second } - /// The timezone is an optional string indicating the name of a timezone, - /// one of: - /// - /// * As used in the Olson timezone database (the "tz database" or - /// "tzdata"), such as "America/New_York". - /// * An absolute timezone offset of the form "+XX:XX" or "-XX:XX", - /// such as "+07:30". - /// - /// Whether a timezone string is present indicates different semantics about - /// the data (see above). - public var timezone: String? { let o = _accessor.offset(VTOFFSET.timezone.v); return o == 0 ? nil : _accessor.string(at: o) } - public var timezoneSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.timezone.v) } - public static func startTimestamp(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) } - public static func add(unit: org_apache_arrow_flatbuf_TimeUnit, _ fbb: inout FlatBufferBuilder) { fbb.add(element: unit.rawValue, def: 0, at: VTOFFSET.unit.p) } - public static func add(timezone: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: timezone, at: VTOFFSET.timezone.p) } - public static func endTimestamp(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createTimestamp( - _ fbb: inout FlatBufferBuilder, - unit: org_apache_arrow_flatbuf_TimeUnit = .second, - timezoneOffset timezone: Offset = Offset() - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Timestamp.startTimestamp(&fbb) - org_apache_arrow_flatbuf_Timestamp.add(unit: unit, &fbb) - org_apache_arrow_flatbuf_Timestamp.add(timezone: timezone, &fbb) - return org_apache_arrow_flatbuf_Timestamp.endTimestamp(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.unit.p, fieldName: "unit", required: false, type: org_apache_arrow_flatbuf_TimeUnit.self) - try _v.visit(field: VTOFFSET.timezone.p, fieldName: "timezone", required: false, type: ForwardOffset.self) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_Interval: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case unit = 4 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var unit: org_apache_arrow_flatbuf_IntervalUnit { let o = _accessor.offset(VTOFFSET.unit.v); return o == 0 ? .yearMonth : org_apache_arrow_flatbuf_IntervalUnit(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .yearMonth } - public static func startInterval(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 1) } - public static func add(unit: org_apache_arrow_flatbuf_IntervalUnit, _ fbb: inout FlatBufferBuilder) { fbb.add(element: unit.rawValue, def: 0, at: VTOFFSET.unit.p) } - public static func endInterval(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createInterval( - _ fbb: inout FlatBufferBuilder, - unit: org_apache_arrow_flatbuf_IntervalUnit = .yearMonth - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Interval.startInterval(&fbb) - org_apache_arrow_flatbuf_Interval.add(unit: unit, &fbb) - return org_apache_arrow_flatbuf_Interval.endInterval(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.unit.p, fieldName: "unit", required: false, type: org_apache_arrow_flatbuf_IntervalUnit.self) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_Duration: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case unit = 4 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var unit: org_apache_arrow_flatbuf_TimeUnit { let o = _accessor.offset(VTOFFSET.unit.v); return o == 0 ? .millisecond : org_apache_arrow_flatbuf_TimeUnit(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .millisecond } - public static func startDuration(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 1) } - public static func add(unit: org_apache_arrow_flatbuf_TimeUnit, _ fbb: inout FlatBufferBuilder) { fbb.add(element: unit.rawValue, def: 1, at: VTOFFSET.unit.p) } - public static func endDuration(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createDuration( - _ fbb: inout FlatBufferBuilder, - unit: org_apache_arrow_flatbuf_TimeUnit = .millisecond - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Duration.startDuration(&fbb) - org_apache_arrow_flatbuf_Duration.add(unit: unit, &fbb) - return org_apache_arrow_flatbuf_Duration.endDuration(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.unit.p, fieldName: "unit", required: false, type: org_apache_arrow_flatbuf_TimeUnit.self) - _v.finish() - } -} - -/// ---------------------------------------------------------------------- -/// user defined key value pairs to add custom metadata to arrow -/// key namespacing is the responsibility of the user -public struct org_apache_arrow_flatbuf_KeyValue: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case key = 4 - case value = 6 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var key: String? { let o = _accessor.offset(VTOFFSET.key.v); return o == 0 ? nil : _accessor.string(at: o) } - public var keySegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.key.v) } - public var value: String? { let o = _accessor.offset(VTOFFSET.value.v); return o == 0 ? nil : _accessor.string(at: o) } - public var valueSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.value.v) } - public static func startKeyValue(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) } - public static func add(key: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: key, at: VTOFFSET.key.p) } - public static func add(value: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: value, at: VTOFFSET.value.p) } - public static func endKeyValue(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createKeyValue( - _ fbb: inout FlatBufferBuilder, - keyOffset key: Offset = Offset(), - valueOffset value: Offset = Offset() - ) -> Offset { - let __start = org_apache_arrow_flatbuf_KeyValue.startKeyValue(&fbb) - org_apache_arrow_flatbuf_KeyValue.add(key: key, &fbb) - org_apache_arrow_flatbuf_KeyValue.add(value: value, &fbb) - return org_apache_arrow_flatbuf_KeyValue.endKeyValue(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.key.p, fieldName: "key", required: false, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.value.p, fieldName: "value", required: false, type: ForwardOffset.self) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_DictionaryEncoding: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case id = 4 - case indexType = 6 - case isOrdered = 8 - case dictionaryKind = 10 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// The known dictionary id in the application where this data is used. In - /// the file or streaming formats, the dictionary ids are found in the - /// DictionaryBatch messages - public var id: Int64 { let o = _accessor.offset(VTOFFSET.id.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) } - /// The dictionary indices are constrained to be non-negative integers. If - /// this field is null, the indices must be signed int32. To maximize - /// cross-language compatibility and performance, implementations are - /// recommended to prefer signed integer types over unsigned integer types - /// and to avoid uint64 indices unless they are required by an application. - public var indexType: org_apache_arrow_flatbuf_Int? { let o = _accessor.offset(VTOFFSET.indexType.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } - /// By default, dictionaries are not ordered, or the order does not have - /// semantic meaning. In some statistical, applications, dictionary-encoding - /// is used to represent ordered categorical data, and we provide a way to - /// preserve that metadata here - public var isOrdered: Bool { let o = _accessor.offset(VTOFFSET.isOrdered.v); return o == 0 ? false : _accessor.readBuffer(of: Bool.self, at: o) } - public var dictionaryKind: org_apache_arrow_flatbuf_DictionaryKind { let o = _accessor.offset(VTOFFSET.dictionaryKind.v); return o == 0 ? .densearray : org_apache_arrow_flatbuf_DictionaryKind(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .densearray } - public static func startDictionaryEncoding(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 4) } - public static func add(id: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: id, def: 0, at: VTOFFSET.id.p) } - public static func add(indexType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indexType, at: VTOFFSET.indexType.p) } - public static func add(isOrdered: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: isOrdered, def: false, - at: VTOFFSET.isOrdered.p) } - public static func add(dictionaryKind: org_apache_arrow_flatbuf_DictionaryKind, _ fbb: inout FlatBufferBuilder) { fbb.add(element: dictionaryKind.rawValue, def: 0, at: VTOFFSET.dictionaryKind.p) } - public static func endDictionaryEncoding(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createDictionaryEncoding( - _ fbb: inout FlatBufferBuilder, - id: Int64 = 0, - indexTypeOffset indexType: Offset = Offset(), - isOrdered: Bool = false, - dictionaryKind: org_apache_arrow_flatbuf_DictionaryKind = .densearray - ) -> Offset { - let __start = org_apache_arrow_flatbuf_DictionaryEncoding.startDictionaryEncoding(&fbb) - org_apache_arrow_flatbuf_DictionaryEncoding.add(id: id, &fbb) - org_apache_arrow_flatbuf_DictionaryEncoding.add(indexType: indexType, &fbb) - org_apache_arrow_flatbuf_DictionaryEncoding.add(isOrdered: isOrdered, &fbb) - org_apache_arrow_flatbuf_DictionaryEncoding.add(dictionaryKind: dictionaryKind, &fbb) - return org_apache_arrow_flatbuf_DictionaryEncoding.endDictionaryEncoding(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.id.p, fieldName: "id", required: false, type: Int64.self) - try _v.visit(field: VTOFFSET.indexType.p, fieldName: "indexType", required: false, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.isOrdered.p, fieldName: "isOrdered", required: false, type: Bool.self) - try _v.visit(field: VTOFFSET.dictionaryKind.p, fieldName: "dictionaryKind", required: false, type: org_apache_arrow_flatbuf_DictionaryKind.self) - _v.finish() - } -} - -/// ---------------------------------------------------------------------- -/// A field represents a named column in a record / row batch or child of a -/// nested type. -public struct org_apache_arrow_flatbuf_Field: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case name = 4 - case nullable = 6 - case typeType = 8 - case type = 10 - case dictionary = 12 - case children = 14 - case customMetadata = 16 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// Name is not required (e.g., in a List) - public var name: String? { let o = _accessor.offset(VTOFFSET.name.v); return o == 0 ? nil : _accessor.string(at: o) } - public var nameSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.name.v) } - /// Whether or not this field can contain nulls. Should be true in general. - public var nullable: Bool { let o = _accessor.offset(VTOFFSET.nullable.v); return o == 0 ? false : _accessor.readBuffer(of: Bool.self, at: o) } - public var typeType: org_apache_arrow_flatbuf_Type_ { let o = _accessor.offset(VTOFFSET.typeType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_Type_(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ } - /// This is the type of the decoded value if the field is dictionary encoded. - public func type(type: T.Type) -> T? { let o = _accessor.offset(VTOFFSET.type.v); return o == 0 ? nil : _accessor.union(o) } - /// Present only if the field is dictionary encoded. - public var dictionary: org_apache_arrow_flatbuf_DictionaryEncoding? { let o = _accessor.offset(VTOFFSET.dictionary.v); return o == 0 ? nil : org_apache_arrow_flatbuf_DictionaryEncoding(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } - /// children apply only to nested data types like Struct, List and Union. For - /// primitive types children will have length 0. - public var hasChildren: Bool { let o = _accessor.offset(VTOFFSET.children.v); return o == 0 ? false : true } - public var childrenCount: Int32 { let o = _accessor.offset(VTOFFSET.children.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func children(at index: Int32) -> org_apache_arrow_flatbuf_Field? { let o = _accessor.offset(VTOFFSET.children.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Field(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } - /// User-defined metadata - public var hasCustomMetadata: Bool { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? false : true } - public var customMetadataCount: Int32 { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func customMetadata(at index: Int32) -> org_apache_arrow_flatbuf_KeyValue? { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? nil : org_apache_arrow_flatbuf_KeyValue(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } - public static func startField(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 7) } - public static func add(name: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: name, at: VTOFFSET.name.p) } - public static func add(nullable: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: nullable, def: false, - at: VTOFFSET.nullable.p) } - public static func add(typeType: org_apache_arrow_flatbuf_Type_, _ fbb: inout FlatBufferBuilder) { fbb.add(element: typeType.rawValue, def: 0, at: VTOFFSET.typeType.p) } - public static func add(type: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: type, at: VTOFFSET.type.p) } - public static func add(dictionary: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: dictionary, at: VTOFFSET.dictionary.p) } - public static func addVectorOf(children: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: children, at: VTOFFSET.children.p) } - public static func addVectorOf(customMetadata: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: customMetadata, at: VTOFFSET.customMetadata.p) } - public static func endField(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createField( - _ fbb: inout FlatBufferBuilder, - nameOffset name: Offset = Offset(), - nullable: Bool = false, - typeType: org_apache_arrow_flatbuf_Type_ = .none_, - typeOffset type: Offset = Offset(), - dictionaryOffset dictionary: Offset = Offset(), - childrenVectorOffset children: Offset = Offset(), - customMetadataVectorOffset customMetadata: Offset = Offset() - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Field.startField(&fbb) - org_apache_arrow_flatbuf_Field.add(name: name, &fbb) - org_apache_arrow_flatbuf_Field.add(nullable: nullable, &fbb) - org_apache_arrow_flatbuf_Field.add(typeType: typeType, &fbb) - org_apache_arrow_flatbuf_Field.add(type: type, &fbb) - org_apache_arrow_flatbuf_Field.add(dictionary: dictionary, &fbb) - org_apache_arrow_flatbuf_Field.addVectorOf(children: children, &fbb) - org_apache_arrow_flatbuf_Field.addVectorOf(customMetadata: customMetadata, &fbb) - return org_apache_arrow_flatbuf_Field.endField(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.name.p, fieldName: "name", required: false, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.nullable.p, fieldName: "nullable", required: false, type: Bool.self) - try _v.visit(unionKey: VTOFFSET.typeType.p, unionField: VTOFFSET.type.p, unionKeyName: "typeType", fieldName: "type", required: false, completion: { (verifier, key: org_apache_arrow_flatbuf_Type_, pos) in - switch key { - case .none_: - break // NOTE - SWIFT doesnt support none - case .null: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Null.self) - case .int: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Int.self) - case .floatingpoint: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FloatingPoint.self) - case .binary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Binary.self) - case .utf8: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8.self) - case .bool: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Bool.self) - case .decimal: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Decimal.self) - case .date: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Date.self) - case .time: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Time.self) - case .timestamp: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Timestamp.self) - case .interval: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Interval.self) - case .list: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_List.self) - case .struct_: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Struct_.self) - case .union: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Union.self) - case .fixedsizebinary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeBinary.self) - case .fixedsizelist: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeList.self) - case .map: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Map.self) - case .duration: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Duration.self) - case .largebinary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeBinary.self) - case .largeutf8: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeUtf8.self) - case .largelist: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeList.self) - case .runendencoded: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_RunEndEncoded.self) - case .binaryview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_BinaryView.self) - case .utf8view: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8View.self) - case .listview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_ListView.self) - case .largelistview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeListView.self) - } - }) - try _v.visit(field: VTOFFSET.dictionary.p, fieldName: "dictionary", required: false, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.children.p, fieldName: "children", required: false, type: ForwardOffset, org_apache_arrow_flatbuf_Field>>.self) - try _v.visit(field: VTOFFSET.customMetadata.p, fieldName: "customMetadata", required: false, type: ForwardOffset, org_apache_arrow_flatbuf_KeyValue>>.self) - _v.finish() - } -} - -/// ---------------------------------------------------------------------- -/// A Schema describes the columns in a row batch -public struct org_apache_arrow_flatbuf_Schema: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case endianness = 4 - case fields = 6 - case customMetadata = 8 - case features = 10 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// endianness of the buffer - /// it is Little Endian by default - /// if endianness doesn't match the underlying system then the vectors need to be converted - public var endianness: org_apache_arrow_flatbuf_Endianness { let o = _accessor.offset(VTOFFSET.endianness.v); return o == 0 ? .little : org_apache_arrow_flatbuf_Endianness(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .little } - public var hasFields: Bool { let o = _accessor.offset(VTOFFSET.fields.v); return o == 0 ? false : true } - public var fieldsCount: Int32 { let o = _accessor.offset(VTOFFSET.fields.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func fields(at index: Int32) -> org_apache_arrow_flatbuf_Field? { let o = _accessor.offset(VTOFFSET.fields.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Field(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } - public var hasCustomMetadata: Bool { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? false : true } - public var customMetadataCount: Int32 { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func customMetadata(at index: Int32) -> org_apache_arrow_flatbuf_KeyValue? { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? nil : org_apache_arrow_flatbuf_KeyValue(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } - /// Features used in the stream/file. - public var hasFeatures: Bool { let o = _accessor.offset(VTOFFSET.features.v); return o == 0 ? false : true } - public var featuresCount: Int32 { let o = _accessor.offset(VTOFFSET.features.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func features(at index: Int32) -> org_apache_arrow_flatbuf_Feature? { let o = _accessor.offset(VTOFFSET.features.v); return o == 0 ? org_apache_arrow_flatbuf_Feature.unused : org_apache_arrow_flatbuf_Feature(rawValue: _accessor.directRead(of: Int64.self, offset: _accessor.vector(at: o) + index * 8)) } - public static func startSchema(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 4) } - public static func add(endianness: org_apache_arrow_flatbuf_Endianness, _ fbb: inout FlatBufferBuilder) { fbb.add(element: endianness.rawValue, def: 0, at: VTOFFSET.endianness.p) } - public static func addVectorOf(fields: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: fields, at: VTOFFSET.fields.p) } - public static func addVectorOf(customMetadata: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: customMetadata, at: VTOFFSET.customMetadata.p) } - public static func addVectorOf(features: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: features, at: VTOFFSET.features.p) } - public static func endSchema(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createSchema( - _ fbb: inout FlatBufferBuilder, - endianness: org_apache_arrow_flatbuf_Endianness = .little, - fieldsVectorOffset fields: Offset = Offset(), - customMetadataVectorOffset customMetadata: Offset = Offset(), - featuresVectorOffset features: Offset = Offset() - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Schema.startSchema(&fbb) - org_apache_arrow_flatbuf_Schema.add(endianness: endianness, &fbb) - org_apache_arrow_flatbuf_Schema.addVectorOf(fields: fields, &fbb) - org_apache_arrow_flatbuf_Schema.addVectorOf(customMetadata: customMetadata, &fbb) - org_apache_arrow_flatbuf_Schema.addVectorOf(features: features, &fbb) - return org_apache_arrow_flatbuf_Schema.endSchema(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.endianness.p, fieldName: "endianness", required: false, type: org_apache_arrow_flatbuf_Endianness.self) - try _v.visit(field: VTOFFSET.fields.p, fieldName: "fields", required: false, type: ForwardOffset, org_apache_arrow_flatbuf_Field>>.self) - try _v.visit(field: VTOFFSET.customMetadata.p, fieldName: "customMetadata", required: false, type: ForwardOffset, org_apache_arrow_flatbuf_KeyValue>>.self) - try _v.visit(field: VTOFFSET.features.p, fieldName: "features", required: false, type: ForwardOffset>.self) - _v.finish() - } -} - diff --git a/Sources/Arrow/Generated/SparseTensor_generated.swift b/Sources/Arrow/Generated/SparseTensor_generated.swift deleted file mode 100644 index 0a9f544..0000000 --- a/Sources/Arrow/Generated/SparseTensor_generated.swift +++ /dev/null @@ -1,530 +0,0 @@ -// swift-format-ignore-file -// Copyright 2025 The Apache Software Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - - -import FlatBuffers - -public enum org_apache_arrow_flatbuf_SparseMatrixCompressedAxis: Int16, Enum, Verifiable { - public typealias T = Int16 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int16 { return self.rawValue } - case row = 0 - case column = 1 - - public static var max: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis { return .column } - public static var min: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis { return .row } -} - - -public enum org_apache_arrow_flatbuf_SparseTensorIndex: UInt8, UnionEnum { - public typealias T = UInt8 - - public init?(value: T) { - self.init(rawValue: value) - } - - public static var byteSize: Int { return MemoryLayout.size } - public var value: UInt8 { return self.rawValue } - case none_ = 0 - case sparsetensorindexcoo = 1 - case sparsematrixindexcsx = 2 - case sparsetensorindexcsf = 3 - - public static var max: org_apache_arrow_flatbuf_SparseTensorIndex { return .sparsetensorindexcsf } - public static var min: org_apache_arrow_flatbuf_SparseTensorIndex { return .none_ } -} - - -/// ---------------------------------------------------------------------- -/// EXPERIMENTAL: Data structures for sparse tensors -/// Coordinate (COO) format of sparse tensor index. -/// -/// COO's index list are represented as a NxM matrix, -/// where N is the number of non-zero values, -/// and M is the number of dimensions of a sparse tensor. -/// -/// indicesBuffer stores the location and size of the data of this indices -/// matrix. The value type and the stride of the indices matrix is -/// specified in indicesType and indicesStrides fields. -/// -/// For example, let X be a 2x3x4x5 tensor, and it has the following -/// 6 non-zero values: -/// ```text -/// X[0, 1, 2, 0] := 1 -/// X[1, 1, 2, 3] := 2 -/// X[0, 2, 1, 0] := 3 -/// X[0, 1, 3, 0] := 4 -/// X[0, 1, 2, 1] := 5 -/// X[1, 2, 0, 4] := 6 -/// ``` -/// In COO format, the index matrix of X is the following 4x6 matrix: -/// ```text -/// [[0, 0, 0, 0, 1, 1], -/// [1, 1, 1, 2, 1, 2], -/// [2, 2, 3, 1, 2, 0], -/// [0, 1, 0, 0, 3, 4]] -/// ``` -/// When isCanonical is true, the indices is sorted in lexicographical order -/// (row-major order), and it does not have duplicated entries. Otherwise, -/// the indices may not be sorted, or may have duplicated entries. -public struct org_apache_arrow_flatbuf_SparseTensorIndexCOO: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case indicesType = 4 - case indicesStrides = 6 - case indicesBuffer = 8 - case isCanonical = 10 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// The type of values in indicesBuffer - public var indicesType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indicesType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } - /// Non-negative byte offsets to advance one value cell along each dimension - /// If omitted, default to row-major order (C-like). - public var hasIndicesStrides: Bool { let o = _accessor.offset(VTOFFSET.indicesStrides.v); return o == 0 ? false : true } - public var indicesStridesCount: Int32 { let o = _accessor.offset(VTOFFSET.indicesStrides.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func indicesStrides(at index: Int32) -> Int64 { let o = _accessor.offset(VTOFFSET.indicesStrides.v); return o == 0 ? 0 : _accessor.directRead(of: Int64.self, offset: _accessor.vector(at: o) + index * 8) } - public var indicesStrides: [Int64] { return _accessor.getVector(at: VTOFFSET.indicesStrides.v) ?? [] } - /// The location and size of the indices matrix's data - public var indicesBuffer: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.indicesBuffer.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) } - public var mutableIndicesBuffer: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.indicesBuffer.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) } - /// This flag is true if and only if the indices matrix is sorted in - /// row-major order, and does not have duplicated entries. - /// This sort order is the same as of Tensorflow's SparseTensor, - /// but it is inverse order of SciPy's canonical coo_matrix - /// (SciPy employs column-major order for its coo_matrix). - public var isCanonical: Bool { let o = _accessor.offset(VTOFFSET.isCanonical.v); return o == 0 ? false : _accessor.readBuffer(of: Bool.self, at: o) } - public static func startSparseTensorIndexCOO(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 4) } - public static func add(indicesType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indicesType, at: VTOFFSET.indicesType.p) } - public static func addVectorOf(indicesStrides: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indicesStrides, at: VTOFFSET.indicesStrides.p) } - public static func add(indicesBuffer: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder) { guard let indicesBuffer = indicesBuffer else { return }; fbb.create(struct: indicesBuffer, position: VTOFFSET.indicesBuffer.p) } - public static func add(isCanonical: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: isCanonical, def: false, - at: VTOFFSET.isCanonical.p) } - public static func endSparseTensorIndexCOO(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); fbb.require(table: end, fields: [4, 8]); return end } - public static func createSparseTensorIndexCOO( - _ fbb: inout FlatBufferBuilder, - indicesTypeOffset indicesType: Offset, - indicesStridesVectorOffset indicesStrides: Offset = Offset(), - indicesBuffer: org_apache_arrow_flatbuf_Buffer, - isCanonical: Bool = false - ) -> Offset { - let __start = org_apache_arrow_flatbuf_SparseTensorIndexCOO.startSparseTensorIndexCOO(&fbb) - org_apache_arrow_flatbuf_SparseTensorIndexCOO.add(indicesType: indicesType, &fbb) - org_apache_arrow_flatbuf_SparseTensorIndexCOO.addVectorOf(indicesStrides: indicesStrides, &fbb) - org_apache_arrow_flatbuf_SparseTensorIndexCOO.add(indicesBuffer: indicesBuffer, &fbb) - org_apache_arrow_flatbuf_SparseTensorIndexCOO.add(isCanonical: isCanonical, &fbb) - return org_apache_arrow_flatbuf_SparseTensorIndexCOO.endSparseTensorIndexCOO(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.indicesType.p, fieldName: "indicesType", required: true, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.indicesStrides.p, fieldName: "indicesStrides", required: false, type: ForwardOffset>.self) - try _v.visit(field: VTOFFSET.indicesBuffer.p, fieldName: "indicesBuffer", required: true, type: org_apache_arrow_flatbuf_Buffer.self) - try _v.visit(field: VTOFFSET.isCanonical.p, fieldName: "isCanonical", required: false, type: Bool.self) - _v.finish() - } -} - -/// Compressed Sparse format, that is matrix-specific. -public struct org_apache_arrow_flatbuf_SparseMatrixIndexCSX: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case compressedAxis = 4 - case indptrType = 6 - case indptrBuffer = 8 - case indicesType = 10 - case indicesBuffer = 12 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// Which axis, row or column, is compressed - public var compressedAxis: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis { let o = _accessor.offset(VTOFFSET.compressedAxis.v); return o == 0 ? .row : org_apache_arrow_flatbuf_SparseMatrixCompressedAxis(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .row } - /// The type of values in indptrBuffer - public var indptrType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indptrType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } - /// indptrBuffer stores the location and size of indptr array that - /// represents the range of the rows. - /// The i-th row spans from `indptr[i]` to `indptr[i+1]` in the data. - /// The length of this array is 1 + (the number of rows), and the type - /// of index value is long. - /// - /// For example, let X be the following 6x4 matrix: - /// ```text - /// X := [[0, 1, 2, 0], - /// [0, 0, 3, 0], - /// [0, 4, 0, 5], - /// [0, 0, 0, 0], - /// [6, 0, 7, 8], - /// [0, 9, 0, 0]]. - /// ``` - /// The array of non-zero values in X is: - /// ```text - /// values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9]. - /// ``` - /// And the indptr of X is: - /// ```text - /// indptr(X) = [0, 2, 3, 5, 5, 8, 10]. - /// ``` - public var indptrBuffer: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.indptrBuffer.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) } - public var mutableIndptrBuffer: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.indptrBuffer.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) } - /// The type of values in indicesBuffer - public var indicesType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indicesType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } - /// indicesBuffer stores the location and size of the array that - /// contains the column indices of the corresponding non-zero values. - /// The type of index value is long. - /// - /// For example, the indices of the above X is: - /// ```text - /// indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1]. - /// ``` - /// Note that the indices are sorted in lexicographical order for each row. - public var indicesBuffer: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.indicesBuffer.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) } - public var mutableIndicesBuffer: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.indicesBuffer.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) } - public static func startSparseMatrixIndexCSX(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) } - public static func add(compressedAxis: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis, _ fbb: inout FlatBufferBuilder) { fbb.add(element: compressedAxis.rawValue, def: 0, at: VTOFFSET.compressedAxis.p) } - public static func add(indptrType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indptrType, at: VTOFFSET.indptrType.p) } - public static func add(indptrBuffer: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder) { guard let indptrBuffer = indptrBuffer else { return }; fbb.create(struct: indptrBuffer, position: VTOFFSET.indptrBuffer.p) } - public static func add(indicesType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indicesType, at: VTOFFSET.indicesType.p) } - public static func add(indicesBuffer: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder) { guard let indicesBuffer = indicesBuffer else { return }; fbb.create(struct: indicesBuffer, position: VTOFFSET.indicesBuffer.p) } - public static func endSparseMatrixIndexCSX(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); fbb.require(table: end, fields: [6, 8, 10, 12]); return end } - public static func createSparseMatrixIndexCSX( - _ fbb: inout FlatBufferBuilder, - compressedAxis: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis = .row, - indptrTypeOffset indptrType: Offset, - indptrBuffer: org_apache_arrow_flatbuf_Buffer, - indicesTypeOffset indicesType: Offset, - indicesBuffer: org_apache_arrow_flatbuf_Buffer - ) -> Offset { - let __start = org_apache_arrow_flatbuf_SparseMatrixIndexCSX.startSparseMatrixIndexCSX(&fbb) - org_apache_arrow_flatbuf_SparseMatrixIndexCSX.add(compressedAxis: compressedAxis, &fbb) - org_apache_arrow_flatbuf_SparseMatrixIndexCSX.add(indptrType: indptrType, &fbb) - org_apache_arrow_flatbuf_SparseMatrixIndexCSX.add(indptrBuffer: indptrBuffer, &fbb) - org_apache_arrow_flatbuf_SparseMatrixIndexCSX.add(indicesType: indicesType, &fbb) - org_apache_arrow_flatbuf_SparseMatrixIndexCSX.add(indicesBuffer: indicesBuffer, &fbb) - return org_apache_arrow_flatbuf_SparseMatrixIndexCSX.endSparseMatrixIndexCSX(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.compressedAxis.p, fieldName: "compressedAxis", required: false, type: org_apache_arrow_flatbuf_SparseMatrixCompressedAxis.self) - try _v.visit(field: VTOFFSET.indptrType.p, fieldName: "indptrType", required: true, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.indptrBuffer.p, fieldName: "indptrBuffer", required: true, type: org_apache_arrow_flatbuf_Buffer.self) - try _v.visit(field: VTOFFSET.indicesType.p, fieldName: "indicesType", required: true, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.indicesBuffer.p, fieldName: "indicesBuffer", required: true, type: org_apache_arrow_flatbuf_Buffer.self) - _v.finish() - } -} - -/// Compressed Sparse Fiber (CSF) sparse tensor index. -public struct org_apache_arrow_flatbuf_SparseTensorIndexCSF: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case indptrType = 4 - case indptrBuffers = 6 - case indicesType = 8 - case indicesBuffers = 10 - case axisOrder = 12 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// CSF is a generalization of compressed sparse row (CSR) index. - /// See [smith2017knl](http://shaden.io/pub-files/smith2017knl.pdf) - /// - /// CSF index recursively compresses each dimension of a tensor into a set - /// of prefix trees. Each path from a root to leaf forms one tensor - /// non-zero index. CSF is implemented with two arrays of buffers and one - /// arrays of integers. - /// - /// For example, let X be a 2x3x4x5 tensor and let it have the following - /// 8 non-zero values: - /// ```text - /// X[0, 0, 0, 1] := 1 - /// X[0, 0, 0, 2] := 2 - /// X[0, 1, 0, 0] := 3 - /// X[0, 1, 0, 2] := 4 - /// X[0, 1, 1, 0] := 5 - /// X[1, 1, 1, 0] := 6 - /// X[1, 1, 1, 1] := 7 - /// X[1, 1, 1, 2] := 8 - /// ``` - /// As a prefix tree this would be represented as: - /// ```text - /// 0 1 - /// / \ | - /// 0 1 1 - /// / / \ | - /// 0 0 1 1 - /// /| /| | /| | - /// 1 2 0 2 0 0 1 2 - /// ``` - /// The type of values in indptrBuffers - public var indptrType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indptrType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } - /// indptrBuffers stores the sparsity structure. - /// Each two consecutive dimensions in a tensor correspond to a buffer in - /// indptrBuffers. A pair of consecutive values at `indptrBuffers[dim][i]` - /// and `indptrBuffers[dim][i + 1]` signify a range of nodes in - /// `indicesBuffers[dim + 1]` who are children of `indicesBuffers[dim][i]` node. - /// - /// For example, the indptrBuffers for the above X is: - /// ```text - /// indptrBuffer(X) = [ - /// [0, 2, 3], - /// [0, 1, 3, 4], - /// [0, 2, 4, 5, 8] - /// ]. - /// ``` - public var hasIndptrBuffers: Bool { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? false : true } - public var indptrBuffersCount: Int32 { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func indptrBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer? { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Buffer.self, offset: _accessor.vector(at: o) + index * 16) } - public func mutableIndptrBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer_Mutable? { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) } - /// The type of values in indicesBuffers - public var indicesType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indicesType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } - /// indicesBuffers stores values of nodes. - /// Each tensor dimension corresponds to a buffer in indicesBuffers. - /// For example, the indicesBuffers for the above X is: - /// ```text - /// indicesBuffer(X) = [ - /// [0, 1], - /// [0, 1, 1], - /// [0, 0, 1, 1], - /// [1, 2, 0, 2, 0, 0, 1, 2] - /// ]. - /// ``` - public var hasIndicesBuffers: Bool { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? false : true } - public var indicesBuffersCount: Int32 { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func indicesBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer? { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Buffer.self, offset: _accessor.vector(at: o) + index * 16) } - public func mutableIndicesBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer_Mutable? { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) } - /// axisOrder stores the sequence in which dimensions were traversed to - /// produce the prefix tree. - /// For example, the axisOrder for the above X is: - /// ```text - /// axisOrder(X) = [0, 1, 2, 3]. - /// ``` - public var hasAxisOrder: Bool { let o = _accessor.offset(VTOFFSET.axisOrder.v); return o == 0 ? false : true } - public var axisOrderCount: Int32 { let o = _accessor.offset(VTOFFSET.axisOrder.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func axisOrder(at index: Int32) -> Int32 { let o = _accessor.offset(VTOFFSET.axisOrder.v); return o == 0 ? 0 : _accessor.directRead(of: Int32.self, offset: _accessor.vector(at: o) + index * 4) } - public var axisOrder: [Int32] { return _accessor.getVector(at: VTOFFSET.axisOrder.v) ?? [] } - public static func startSparseTensorIndexCSF(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) } - public static func add(indptrType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indptrType, at: VTOFFSET.indptrType.p) } - public static func addVectorOf(indptrBuffers: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indptrBuffers, at: VTOFFSET.indptrBuffers.p) } - public static func startVectorOfIndptrBuffers(_ size: Int, in builder: inout FlatBufferBuilder) { - builder.startVector(size * MemoryLayout.size, elementSize: MemoryLayout.alignment) - } - public static func add(indicesType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indicesType, at: VTOFFSET.indicesType.p) } - public static func addVectorOf(indicesBuffers: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indicesBuffers, at: VTOFFSET.indicesBuffers.p) } - public static func startVectorOfIndicesBuffers(_ size: Int, in builder: inout FlatBufferBuilder) { - builder.startVector(size * MemoryLayout.size, elementSize: MemoryLayout.alignment) - } - public static func addVectorOf(axisOrder: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: axisOrder, at: VTOFFSET.axisOrder.p) } - public static func endSparseTensorIndexCSF(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); fbb.require(table: end, fields: [4, 6, 8, 10, 12]); return end } - public static func createSparseTensorIndexCSF( - _ fbb: inout FlatBufferBuilder, - indptrTypeOffset indptrType: Offset, - indptrBuffersVectorOffset indptrBuffers: Offset, - indicesTypeOffset indicesType: Offset, - indicesBuffersVectorOffset indicesBuffers: Offset, - axisOrderVectorOffset axisOrder: Offset - ) -> Offset { - let __start = org_apache_arrow_flatbuf_SparseTensorIndexCSF.startSparseTensorIndexCSF(&fbb) - org_apache_arrow_flatbuf_SparseTensorIndexCSF.add(indptrType: indptrType, &fbb) - org_apache_arrow_flatbuf_SparseTensorIndexCSF.addVectorOf(indptrBuffers: indptrBuffers, &fbb) - org_apache_arrow_flatbuf_SparseTensorIndexCSF.add(indicesType: indicesType, &fbb) - org_apache_arrow_flatbuf_SparseTensorIndexCSF.addVectorOf(indicesBuffers: indicesBuffers, &fbb) - org_apache_arrow_flatbuf_SparseTensorIndexCSF.addVectorOf(axisOrder: axisOrder, &fbb) - return org_apache_arrow_flatbuf_SparseTensorIndexCSF.endSparseTensorIndexCSF(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.indptrType.p, fieldName: "indptrType", required: true, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.indptrBuffers.p, fieldName: "indptrBuffers", required: true, type: ForwardOffset>.self) - try _v.visit(field: VTOFFSET.indicesType.p, fieldName: "indicesType", required: true, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.indicesBuffers.p, fieldName: "indicesBuffers", required: true, type: ForwardOffset>.self) - try _v.visit(field: VTOFFSET.axisOrder.p, fieldName: "axisOrder", required: true, type: ForwardOffset>.self) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_SparseTensor: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case typeType = 4 - case type = 6 - case shape = 8 - case nonZeroLength = 10 - case sparseIndexType = 12 - case sparseIndex = 14 - case data = 16 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var typeType: org_apache_arrow_flatbuf_Type_ { let o = _accessor.offset(VTOFFSET.typeType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_Type_(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ } - /// The type of data contained in a value cell. - /// Currently only fixed-width value types are supported, - /// no strings or nested types. - public func type(type: T.Type) -> T! { let o = _accessor.offset(VTOFFSET.type.v); return _accessor.union(o) } - /// The dimensions of the tensor, optionally named. - public var hasShape: Bool { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? false : true } - public var shapeCount: Int32 { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func shape(at index: Int32) -> org_apache_arrow_flatbuf_TensorDim? { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? nil : org_apache_arrow_flatbuf_TensorDim(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } - /// The number of non-zero values in a sparse tensor. - public var nonZeroLength: Int64 { let o = _accessor.offset(VTOFFSET.nonZeroLength.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) } - public var sparseIndexType: org_apache_arrow_flatbuf_SparseTensorIndex { let o = _accessor.offset(VTOFFSET.sparseIndexType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_SparseTensorIndex(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ } - /// Sparse tensor index - public func sparseIndex(type: T.Type) -> T! { let o = _accessor.offset(VTOFFSET.sparseIndex.v); return _accessor.union(o) } - /// The location and size of the tensor's data - public var data: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.data.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) } - public var mutableData: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.data.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) } - public static func startSparseTensor(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 7) } - public static func add(typeType: org_apache_arrow_flatbuf_Type_, _ fbb: inout FlatBufferBuilder) { fbb.add(element: typeType.rawValue, def: 0, at: VTOFFSET.typeType.p) } - public static func add(type: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: type, at: VTOFFSET.type.p) } - public static func addVectorOf(shape: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: shape, at: VTOFFSET.shape.p) } - public static func add(nonZeroLength: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: nonZeroLength, def: 0, at: VTOFFSET.nonZeroLength.p) } - public static func add(sparseIndexType: org_apache_arrow_flatbuf_SparseTensorIndex, _ fbb: inout FlatBufferBuilder) { fbb.add(element: sparseIndexType.rawValue, def: 0, at: VTOFFSET.sparseIndexType.p) } - public static func add(sparseIndex: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: sparseIndex, at: VTOFFSET.sparseIndex.p) } - public static func add(data: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder) { guard let data = data else { return }; fbb.create(struct: data, position: VTOFFSET.data.p) } - public static func endSparseTensor(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); fbb.require(table: end, fields: [6, 8, 14, 16]); return end } - public static func createSparseTensor( - _ fbb: inout FlatBufferBuilder, - typeType: org_apache_arrow_flatbuf_Type_ = .none_, - typeOffset type: Offset, - shapeVectorOffset shape: Offset, - nonZeroLength: Int64 = 0, - sparseIndexType: org_apache_arrow_flatbuf_SparseTensorIndex = .none_, - sparseIndexOffset sparseIndex: Offset, - data: org_apache_arrow_flatbuf_Buffer - ) -> Offset { - let __start = org_apache_arrow_flatbuf_SparseTensor.startSparseTensor(&fbb) - org_apache_arrow_flatbuf_SparseTensor.add(typeType: typeType, &fbb) - org_apache_arrow_flatbuf_SparseTensor.add(type: type, &fbb) - org_apache_arrow_flatbuf_SparseTensor.addVectorOf(shape: shape, &fbb) - org_apache_arrow_flatbuf_SparseTensor.add(nonZeroLength: nonZeroLength, &fbb) - org_apache_arrow_flatbuf_SparseTensor.add(sparseIndexType: sparseIndexType, &fbb) - org_apache_arrow_flatbuf_SparseTensor.add(sparseIndex: sparseIndex, &fbb) - org_apache_arrow_flatbuf_SparseTensor.add(data: data, &fbb) - return org_apache_arrow_flatbuf_SparseTensor.endSparseTensor(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(unionKey: VTOFFSET.typeType.p, unionField: VTOFFSET.type.p, unionKeyName: "typeType", fieldName: "type", required: true, completion: { (verifier, key: org_apache_arrow_flatbuf_Type_, pos) in - switch key { - case .none_: - break // NOTE - SWIFT doesnt support none - case .null: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Null.self) - case .int: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Int.self) - case .floatingpoint: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FloatingPoint.self) - case .binary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Binary.self) - case .utf8: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8.self) - case .bool: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Bool.self) - case .decimal: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Decimal.self) - case .date: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Date.self) - case .time: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Time.self) - case .timestamp: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Timestamp.self) - case .interval: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Interval.self) - case .list: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_List.self) - case .struct_: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Struct_.self) - case .union: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Union.self) - case .fixedsizebinary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeBinary.self) - case .fixedsizelist: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeList.self) - case .map: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Map.self) - case .duration: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Duration.self) - case .largebinary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeBinary.self) - case .largeutf8: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeUtf8.self) - case .largelist: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeList.self) - case .runendencoded: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_RunEndEncoded.self) - case .binaryview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_BinaryView.self) - case .utf8view: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8View.self) - case .listview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_ListView.self) - case .largelistview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeListView.self) - } - }) - try _v.visit(field: VTOFFSET.shape.p, fieldName: "shape", required: true, type: ForwardOffset, org_apache_arrow_flatbuf_TensorDim>>.self) - try _v.visit(field: VTOFFSET.nonZeroLength.p, fieldName: "nonZeroLength", required: false, type: Int64.self) - try _v.visit(unionKey: VTOFFSET.sparseIndexType.p, unionField: VTOFFSET.sparseIndex.p, unionKeyName: "sparseIndexType", fieldName: "sparseIndex", required: true, completion: { (verifier, key: org_apache_arrow_flatbuf_SparseTensorIndex, pos) in - switch key { - case .none_: - break // NOTE - SWIFT doesnt support none - case .sparsetensorindexcoo: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_SparseTensorIndexCOO.self) - case .sparsematrixindexcsx: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_SparseMatrixIndexCSX.self) - case .sparsetensorindexcsf: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_SparseTensorIndexCSF.self) - } - }) - try _v.visit(field: VTOFFSET.data.p, fieldName: "data", required: true, type: org_apache_arrow_flatbuf_Buffer.self) - _v.finish() - } -} - diff --git a/Sources/Arrow/Generated/Tensor_generated.swift b/Sources/Arrow/Generated/Tensor_generated.swift deleted file mode 100644 index 4e6d61f..0000000 --- a/Sources/Arrow/Generated/Tensor_generated.swift +++ /dev/null @@ -1,192 +0,0 @@ -// swift-format-ignore-file -// Copyright 2025 The Apache Software Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - - -import FlatBuffers - -/// ---------------------------------------------------------------------- -/// Data structures for dense tensors -/// Shape data for a single axis in a tensor -public struct org_apache_arrow_flatbuf_TensorDim: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case size = 4 - case name = 6 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - /// Length of dimension - public var size: Int64 { let o = _accessor.offset(VTOFFSET.size.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) } - /// Name of the dimension, optional - public var name: String? { let o = _accessor.offset(VTOFFSET.name.v); return o == 0 ? nil : _accessor.string(at: o) } - public var nameSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.name.v) } - public static func startTensorDim(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) } - public static func add(size: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: size, def: 0, at: VTOFFSET.size.p) } - public static func add(name: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: name, at: VTOFFSET.name.p) } - public static func endTensorDim(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } - public static func createTensorDim( - _ fbb: inout FlatBufferBuilder, - size: Int64 = 0, - nameOffset name: Offset = Offset() - ) -> Offset { - let __start = org_apache_arrow_flatbuf_TensorDim.startTensorDim(&fbb) - org_apache_arrow_flatbuf_TensorDim.add(size: size, &fbb) - org_apache_arrow_flatbuf_TensorDim.add(name: name, &fbb) - return org_apache_arrow_flatbuf_TensorDim.endTensorDim(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.size.p, fieldName: "size", required: false, type: Int64.self) - try _v.visit(field: VTOFFSET.name.p, fieldName: "name", required: false, type: ForwardOffset.self) - _v.finish() - } -} - -public struct org_apache_arrow_flatbuf_Tensor: FlatBufferObject, Verifiable { - - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } - private var _accessor: Table - - private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } - - private enum VTOFFSET: VOffset { - case typeType = 4 - case type = 6 - case shape = 8 - case strides = 10 - case data = 12 - var v: Int32 { Int32(self.rawValue) } - var p: VOffset { self.rawValue } - } - - public var typeType: org_apache_arrow_flatbuf_Type_ { let o = _accessor.offset(VTOFFSET.typeType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_Type_(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ } - /// The type of data contained in a value cell. Currently only fixed-width - /// value types are supported, no strings or nested types - public func type(type: T.Type) -> T! { let o = _accessor.offset(VTOFFSET.type.v); return _accessor.union(o) } - /// The dimensions of the tensor, optionally named - public var hasShape: Bool { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? false : true } - public var shapeCount: Int32 { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func shape(at index: Int32) -> org_apache_arrow_flatbuf_TensorDim? { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? nil : org_apache_arrow_flatbuf_TensorDim(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } - /// Non-negative byte offsets to advance one value cell along each dimension - /// If omitted, default to row-major order (C-like). - public var hasStrides: Bool { let o = _accessor.offset(VTOFFSET.strides.v); return o == 0 ? false : true } - public var stridesCount: Int32 { let o = _accessor.offset(VTOFFSET.strides.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func strides(at index: Int32) -> Int64 { let o = _accessor.offset(VTOFFSET.strides.v); return o == 0 ? 0 : _accessor.directRead(of: Int64.self, offset: _accessor.vector(at: o) + index * 8) } - public var strides: [Int64] { return _accessor.getVector(at: VTOFFSET.strides.v) ?? [] } - /// The location and size of the tensor's data - public var data: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.data.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) } - public var mutableData: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.data.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) } - public static func startTensor(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) } - public static func add(typeType: org_apache_arrow_flatbuf_Type_, _ fbb: inout FlatBufferBuilder) { fbb.add(element: typeType.rawValue, def: 0, at: VTOFFSET.typeType.p) } - public static func add(type: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: type, at: VTOFFSET.type.p) } - public static func addVectorOf(shape: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: shape, at: VTOFFSET.shape.p) } - public static func addVectorOf(strides: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: strides, at: VTOFFSET.strides.p) } - public static func add(data: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder) { guard let data = data else { return }; fbb.create(struct: data, position: VTOFFSET.data.p) } - public static func endTensor(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); fbb.require(table: end, fields: [6, 8, 12]); return end } - public static func createTensor( - _ fbb: inout FlatBufferBuilder, - typeType: org_apache_arrow_flatbuf_Type_ = .none_, - typeOffset type: Offset, - shapeVectorOffset shape: Offset, - stridesVectorOffset strides: Offset = Offset(), - data: org_apache_arrow_flatbuf_Buffer - ) -> Offset { - let __start = org_apache_arrow_flatbuf_Tensor.startTensor(&fbb) - org_apache_arrow_flatbuf_Tensor.add(typeType: typeType, &fbb) - org_apache_arrow_flatbuf_Tensor.add(type: type, &fbb) - org_apache_arrow_flatbuf_Tensor.addVectorOf(shape: shape, &fbb) - org_apache_arrow_flatbuf_Tensor.addVectorOf(strides: strides, &fbb) - org_apache_arrow_flatbuf_Tensor.add(data: data, &fbb) - return org_apache_arrow_flatbuf_Tensor.endTensor(&fbb, start: __start) - } - - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - var _v = try verifier.visitTable(at: position) - try _v.visit(unionKey: VTOFFSET.typeType.p, unionField: VTOFFSET.type.p, unionKeyName: "typeType", fieldName: "type", required: true, completion: { (verifier, key: org_apache_arrow_flatbuf_Type_, pos) in - switch key { - case .none_: - break // NOTE - SWIFT doesnt support none - case .null: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Null.self) - case .int: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Int.self) - case .floatingpoint: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FloatingPoint.self) - case .binary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Binary.self) - case .utf8: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8.self) - case .bool: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Bool.self) - case .decimal: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Decimal.self) - case .date: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Date.self) - case .time: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Time.self) - case .timestamp: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Timestamp.self) - case .interval: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Interval.self) - case .list: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_List.self) - case .struct_: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Struct_.self) - case .union: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Union.self) - case .fixedsizebinary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeBinary.self) - case .fixedsizelist: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeList.self) - case .map: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Map.self) - case .duration: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Duration.self) - case .largebinary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeBinary.self) - case .largeutf8: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeUtf8.self) - case .largelist: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeList.self) - case .runendencoded: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_RunEndEncoded.self) - case .binaryview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_BinaryView.self) - case .utf8view: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8View.self) - case .listview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_ListView.self) - case .largelistview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeListView.self) - } - }) - try _v.visit(field: VTOFFSET.shape.p, fieldName: "shape", required: true, type: ForwardOffset, org_apache_arrow_flatbuf_TensorDim>>.self) - try _v.visit(field: VTOFFSET.strides.p, fieldName: "strides", required: false, type: ForwardOffset>.self) - try _v.visit(field: VTOFFSET.data.p, fieldName: "data", required: true, type: org_apache_arrow_flatbuf_Buffer.self) - _v.finish() - } -} - diff --git a/Sources/ArrowIPC/ArrowBufferIPC.swift b/Sources/ArrowIPC/ArrowBufferIPC.swift index f450a73..7e763c2 100644 --- a/Sources/ArrowIPC/ArrowBufferIPC.swift +++ b/Sources/ArrowIPC/ArrowBufferIPC.swift @@ -44,7 +44,6 @@ struct NullBufferIPC: NullBuffer, ArrowBufferIPC { func isSet(_ bit: Int) -> Bool { precondition(bit < valueCount, "Bit index \(bit) out of range") let byteIndex = bit / 8 - // precondition(length > byteIndex, "Bit index \(bit) out of range") let offsetIndex = buffer.range.lowerBound + byteIndex let byte = self.buffer.data[offsetIndex] return byte & (1 << (bit % 8)) > 0 diff --git a/Sources/ArrowIPC/ArrowField+IPC.swift b/Sources/ArrowIPC/ArrowField+IPC.swift index 679fcc4..abd8d82 100644 --- a/Sources/ArrowIPC/ArrowField+IPC.swift +++ b/Sources/ArrowIPC/ArrowField+IPC.swift @@ -36,12 +36,11 @@ extension ArrowField { guard let fieldName = field.name else { throw .init(.invalid("Field name not found")) } - let fieldMetadata = (0.. RecordBatch { + + // MARK: Load arrays + var arrays: [AnyArrowArrayProtocol] = .init() + var nodeIndex: Int32 = 0 + var bufferIndex: Int32 = 0 + var variadicBufferIndex: Int32 = 0 + + for field in arrowSchema.fields { + + let array = try Self.loadField( + data: data, + rbMessage: rbMessage, + field: field, + offset: offset, + nodeIndex: &nodeIndex, + bufferIndex: &bufferIndex, + variadicBufferIndex: &variadicBufferIndex + ) + arrays.append(array) + } + + return .init(schema: arrowSchema, columns: arrays) + } + + static func loadField( + data: Data, rbMessage: FRecordBatch, field: ArrowField, offset: Int64, @@ -157,8 +174,8 @@ public struct ArrowReader { bufferIndex: inout Int32, variadicBufferIndex: inout Int32, ) throws(ArrowError) -> AnyArrowArrayProtocol { - guard nodeIndex < rbMessage.nodesCount, - let node = rbMessage.nodes(at: nodeIndex) + guard nodeIndex < rbMessage.nodes.count, + let node = rbMessage.nodes[ifInBounds: nodeIndex] else { throw ArrowError(.invalid("Missing node at index \(nodeIndex)")) } @@ -172,26 +189,18 @@ public struct ArrowReader { // MARK: Load arrays let nullCount = Int(node.nullCount) + if nullCount > 0 && !field.isNullable { + print("Nullability violated for field \(field.name)") + } let length = Int(node.length) - let nullsPresent = nullCount > 0 let nullBuffer: NullBuffer - if nullsPresent { - if nullCount == 0 { - nullBuffer = AllValidNullBuffer(valueCount: length) - } else if length == 0 { - nullBuffer = AllValidNullBuffer(valueCount: 0) - } else if nullCount == length { - nullBuffer = AllNullBuffer(valueCount: length) - } else { - nullBuffer = NullBufferIPC( - buffer: buffer0, valueCount: length, nullCount: nullCount) - } - } else { + if nullCount == 0 { nullBuffer = AllValidNullBuffer(valueCount: length) - } - - if nullsPresent && !field.isNullable { - print("Nullabity violated for field \(field.name)") + } else if nullCount == length { + nullBuffer = AllNullBuffer(valueCount: length) + } else { + nullBuffer = NullBufferIPC( + buffer: buffer0, valueCount: length, nullCount: nullCount) } let arrowType = field.type @@ -248,7 +257,7 @@ public struct ArrowReader { length: length, elementType: UInt64.self, nullBuffer: nullBuffer, buffer: buffer1) default: - throw .init(.notImplemented) + throw .init(.notImplemented("\(arrowType)")) } } else if arrowType.isTemporal { let buffer1 = try nextBuffer( @@ -283,17 +292,17 @@ public struct ArrowReader { length: length, elementType: Int32.self, nullBuffer: nullBuffer, buffer: buffer1) default: - throw ArrowError(.notImplemented) + throw .init(.notImplemented("\(arrowType)")) } } else if arrowType.isVariable { let buffer1 = try nextBuffer( message: rbMessage, index: &bufferIndex, offset: offset, data: data) let buffer2 = try nextBuffer( message: rbMessage, index: &bufferIndex, offset: offset, data: data) - let offsetsBufferTyped = FixedWidthBufferIPC( - buffer: buffer1 - ) if arrowType == .utf8 { + let offsetsBufferTyped = FixedWidthBufferIPC( + buffer: buffer1 + ) let valueBufferTyped = VariableLengthBufferIPC( buffer: buffer2 ) @@ -304,6 +313,9 @@ public struct ArrowReader { valueBuffer: valueBufferTyped ) } else if arrowType == .binary { + let offsetsBufferTyped = FixedWidthBufferIPC( + buffer: buffer1 + ) let valueBufferTyped = VariableLengthBufferIPC( buffer: buffer2) return ArrowArrayVariable( @@ -312,8 +324,32 @@ public struct ArrowReader { offsetsBuffer: offsetsBufferTyped, valueBuffer: valueBufferTyped ) + } else if arrowType == .largeBinary { + let offsetsBufferTyped = FixedWidthBufferIPC( + buffer: buffer1 + ) + let valueBufferTyped = VariableLengthBufferIPC( + buffer: buffer2) + return ArrowArrayVariable( + length: length, + nullBuffer: nullBuffer, + offsetsBuffer: offsetsBufferTyped, + valueBuffer: valueBufferTyped + ) + } else if arrowType == .largeUtf8 { + let offsetsBufferTyped = FixedWidthBufferIPC( + buffer: buffer1 + ) + let valueBufferTyped = VariableLengthBufferIPC( + buffer: buffer2) + return ArrowArrayVariable( + length: length, + nullBuffer: nullBuffer, + offsetsBuffer: offsetsBufferTyped, + valueBuffer: valueBufferTyped + ) } else { - throw .init(.notImplemented) + throw .init(.notImplemented("\(arrowType)")) } } else if arrowType.isBinaryView { let viewsBuffer = try nextBuffer( @@ -325,8 +361,12 @@ public struct ArrowReader { let viewsBufferTyped = FixedWidthBufferIPC( buffer: viewsBuffer) - let variadicCount = rbMessage.variadicBufferCounts( - at: variadicBufferIndex) + guard + let variadicCount = rbMessage.variadicBufferCounts[ + ifInBounds: variadicBufferIndex] + else { + throw .init(.invalid("Unable to get variadic buffer count.")) + } variadicBufferIndex += 1 switch arrowType { @@ -372,64 +412,38 @@ public struct ArrowReader { dataBuffers: dataBuffers ) default: - throw .init(.notImplemented) + throw .init(.notImplemented("\(arrowType)")) } } else if arrowType.isNested { switch arrowType { case .list(let childField), .map(let childField, _): // A map is simply a list of struct items. - let buffer1 = try nextBuffer( - message: rbMessage, index: &bufferIndex, offset: offset, data: data) - var offsetsBuffer = FixedWidthBufferIPC( - buffer: buffer1 - ) - - let array: AnyArrowArrayProtocol = try loadField( + return try loadListArray( + offsetType: Int32.self, rbMessage: rbMessage, - field: childField, - offset: offset, - nodeIndex: &nodeIndex, bufferIndex: &bufferIndex, - variadicBufferIndex: &variadicBufferIndex + offset: offset, data: data, + childField: childField, + nodeIndex: &nodeIndex, + variadicBufferIndex: &variadicBufferIndex, + length: length, + nullBuffer: nullBuffer ) - - if offsetsBuffer.length == 0 { - // Empty offsets buffer is valid when child array is empty - // There could be any number of empty lists referencing into an empty list - guard array.length == 0 else { - throw .init( - .invalid("Empty offsets buffer but non-empty child array") - ) - } - let emptyBuffer = emptyOffsetBuffer(offsetCount: length + 1) - offsetsBuffer = FixedWidthBufferIPC( - buffer: emptyBuffer - ) - } else { - let requiredBytes = (length + 1) * MemoryLayout.stride - guard offsetsBuffer.length >= requiredBytes else { - throw ArrowError( - .invalid( - "Offsets buffer of length: \(offsetsBuffer.length) too small: need \(requiredBytes) bytes for \(length) lists" - ) - ) - } - // Verify last offset matches child array length - let lastOffset = offsetsBuffer[length] - guard lastOffset == Int32(array.length) else { - throw ArrowError( - .invalid( - "Expected last offset to match child array length.")) - } - } - return makeListArray( + case .largeList(let childField): + return try loadListArray( + offsetType: Int64.self, + rbMessage: rbMessage, + bufferIndex: &bufferIndex, + offset: offset, data: data, + childField: childField, + nodeIndex: &nodeIndex, + variadicBufferIndex: &variadicBufferIndex, length: length, - nullBuffer: nullBuffer, - offsetsBuffer: offsetsBuffer, - values: array + nullBuffer: nullBuffer ) case .fixedSizeList(let field, let listSize): let array: AnyArrowArrayProtocol = try loadField( + data: data, rbMessage: rbMessage, field: field, offset: offset, @@ -447,6 +461,7 @@ public struct ArrowReader { var arrays: [(String, AnyArrowArrayProtocol)] = [] for field in fields { let array = try loadField( + data: data, rbMessage: rbMessage, field: field, offset: offset, @@ -462,7 +477,7 @@ public struct ArrowReader { fields: arrays ) default: - throw ArrowError(.notImplemented) + throw .init(.notImplemented("\(arrowType)")) } } else { // MARK: Unclassifiable types. @@ -478,21 +493,22 @@ public struct ArrowReader { valueBuffer: valueBufferTyped ) } - throw ArrowError(.notImplemented) + throw .init(.notImplemented("\(arrowType)")) } } - func nextBuffer( + static func nextBuffer( message: FRecordBatch, index: inout Int32, offset: Int64, data: Data ) throws(ArrowError) -> FileDataBuffer { - guard index < message.buffersCount, let buffer = message.buffers(at: index) + guard index < message.buffers.count, + let buffer = message.buffers[ifInBounds: index] else { throw .init( .invalid( - "Buffer index \(index) requested for message with \(message.buffersCount) buffers." + "Buffer index \(index) requested for message with \(message.buffers.count) buffers." )) } index += 1 @@ -503,7 +519,7 @@ public struct ArrowReader { return fileDataBuffer } - func makeFixedArray( + static func makeFixedArray( length: Int, elementType: T.Type, nullBuffer: NullBuffer, @@ -519,13 +535,61 @@ public struct ArrowReader { ) } - func makeListArray( + private static func loadListArray< + T: FixedWidthInteger & BitwiseCopyable & SignedInteger + >( + offsetType: T.Type, + rbMessage: FRecordBatch, + bufferIndex: inout Int32, + offset: Int64, + data: Data, + childField: ArrowField, + nodeIndex: inout Int32, + variadicBufferIndex: inout Int32, + length: Int, + nullBuffer: any NullBuffer + ) throws(ArrowError) -> any AnyArrowArrayProtocol { + let buffer1 = try nextBuffer( + message: rbMessage, index: &bufferIndex, offset: offset, data: data) + let offsetsBuffer = FixedWidthBufferIPC(buffer: buffer1) + let array: AnyArrowArrayProtocol = try loadField( + data: data, + rbMessage: rbMessage, + field: childField, + offset: offset, + nodeIndex: &nodeIndex, + bufferIndex: &bufferIndex, + variadicBufferIndex: &variadicBufferIndex + ) + return try makeListArray( + length: length, + nullBuffer: nullBuffer, + offsetsBuffer: offsetsBuffer, + values: array + ) + } + + private static func makeListArray( length: Int, nullBuffer: NullBuffer, offsetsBuffer: any FixedWidthBufferProtocol, values: AnyArrowArrayProtocol - ) -> ArrowListArray { - ArrowListArray( + ) throws(ArrowError) -> ArrowListArray { + let requiredBytes = (length + 1) * MemoryLayout.stride + guard offsetsBuffer.length >= requiredBytes else { + throw ArrowError( + .invalid( + "Offsets buffer of length: \(offsetsBuffer.length) too small: need \(requiredBytes) bytes for \(length) lists" + ) + ) + } + // Verify last offset matches child array length + let lastOffset = offsetsBuffer[length] + guard lastOffset == values.length else { + throw ArrowError( + .invalid("Expected last offset to match child array length.")) + } + return ArrowListArray( length: length, nullBuffer: nullBuffer, offsetsBuffer: offsetsBuffer, @@ -533,33 +597,18 @@ public struct ArrowReader { ) } - private func loadSchema(schema: FSchema) throws(ArrowError) -> ArrowSchema { - let metadata = (0.. ArrowSchema { + let metadata = schema.customMetadata.reduce(into: [String: String]()) { + dict, kv in + guard let key = kv.key else { return } + dict[key] = kv.value + } var fields: [ArrowField] = [] - for index in 0.. FileDataBuffer { - let byteCount = offsetCount * MemoryLayout.stride - return FileDataBuffer( - data: Data(count: byteCount), // Zero-initialized - range: 0.. FMessage? { + + let message: FMessage? = try data.withParserSpan { input in + try input.seek(toAbsoluteOffset: offset) + + let header = try MessageHeader(parsing: &input) + let messageLength = header.metadataLength + + if messageLength == 0 { + return nil + } + let data = try [UInt8](parsing: &input, byteCount: Int(messageLength)) + + // TODO: Not zero-copy. Maybe new API fixes this. + var mbb = ByteBuffer(data: Data(data)) + let message: FMessage = getRoot(byteBuffer: &mbb) + offset = Int64(input.startPosition) + return message + } + return message + } + + func read(data: Data) throws -> [RecordBatch] { + var offset: Int64 = 0 + var recordBatches: [RecordBatch] = [] + while true { + let message = try readMessage(data: data, offset: &offset) + guard let message else { + return recordBatches + } + switch message.headerType { + case .schema: + guard let sMessage = message.header(type: FSchema.self) else { + throw ArrowError(.invalid("Expected RecordBatch as message header")) + } + self.arrowSchema = try ArrowReader.loadSchema(schema: sMessage) + case .recordbatch: + guard let rbMessage = message.header(type: FRecordBatch.self) else { + throw ArrowError(.invalid("Expected RecordBatch as message header")) + } + guard let arrowSchema else { + throw ArrowError(.invalid("ArrowSchema not available.")) + } + let recordBatch = try ArrowReader.loadRecordBatch( + data: data, + arrowSchema: arrowSchema, + rbMessage: rbMessage, + offset: offset, + ) + offset += message.bodyLength + recordBatches.append(recordBatch) + default: + throw ArrowError(.notImplemented("Unexpected message header type.")) + } + } + return recordBatches + } +} diff --git a/Sources/ArrowIPC/ArrowWriter+flatbuffers.swift b/Sources/ArrowIPC/ArrowWriter+flatbuffers.swift index d274004..b542037 100644 --- a/Sources/ArrowIPC/ArrowWriter+flatbuffers.swift +++ b/Sources/ArrowIPC/ArrowWriter+flatbuffers.swift @@ -57,6 +57,9 @@ extension ArrowWriter { } else if case .list(let childField) = field.type { let offset = try write(field: childField, to: &fbb) fieldsOffset = fbb.createVector(ofOffsets: [offset]) + } else if case .largeList(let childField) = field.type { + let offset = try write(field: childField, to: &fbb) + fieldsOffset = fbb.createVector(ofOffsets: [offset]) } else if case .fixedSizeList(let childField, _) = field.type { let offset = try write(field: childField, to: &fbb) fieldsOffset = fbb.createVector(ofOffsets: [offset]) @@ -112,10 +115,16 @@ extension ArrowWriter { return FFloatingPoint.createFloatingPoint(&fbb, precision: .double) case .utf8: return FUtf8.endUtf8(&fbb, start: FUtf8.startUtf8(&fbb)) + case .largeUtf8: + return FLargeUtf8.endLargeUtf8( + &fbb, start: FLargeUtf8.startLargeUtf8(&fbb)) case .utf8View: return FUtf8View.endUtf8View(&fbb, start: FUtf8View.startUtf8View(&fbb)) case .binary: return FBinary.endBinary(&fbb, start: FBinary.startBinary(&fbb)) + case .largeBinary: + return FLargeBinary.endLargeBinary( + &fbb, start: FLargeBinary.startLargeBinary(&fbb)) case .binaryView: return FBinaryView.endBinaryView( &fbb, start: FBinaryView.startBinaryView(&fbb)) @@ -173,6 +182,9 @@ extension ArrowWriter { case .list: let startOffset = FList.startList(&fbb) return FList.endList(&fbb, start: startOffset) + case .largeList: + let startOffset = FLargeList.startLargeList(&fbb) + return FLargeList.endLargeList(&fbb, start: startOffset) case .fixedSizeList(_, let listSize): let startOffset = FFixedSizeList.startFixedSizeList(&fbb) FFixedSizeList.add(listSize: listSize, &fbb) diff --git a/Sources/ArrowIPC/ArrowWriter.swift b/Sources/ArrowIPC/ArrowWriter.swift index fbc59a1..70dfe30 100644 --- a/Sources/ArrowIPC/ArrowWriter.swift +++ b/Sources/ArrowIPC/ArrowWriter.swift @@ -208,33 +208,36 @@ public struct ArrowWriter { pad() precondition(data.count % 8 == 0, "Data size must be multiple of 8") } - switch field.type { - case .strct(let fields): - guard let structArray = array as? ArrowStructArray else { - throw .init(.invalid("Struct type array expected for nested type")) - } - try writeRecordBatchData( - fields: fields, - arrays: structArray.fields.map(\.array) - ) - case .list(let childType), .map(let childType, _): - guard let listArray = array as? ArrowListArray else { - throw .init(.invalid("List type array expected.")) - } - try writeRecordBatchData( - fields: [childType], - arrays: [listArray.values] - ) - case .fixedSizeList(let childType, _): - guard let listArray = array as? ArrowFixedSizeListArray else { - throw .init(.invalid("Fixed size list type array expected.")) + if field.type.isNested { + switch field.type { + case .strct(let fields): + guard let structArray = array as? ArrowStructArray else { + throw .init(.invalid("Struct type array expected for nested type")) + } + try writeRecordBatchData( + fields: fields, + arrays: structArray.fields.map(\.array) + ) + case .list(let childType), .largeList(let childType), + .map(let childType, _): + guard let listArray = array as? ListArrayProtocol else { + throw .init(.invalid("List type array expected.")) + } + try writeRecordBatchData( + fields: [childType], + arrays: [listArray.values] + ) + case .fixedSizeList(let childType, _): + guard let listArray = array as? ArrowFixedSizeListArray else { + throw .init(.invalid("Fixed size list type array expected.")) + } + try writeRecordBatchData( + fields: [childType], + arrays: [listArray.values] + ) + default: + throw .init(.notImplemented("\(field.type)")) } - try writeRecordBatchData( - fields: [childType], - arrays: [listArray.values] - ) - default: - break } } } @@ -251,7 +254,7 @@ public struct ArrowWriter { // MARK: Field nodes. var fieldNodes: [FFieldNode] = [] - writeFieldNodes( + try writeFieldNodes( fields: schema.fields, arrays: batch.arrays, nodes: &fieldNodes, @@ -299,11 +302,12 @@ public struct ArrowWriter { /// - fields: The Arrow fields. /// - arrays: The Arrow arrays. /// - nodes: The field node array being written to. + /// - Throws: An `ArrowError` if a field type is not handled. private func writeFieldNodes( fields: [ArrowField], arrays: [AnyArrowArrayProtocol], nodes: inout [FFieldNode] - ) { + ) throws(ArrowError) { for index in 0.. { - writeFieldNodes( - fields: [childField], - arrays: [column.values], - nodes: &nodes, - ) - } - case .fixedSizeList(let childField, _): - if let column = column as? ArrowFixedSizeListArray { - writeFieldNodes( - fields: [childField], - arrays: [column.values], - nodes: &nodes, - ) + if field.type.isNested { + switch field.type { + case .strct(let fields): + if let column = column as? ArrowStructArray { + try writeFieldNodes( + fields: fields, + arrays: column.fields.map(\.array), + nodes: &nodes, + ) + } + case .list(let childField), .largeList(let childField), + .map(let childField, _): + if let column = column as? ListArrayProtocol { + try writeFieldNodes( + fields: [childField], + arrays: [column.values], + nodes: &nodes, + ) + } + case .fixedSizeList(let childField, _): + if let column = column as? ArrowFixedSizeListArray { + try writeFieldNodes( + fields: [childField], + arrays: [column.values], + nodes: &nodes, + ) + } + default: + throw .init(.notImplemented("Unhandled field type: \(field.type)")) } - default: - break } } } @@ -368,42 +375,45 @@ public struct ArrowWriter { variadicBufferCounts.append(Int64(array.buffers.count - 2)) } // After writing this field's buffers, recurse into children - switch field.type { - case .strct(let fields): - guard let column = array as? ArrowStructArray else { - throw .init(.invalid("Expected ArrowStructArray for nested struct")) - } - try writeBufferInfo( - fields, - arrays: column.fields.map(\.array), - bufferOffset: &bufferOffset, - buffers: &buffers, - variadicBufferCounts: &variadicBufferCounts - ) - case .list(let childField), .map(let childField, _): - guard let column = array as? ArrowListArray else { - throw .init(.invalid("Expected ArrowListArray")) - } - try writeBufferInfo( - [childField], - arrays: [column.values], - bufferOffset: &bufferOffset, - buffers: &buffers, - variadicBufferCounts: &variadicBufferCounts - ) - case .fixedSizeList(let childField, _): - guard let column = array as? ArrowFixedSizeListArray else { - throw .init(.invalid("Expected ArrowFixedSizeListArray")) + if field.type.isNested { + switch field.type { + case .strct(let fields): + guard let column = array as? ArrowStructArray else { + throw .init(.invalid("Expected ArrowStructArray for nested struct")) + } + try writeBufferInfo( + fields, + arrays: column.fields.map(\.array), + bufferOffset: &bufferOffset, + buffers: &buffers, + variadicBufferCounts: &variadicBufferCounts + ) + case .list(let childField), .largeList(let childField), + .map(let childField, _): + guard let column = array as? ListArrayProtocol else { + throw .init(.invalid("Expected list array.")) + } + try writeBufferInfo( + [childField], + arrays: [column.values], + bufferOffset: &bufferOffset, + buffers: &buffers, + variadicBufferCounts: &variadicBufferCounts + ) + case .fixedSizeList(let childField, _): + guard let column = array as? ArrowFixedSizeListArray else { + throw .init(.invalid("Expected ArrowFixedSizeListArray")) + } + try writeBufferInfo( + [childField], + arrays: [column.values], + bufferOffset: &bufferOffset, + buffers: &buffers, + variadicBufferCounts: &variadicBufferCounts + ) + default: + throw .init(.notImplemented("Unsupported type: \(field.type)")) } - try writeBufferInfo( - [childField], - arrays: [column.values], - bufferOffset: &bufferOffset, - buffers: &buffers, - variadicBufferCounts: &variadicBufferCounts - ) - default: - break } } } diff --git a/Sources/ArrowIPC/Generated/FlatBuffersTypes.swift b/Sources/ArrowIPC/FlatBuffersTypeAliases.swift similarity index 88% rename from Sources/ArrowIPC/Generated/FlatBuffersTypes.swift rename to Sources/ArrowIPC/FlatBuffersTypeAliases.swift index 7656c23..9a450d4 100644 --- a/Sources/ArrowIPC/Generated/FlatBuffersTypes.swift +++ b/Sources/ArrowIPC/FlatBuffersTypeAliases.swift @@ -20,7 +20,13 @@ typealias FSchema = org_apache_arrow_flatbuf_Schema typealias FBuffer = org_apache_arrow_flatbuf_Buffer typealias FFieldNode = org_apache_arrow_flatbuf_FieldNode typealias FRecordBatch = org_apache_arrow_flatbuf_RecordBatch +typealias FMessageHeader = org_apache_arrow_flatbuf_MessageHeader +typealias FKeyValue = org_apache_arrow_flatbuf_KeyValue + +// MARK: Top level type. typealias FType = org_apache_arrow_flatbuf_Type_ + +// MARK: Primitive types. typealias FFloatingPoint = org_apache_arrow_flatbuf_FloatingPoint typealias FInt = org_apache_arrow_flatbuf_Int typealias FBool = org_apache_arrow_flatbuf_Bool @@ -29,16 +35,19 @@ typealias FTime = org_apache_arrow_flatbuf_Time typealias FDuration = org_apache_arrow_flatbuf_Duration typealias FTimestamp = org_apache_arrow_flatbuf_Timestamp typealias FTimeUnit = org_apache_arrow_flatbuf_TimeUnit + +// MARK: Nested types. typealias FStruct = org_apache_arrow_flatbuf_Struct_ typealias FList = org_apache_arrow_flatbuf_List +typealias FLargeList = org_apache_arrow_flatbuf_LargeList +typealias FMap = org_apache_arrow_flatbuf_Map +typealias FFixedSizeList = org_apache_arrow_flatbuf_FixedSizeList -typealias FUtf8 = org_apache_arrow_flatbuf_Utf8 -typealias FUtf8View = org_apache_arrow_flatbuf_Utf8View +// MARK: Binary types. typealias FBinary = org_apache_arrow_flatbuf_Binary +typealias FLargeBinary = org_apache_arrow_flatbuf_LargeBinary typealias FBinaryView = org_apache_arrow_flatbuf_BinaryView +typealias FUtf8 = org_apache_arrow_flatbuf_Utf8 +typealias FUtf8View = org_apache_arrow_flatbuf_Utf8View +typealias FLargeUtf8 = org_apache_arrow_flatbuf_LargeUtf8 typealias FFixedSizeBinary = org_apache_arrow_flatbuf_FixedSizeBinary -typealias FFixedSizeList = org_apache_arrow_flatbuf_FixedSizeList -typealias FMap = org_apache_arrow_flatbuf_Map - -typealias FMessageHeader = org_apache_arrow_flatbuf_MessageHeader -typealias FKeyValue = org_apache_arrow_flatbuf_KeyValue diff --git a/Sources/ArrowIPC/Generated/File_generated.swift b/Sources/ArrowIPC/Generated/File_generated.swift index 28994c6..92f7117 100644 --- a/Sources/ArrowIPC/Generated/File_generated.swift +++ b/Sources/ArrowIPC/Generated/File_generated.swift @@ -1,38 +1,29 @@ // swift-format-ignore-file -// Copyright 2025 The Apache Software Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// automatically generated by the FlatBuffers compiler, do not modify +// swiftlint:disable all +// swiftformat:disable all + +#if canImport(Common) +import Common +#endif import FlatBuffers -public struct org_apache_arrow_flatbuf_Block: NativeStruct, Verifiable, FlatbuffersInitializable { +public struct org_apache_arrow_flatbuf_Block: NativeStruct, FlatbuffersVectorInitializable, Verifiable, FlatbuffersInitializable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } /// Index to the start of the RecordBlock (note this is past the Message header) private var _offset: Int64 /// Length of the metadata private var _metaDataLength: Int32 - private let padding0__: UInt32 = 0 + private var padding0__: UInt32 = 0 /// Length of the data (this is aligned so there can be a gap between this and /// the metadata). private var _bodyLength: Int64 public init(_ bb: ByteBuffer, o: Int32) { - let _accessor = Struct(bb: bb, position: o) - _offset = _accessor.readBuffer(of: Int64.self, at: 0) - _metaDataLength = _accessor.readBuffer(of: Int32.self, at: 8) - _bodyLength = _accessor.readBuffer(of: Int64.self, at: 16) + self = bb.read(def: Self.self, position: Int(o)) } public init(offset: Int64, metaDataLength: Int32, bodyLength: Int64) { @@ -60,9 +51,9 @@ public struct org_apache_arrow_flatbuf_Block: NativeStruct, Verifiable, Flatbuff } } -public struct org_apache_arrow_flatbuf_Block_Mutable: FlatBufferObject { +public struct org_apache_arrow_flatbuf_Block_Mutable: FlatBufferStruct, FlatbuffersVectorInitializable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Struct @@ -76,9 +67,9 @@ public struct org_apache_arrow_flatbuf_Block_Mutable: FlatBufferObject { /// ---------------------------------------------------------------------- /// Arrow File metadata /// -public struct org_apache_arrow_flatbuf_Footer: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Footer: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -97,18 +88,14 @@ public struct org_apache_arrow_flatbuf_Footer: FlatBufferObject, Verifiable { public var version: org_apache_arrow_flatbuf_MetadataVersion { let o = _accessor.offset(VTOFFSET.version.v); return o == 0 ? .v1 : org_apache_arrow_flatbuf_MetadataVersion(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .v1 } public var schema: org_apache_arrow_flatbuf_Schema? { let o = _accessor.offset(VTOFFSET.schema.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Schema(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } - public var hasDictionaries: Bool { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? false : true } - public var dictionariesCount: Int32 { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func dictionaries(at index: Int32) -> org_apache_arrow_flatbuf_Block? { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Block.self, offset: _accessor.vector(at: o) + index * 24) } - public func mutableDictionaries(at index: Int32) -> org_apache_arrow_flatbuf_Block_Mutable? { let o = _accessor.offset(VTOFFSET.dictionaries.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Block_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 24) } - public var hasRecordBatches: Bool { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? false : true } - public var recordBatchesCount: Int32 { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func recordBatches(at index: Int32) -> org_apache_arrow_flatbuf_Block? { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Block.self, offset: _accessor.vector(at: o) + index * 24) } - public func mutableRecordBatches(at index: Int32) -> org_apache_arrow_flatbuf_Block_Mutable? { let o = _accessor.offset(VTOFFSET.recordBatches.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Block_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 24) } + public var dictionaries: FlatbufferVector { return _accessor.vector(at: VTOFFSET.dictionaries.v, byteSize: 24) } + public var mutableDictionaries: FlatbufferVector { return _accessor.vector(at: VTOFFSET.dictionaries.v, byteSize: 24) } + public func withUnsafePointerToDictionaries(_ body: (UnsafeRawBufferPointer, Int) throws -> T) rethrows -> T? { return try _accessor.withUnsafePointerToSlice(at: VTOFFSET.dictionaries.v, body: body) } + public var recordBatches: FlatbufferVector { return _accessor.vector(at: VTOFFSET.recordBatches.v, byteSize: 24) } + public var mutableRecordBatches: FlatbufferVector { return _accessor.vector(at: VTOFFSET.recordBatches.v, byteSize: 24) } + public func withUnsafePointerToRecordBatches(_ body: (UnsafeRawBufferPointer, Int) throws -> T) rethrows -> T? { return try _accessor.withUnsafePointerToSlice(at: VTOFFSET.recordBatches.v, body: body) } /// User-defined metadata - public var hasCustomMetadata: Bool { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? false : true } - public var customMetadataCount: Int32 { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func customMetadata(at index: Int32) -> org_apache_arrow_flatbuf_KeyValue? { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? nil : org_apache_arrow_flatbuf_KeyValue(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } + public var customMetadata: FlatbufferVector { return _accessor.vector(at: VTOFFSET.customMetadata.v, byteSize: 4) } public static func startFooter(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) } public static func add(version: org_apache_arrow_flatbuf_MetadataVersion, _ fbb: inout FlatBufferBuilder) { fbb.add(element: version.rawValue, def: 0, at: VTOFFSET.version.p) } public static func add(schema: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: schema, at: VTOFFSET.schema.p) } diff --git a/Sources/ArrowIPC/Generated/Message_generated.swift b/Sources/ArrowIPC/Generated/Message_generated.swift index f18f546..16080e1 100644 --- a/Sources/ArrowIPC/Generated/Message_generated.swift +++ b/Sources/ArrowIPC/Generated/Message_generated.swift @@ -1,40 +1,36 @@ // swift-format-ignore-file -// Copyright 2025 The Apache Software Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - +// automatically generated by the FlatBuffers compiler, do not modify +// swiftlint:disable all +// swiftformat:disable all import FlatBuffers -public enum org_apache_arrow_flatbuf_CompressionType: Int8, Enum, Verifiable { +#if canImport(Common) +import Common +#endif + +public enum org_apache_arrow_flatbuf_CompressionType: Int8, + FlatbuffersVectorInitializable, Enum, Verifiable +{ public typealias T = Int8 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int8 { return self.rawValue } + public static var byteSize: Int { MemoryLayout.size } + public var value: Int8 { self.rawValue } case lz4Frame = 0 case zstd = 1 - public static var max: org_apache_arrow_flatbuf_CompressionType { return .zstd } - public static var min: org_apache_arrow_flatbuf_CompressionType { return .lz4Frame } + public static var max: org_apache_arrow_flatbuf_CompressionType { .zstd } + public static var min: org_apache_arrow_flatbuf_CompressionType { .lz4Frame } } - /// Provided for forward compatibility in case we need to support different /// strategies for compressing the IPC message body (like whole-body /// compression rather than buffer-level) in the future -public enum org_apache_arrow_flatbuf_BodyCompressionMethod: Int8, Enum, Verifiable { +public enum org_apache_arrow_flatbuf_BodyCompressionMethod: Int8, + FlatbuffersVectorInitializable, Enum, Verifiable +{ public typealias T = Int8 - public static var byteSize: Int { return MemoryLayout.size } - public var value: Int8 { return self.rawValue } + public static var byteSize: Int { MemoryLayout.size } + public var value: Int8 { self.rawValue } /// Each constituent buffer is first compressed with the indicated /// compressor, and then written with the uncompressed length in the first 8 /// bytes as a 64-bit little-endian signed integer followed by the compressed @@ -44,28 +40,33 @@ public enum org_apache_arrow_flatbuf_BodyCompressionMethod: Int8, Enum, Verifiab /// compression does not yield appreciable savings. case buffer = 0 - public static var max: org_apache_arrow_flatbuf_BodyCompressionMethod { return .buffer } - public static var min: org_apache_arrow_flatbuf_BodyCompressionMethod { return .buffer } + public static var max: org_apache_arrow_flatbuf_BodyCompressionMethod { + .buffer + } + public static var min: org_apache_arrow_flatbuf_BodyCompressionMethod { + .buffer + } } - /// ---------------------------------------------------------------------- /// The root Message type /// This union enables us to easily send different message types without /// redundant storage, and in the future we can easily add new message types. -/// +/// /// Arrow implementations do not need to implement all of the message types, /// which may include experimental metadata types. For maximum compatibility, /// it is best to send data using RecordBatch -public enum org_apache_arrow_flatbuf_MessageHeader: UInt8, UnionEnum { +public enum org_apache_arrow_flatbuf_MessageHeader: UInt8, + FlatbuffersVectorInitializable, UnionEnum +{ public typealias T = UInt8 public init?(value: T) { self.init(rawValue: value) } - public static var byteSize: Int { return MemoryLayout.size } - public var value: UInt8 { return self.rawValue } + public static var byteSize: Int { MemoryLayout.size } + public var value: UInt8 { self.rawValue } case none_ = 0 case schema = 1 case dictionarybatch = 2 @@ -73,23 +74,26 @@ public enum org_apache_arrow_flatbuf_MessageHeader: UInt8, UnionEnum { case tensor = 4 case sparsetensor = 5 - public static var max: org_apache_arrow_flatbuf_MessageHeader { return .sparsetensor } - public static var min: org_apache_arrow_flatbuf_MessageHeader { return .none_ } + public static var max: org_apache_arrow_flatbuf_MessageHeader { + .sparsetensor + } + public static var min: org_apache_arrow_flatbuf_MessageHeader { .none_ } } - /// ---------------------------------------------------------------------- /// Data structures for describing a table row batch (a collection of /// equal-length Arrow arrays) /// Metadata about a field at some level of a nested type tree (but not /// its children). -/// +/// /// For example, a List with values `[[1, 2, 3], null, [4], [5, 6], null]` /// would have {length: 5, null_count: 2} for its List node, and {length: 6, /// null_count: 0} for its Int16 node, as separate FieldNode structs -public struct org_apache_arrow_flatbuf_FieldNode: NativeStruct, Verifiable, FlatbuffersInitializable { +public struct org_apache_arrow_flatbuf_FieldNode: NativeStruct, + FlatbuffersVectorInitializable, Verifiable, FlatbuffersInitializable +{ - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } /// The number of value slots in the Arrow array at this level of a nested /// tree @@ -100,9 +104,7 @@ public struct org_apache_arrow_flatbuf_FieldNode: NativeStruct, Verifiable, Flat private var _nullCount: Int64 public init(_ bb: ByteBuffer, o: Int32) { - let _accessor = Struct(bb: bb, position: o) - _length = _accessor.readBuffer(of: Int64.self, at: 0) - _nullCount = _accessor.readBuffer(of: Int64.self, at: 8) + self = bb.read(def: Self.self, position: Int(o)) } public init(length: Int64, nullCount: Int64) { @@ -123,8 +125,11 @@ public struct org_apache_arrow_flatbuf_FieldNode: NativeStruct, Verifiable, Flat /// instead setting the length of the bitmap buffer to 0. public var nullCount: Int64 { _nullCount } - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { - try verifier.inBuffer(position: position, of: org_apache_arrow_flatbuf_FieldNode.self) + public static func verify( + _ verifier: inout Verifier, at position: Int, of type: T.Type + ) throws where T: Verifiable { + try verifier.inBuffer( + position: position, of: org_apache_arrow_flatbuf_FieldNode.self) } } @@ -133,33 +138,41 @@ public struct org_apache_arrow_flatbuf_FieldNode: NativeStruct, Verifiable, Flat /// equal-length Arrow arrays) /// Metadata about a field at some level of a nested type tree (but not /// its children). -/// +/// /// For example, a List with values `[[1, 2, 3], null, [4], [5, 6], null]` /// would have {length: 5, null_count: 2} for its List node, and {length: 6, /// null_count: 0} for its Int16 node, as separate FieldNode structs -public struct org_apache_arrow_flatbuf_FieldNode_Mutable: FlatBufferObject { +public struct org_apache_arrow_flatbuf_FieldNode_Mutable: FlatBufferStruct, + FlatbuffersVectorInitializable +{ - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } + static func validateVersion() { FlatBuffersVersion_25_12_19() } + public var __buffer: ByteBuffer! { _accessor.bb } private var _accessor: Struct - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Struct(bb: bb, position: o) } + public init(_ bb: ByteBuffer, o: Int32) { + _accessor = Struct(bb: bb, position: o) + } - public var length: Int64 { return _accessor.readBuffer(of: Int64.self, at: 0) } - public var nullCount: Int64 { return _accessor.readBuffer(of: Int64.self, at: 8) } + public var length: Int64 { _accessor.readBuffer(of: Int64.self, at: 0) } + public var nullCount: Int64 { _accessor.readBuffer(of: Int64.self, at: 8) } } /// Optional compression for the memory buffers constituting IPC message /// bodies. Intended for use with RecordBatch but could be used for other /// message types -public struct org_apache_arrow_flatbuf_BodyCompression: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_BodyCompression: FlatBufferTable, + FlatbuffersVectorInitializable, Verifiable +{ - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } + static func validateVersion() { FlatBuffersVersion_25_12_19() } + public var __buffer: ByteBuffer! { _accessor.bb } private var _accessor: Table private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } + public init(_ bb: ByteBuffer, o: Int32) { + _accessor = Table(bb: bb, position: o) + } private enum VTOFFSET: VOffset { case codec = 4 @@ -170,28 +183,61 @@ public struct org_apache_arrow_flatbuf_BodyCompression: FlatBufferObject, Verifi /// Compressor library. /// For LZ4_FRAME, each compressed buffer must consist of a single frame. - public var codec: org_apache_arrow_flatbuf_CompressionType { let o = _accessor.offset(VTOFFSET.codec.v); return o == 0 ? .lz4Frame : org_apache_arrow_flatbuf_CompressionType(rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .lz4Frame } + public var codec: org_apache_arrow_flatbuf_CompressionType { + let o = _accessor.offset(VTOFFSET.codec.v) + return o == 0 + ? .lz4Frame + : org_apache_arrow_flatbuf_CompressionType( + rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .lz4Frame + } /// Indicates the way the record batch body was compressed - public var method: org_apache_arrow_flatbuf_BodyCompressionMethod { let o = _accessor.offset(VTOFFSET.method.v); return o == 0 ? .buffer : org_apache_arrow_flatbuf_BodyCompressionMethod(rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .buffer } - public static func startBodyCompression(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) } - public static func add(codec: org_apache_arrow_flatbuf_CompressionType, _ fbb: inout FlatBufferBuilder) { fbb.add(element: codec.rawValue, def: 0, at: VTOFFSET.codec.p) } - public static func add(method: org_apache_arrow_flatbuf_BodyCompressionMethod, _ fbb: inout FlatBufferBuilder) { fbb.add(element: method.rawValue, def: 0, at: VTOFFSET.method.p) } - public static func endBodyCompression(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } + public var method: org_apache_arrow_flatbuf_BodyCompressionMethod { + let o = _accessor.offset(VTOFFSET.method.v) + return o == 0 + ? .buffer + : org_apache_arrow_flatbuf_BodyCompressionMethod( + rawValue: _accessor.readBuffer(of: Int8.self, at: o)) ?? .buffer + } + public static func startBodyCompression(_ fbb: inout FlatBufferBuilder) + -> UOffset + { fbb.startTable(with: 2) } + public static func add( + codec: org_apache_arrow_flatbuf_CompressionType, + _ fbb: inout FlatBufferBuilder + ) { fbb.add(element: codec.rawValue, def: 0, at: VTOFFSET.codec.p) } + public static func add( + method: org_apache_arrow_flatbuf_BodyCompressionMethod, + _ fbb: inout FlatBufferBuilder + ) { fbb.add(element: method.rawValue, def: 0, at: VTOFFSET.method.p) } + public static func endBodyCompression( + _ fbb: inout FlatBufferBuilder, start: UOffset + ) -> Offset { + let end = Offset(offset: fbb.endTable(at: start)) + return end + } public static func createBodyCompression( _ fbb: inout FlatBufferBuilder, codec: org_apache_arrow_flatbuf_CompressionType = .lz4Frame, method: org_apache_arrow_flatbuf_BodyCompressionMethod = .buffer ) -> Offset { - let __start = org_apache_arrow_flatbuf_BodyCompression.startBodyCompression(&fbb) + let __start = org_apache_arrow_flatbuf_BodyCompression.startBodyCompression( + &fbb) org_apache_arrow_flatbuf_BodyCompression.add(codec: codec, &fbb) org_apache_arrow_flatbuf_BodyCompression.add(method: method, &fbb) - return org_apache_arrow_flatbuf_BodyCompression.endBodyCompression(&fbb, start: __start) + return org_apache_arrow_flatbuf_BodyCompression.endBodyCompression( + &fbb, start: __start) } - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { + public static func verify( + _ verifier: inout Verifier, at position: Int, of type: T.Type + ) throws where T: Verifiable { var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.codec.p, fieldName: "codec", required: false, type: org_apache_arrow_flatbuf_CompressionType.self) - try _v.visit(field: VTOFFSET.method.p, fieldName: "method", required: false, type: org_apache_arrow_flatbuf_BodyCompressionMethod.self) + try _v.visit( + field: VTOFFSET.codec.p, fieldName: "codec", required: false, + type: org_apache_arrow_flatbuf_CompressionType.self) + try _v.visit( + field: VTOFFSET.method.p, fieldName: "method", required: false, + type: org_apache_arrow_flatbuf_BodyCompressionMethod.self) _v.finish() } } @@ -199,14 +245,18 @@ public struct org_apache_arrow_flatbuf_BodyCompression: FlatBufferObject, Verifi /// A data header describing the shared memory layout of a "record" or "row" /// batch. Some systems call this a "row batch" internally and others a "record /// batch". -public struct org_apache_arrow_flatbuf_RecordBatch: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_RecordBatch: FlatBufferTable, + FlatbuffersVectorInitializable, Verifiable +{ - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } + static func validateVersion() { FlatBuffersVersion_25_12_19() } + public var __buffer: ByteBuffer! { _accessor.bb } private var _accessor: Table private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } + public init(_ bb: ByteBuffer, o: Int32) { + _accessor = Table(bb: bb, position: o) + } private enum VTOFFSET: VOffset { case length = 4 @@ -220,55 +270,108 @@ public struct org_apache_arrow_flatbuf_RecordBatch: FlatBufferObject, Verifiable /// number of records / rows. The arrays in the batch should all have this /// length - public var length: Int64 { let o = _accessor.offset(VTOFFSET.length.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) } + public var length: Int64 { + let o = _accessor.offset(VTOFFSET.length.v) + return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) + } /// Nodes correspond to the pre-ordered flattened logical schema - public var hasNodes: Bool { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? false : true } - public var nodesCount: Int32 { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func nodes(at index: Int32) -> org_apache_arrow_flatbuf_FieldNode? { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_FieldNode.self, offset: _accessor.vector(at: o) + index * 16) } - public func mutableNodes(at index: Int32) -> org_apache_arrow_flatbuf_FieldNode_Mutable? { let o = _accessor.offset(VTOFFSET.nodes.v); return o == 0 ? nil : org_apache_arrow_flatbuf_FieldNode_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) } + public var nodes: FlatbufferVector { + _accessor.vector(at: VTOFFSET.nodes.v, byteSize: 16) + } + public var mutableNodes: + FlatbufferVector + { _accessor.vector(at: VTOFFSET.nodes.v, byteSize: 16) } + public func withUnsafePointerToNodes( + _ body: (UnsafeRawBufferPointer, Int) throws -> T + ) rethrows -> T? { + try _accessor.withUnsafePointerToSlice(at: VTOFFSET.nodes.v, body: body) + } /// Buffers correspond to the pre-ordered flattened buffer tree - /// + /// /// The number of buffers appended to this list depends on the schema. For /// example, most primitive arrays will have 2 buffers, 1 for the validity /// bitmap and 1 for the values. For struct arrays, there will only be a /// single buffer for the validity (nulls) bitmap - public var hasBuffers: Bool { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? false : true } - public var buffersCount: Int32 { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func buffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer? { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Buffer.self, offset: _accessor.vector(at: o) + index * 16) } - public func mutableBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer_Mutable? { let o = _accessor.offset(VTOFFSET.buffers.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) } + public var buffers: FlatbufferVector { + _accessor.vector(at: VTOFFSET.buffers.v, byteSize: 16) + } + public var mutableBuffers: + FlatbufferVector + { _accessor.vector(at: VTOFFSET.buffers.v, byteSize: 16) } + public func withUnsafePointerToBuffers( + _ body: (UnsafeRawBufferPointer, Int) throws -> T + ) rethrows -> T? { + try _accessor.withUnsafePointerToSlice(at: VTOFFSET.buffers.v, body: body) + } /// Optional compression of the message body - public var compression: org_apache_arrow_flatbuf_BodyCompression? { let o = _accessor.offset(VTOFFSET.compression.v); return o == 0 ? nil : org_apache_arrow_flatbuf_BodyCompression(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } + public var compression: org_apache_arrow_flatbuf_BodyCompression? { + let o = _accessor.offset(VTOFFSET.compression.v) + return o == 0 + ? nil + : org_apache_arrow_flatbuf_BodyCompression( + _accessor.bb, o: _accessor.indirect(o + _accessor.position)) + } /// Some types such as Utf8View are represented using a variable number of buffers. /// For each such Field in the pre-ordered flattened logical schema, there will be /// an entry in variadicBufferCounts to indicate the number of number of variadic /// buffers which belong to that Field in the current RecordBatch. - /// + /// /// For example, the schema /// col1: Struct /// col2: Utf8View /// contains two Fields with variadic buffers so variadicBufferCounts will have /// two entries, the first counting the variadic buffers of `col1.beta` and the /// second counting `col2`'s. - /// + /// /// This field may be omitted if and only if the schema contains no Fields with /// a variable number of buffers, such as BinaryView and Utf8View. - public var hasVariadicBufferCounts: Bool { let o = _accessor.offset(VTOFFSET.variadicBufferCounts.v); return o == 0 ? false : true } - public var variadicBufferCountsCount: Int32 { let o = _accessor.offset(VTOFFSET.variadicBufferCounts.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func variadicBufferCounts(at index: Int32) -> Int64 { let o = _accessor.offset(VTOFFSET.variadicBufferCounts.v); return o == 0 ? 0 : _accessor.directRead(of: Int64.self, offset: _accessor.vector(at: o) + index * 8) } - public var variadicBufferCounts: [Int64] { return _accessor.getVector(at: VTOFFSET.variadicBufferCounts.v) ?? [] } - public static func startRecordBatch(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) } - public static func add(length: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: length, def: 0, at: VTOFFSET.length.p) } - public static func addVectorOf(nodes: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: nodes, at: VTOFFSET.nodes.p) } - public static func startVectorOfNodes(_ size: Int, in builder: inout FlatBufferBuilder) { - builder.startVector(size * MemoryLayout.size, elementSize: MemoryLayout.alignment) - } - public static func addVectorOf(buffers: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: buffers, at: VTOFFSET.buffers.p) } - public static func startVectorOfBuffers(_ size: Int, in builder: inout FlatBufferBuilder) { - builder.startVector(size * MemoryLayout.size, elementSize: MemoryLayout.alignment) - } - public static func add(compression: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: compression, at: VTOFFSET.compression.p) } - public static func addVectorOf(variadicBufferCounts: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: variadicBufferCounts, at: VTOFFSET.variadicBufferCounts.p) } - public static func endRecordBatch(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } + public var variadicBufferCounts: FlatbufferVector { + _accessor.vector(at: VTOFFSET.variadicBufferCounts.v, byteSize: 8) + } + public func withUnsafePointerToVariadicBufferCounts( + _ body: (UnsafeRawBufferPointer, Int) throws -> T + ) rethrows -> T? { + try _accessor.withUnsafePointerToSlice( + at: VTOFFSET.variadicBufferCounts.v, body: body) + } + public static func startRecordBatch(_ fbb: inout FlatBufferBuilder) -> UOffset + { fbb.startTable(with: 5) } + public static func add(length: Int64, _ fbb: inout FlatBufferBuilder) { + fbb.add(element: length, def: 0, at: VTOFFSET.length.p) + } + public static func addVectorOf(nodes: Offset, _ fbb: inout FlatBufferBuilder) + { fbb.add(offset: nodes, at: VTOFFSET.nodes.p) } + public static func startVectorOfNodes( + _ size: Int, in builder: inout FlatBufferBuilder + ) { + builder.startVector( + size * MemoryLayout.size, + elementSize: MemoryLayout.alignment) + } + public static func addVectorOf( + buffers: Offset, _ fbb: inout FlatBufferBuilder + ) { fbb.add(offset: buffers, at: VTOFFSET.buffers.p) } + public static func startVectorOfBuffers( + _ size: Int, in builder: inout FlatBufferBuilder + ) { + builder.startVector( + size * MemoryLayout.size, + elementSize: MemoryLayout.alignment) + } + public static func add(compression: Offset, _ fbb: inout FlatBufferBuilder) { + fbb.add(offset: compression, at: VTOFFSET.compression.p) + } + public static func addVectorOf( + variadicBufferCounts: Offset, _ fbb: inout FlatBufferBuilder + ) { + fbb.add(offset: variadicBufferCounts, at: VTOFFSET.variadicBufferCounts.p) + } + public static func endRecordBatch( + _ fbb: inout FlatBufferBuilder, start: UOffset + ) -> Offset { + let end = Offset(offset: fbb.endTable(at: start)) + return end + } public static func createRecordBatch( _ fbb: inout FlatBufferBuilder, length: Int64 = 0, @@ -282,17 +385,37 @@ public struct org_apache_arrow_flatbuf_RecordBatch: FlatBufferObject, Verifiable org_apache_arrow_flatbuf_RecordBatch.addVectorOf(nodes: nodes, &fbb) org_apache_arrow_flatbuf_RecordBatch.addVectorOf(buffers: buffers, &fbb) org_apache_arrow_flatbuf_RecordBatch.add(compression: compression, &fbb) - org_apache_arrow_flatbuf_RecordBatch.addVectorOf(variadicBufferCounts: variadicBufferCounts, &fbb) - return org_apache_arrow_flatbuf_RecordBatch.endRecordBatch(&fbb, start: __start) + org_apache_arrow_flatbuf_RecordBatch.addVectorOf( + variadicBufferCounts: variadicBufferCounts, &fbb) + return org_apache_arrow_flatbuf_RecordBatch.endRecordBatch( + &fbb, start: __start) } - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { + public static func verify( + _ verifier: inout Verifier, at position: Int, of type: T.Type + ) throws where T: Verifiable { var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.length.p, fieldName: "length", required: false, type: Int64.self) - try _v.visit(field: VTOFFSET.nodes.p, fieldName: "nodes", required: false, type: ForwardOffset>.self) - try _v.visit(field: VTOFFSET.buffers.p, fieldName: "buffers", required: false, type: ForwardOffset>.self) - try _v.visit(field: VTOFFSET.compression.p, fieldName: "compression", required: false, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.variadicBufferCounts.p, fieldName: "variadicBufferCounts", required: false, type: ForwardOffset>.self) + try _v.visit( + field: VTOFFSET.length.p, fieldName: "length", required: false, + type: Int64.self) + try _v.visit( + field: VTOFFSET.nodes.p, fieldName: "nodes", required: false, + type: ForwardOffset< + Vector< + org_apache_arrow_flatbuf_FieldNode, org_apache_arrow_flatbuf_FieldNode + > + >.self) + try _v.visit( + field: VTOFFSET.buffers.p, fieldName: "buffers", required: false, + type: ForwardOffset< + Vector + >.self) + try _v.visit( + field: VTOFFSET.compression.p, fieldName: "compression", required: false, + type: ForwardOffset.self) + try _v.visit( + field: VTOFFSET.variadicBufferCounts.p, fieldName: "variadicBufferCounts", + required: false, type: ForwardOffset>.self) _v.finish() } } @@ -303,14 +426,18 @@ public struct org_apache_arrow_flatbuf_RecordBatch: FlatBufferObject, Verifiable /// There is one vector / column per dictionary, but that vector / column /// may be spread across multiple dictionary batches by using the isDelta /// flag -public struct org_apache_arrow_flatbuf_DictionaryBatch: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_DictionaryBatch: FlatBufferTable, + FlatbuffersVectorInitializable, Verifiable +{ - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } + static func validateVersion() { FlatBuffersVersion_25_12_19() } + public var __buffer: ByteBuffer! { _accessor.bb } private var _accessor: Table private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } + public init(_ bb: ByteBuffer, o: Int32) { + _accessor = Table(bb: bb, position: o) + } private enum VTOFFSET: VOffset { case id = 4 @@ -320,48 +447,87 @@ public struct org_apache_arrow_flatbuf_DictionaryBatch: FlatBufferObject, Verifi var p: VOffset { self.rawValue } } - public var id: Int64 { let o = _accessor.offset(VTOFFSET.id.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) } - public var data: org_apache_arrow_flatbuf_RecordBatch? { let o = _accessor.offset(VTOFFSET.data.v); return o == 0 ? nil : org_apache_arrow_flatbuf_RecordBatch(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } + public var id: Int64 { + let o = _accessor.offset(VTOFFSET.id.v) + return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) + } + public var data: org_apache_arrow_flatbuf_RecordBatch? { + let o = _accessor.offset(VTOFFSET.data.v) + return o == 0 + ? nil + : org_apache_arrow_flatbuf_RecordBatch( + _accessor.bb, o: _accessor.indirect(o + _accessor.position)) + } /// If isDelta is true the values in the dictionary are to be appended to a /// dictionary with the indicated id. If isDelta is false this dictionary /// should replace the existing dictionary. - public var isDelta: Bool { let o = _accessor.offset(VTOFFSET.isDelta.v); return o == 0 ? false : _accessor.readBuffer(of: Bool.self, at: o) } - public static func startDictionaryBatch(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 3) } - public static func add(id: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: id, def: 0, at: VTOFFSET.id.p) } - public static func add(data: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: data, at: VTOFFSET.data.p) } - public static func add(isDelta: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: isDelta, def: false, - at: VTOFFSET.isDelta.p) } - public static func endDictionaryBatch(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } + public var isDelta: Bool { + let o = _accessor.offset(VTOFFSET.isDelta.v) + return o == 0 ? false : _accessor.readBuffer(of: Bool.self, at: o) + } + public static func startDictionaryBatch(_ fbb: inout FlatBufferBuilder) + -> UOffset + { fbb.startTable(with: 3) } + public static func add(id: Int64, _ fbb: inout FlatBufferBuilder) { + fbb.add(element: id, def: 0, at: VTOFFSET.id.p) + } + public static func add(data: Offset, _ fbb: inout FlatBufferBuilder) { + fbb.add(offset: data, at: VTOFFSET.data.p) + } + public static func add(isDelta: Bool, _ fbb: inout FlatBufferBuilder) { + fbb.add( + element: isDelta, def: false, + at: VTOFFSET.isDelta.p) + } + public static func endDictionaryBatch( + _ fbb: inout FlatBufferBuilder, start: UOffset + ) -> Offset { + let end = Offset(offset: fbb.endTable(at: start)) + return end + } public static func createDictionaryBatch( _ fbb: inout FlatBufferBuilder, id: Int64 = 0, dataOffset data: Offset = Offset(), isDelta: Bool = false ) -> Offset { - let __start = org_apache_arrow_flatbuf_DictionaryBatch.startDictionaryBatch(&fbb) + let __start = org_apache_arrow_flatbuf_DictionaryBatch.startDictionaryBatch( + &fbb) org_apache_arrow_flatbuf_DictionaryBatch.add(id: id, &fbb) org_apache_arrow_flatbuf_DictionaryBatch.add(data: data, &fbb) org_apache_arrow_flatbuf_DictionaryBatch.add(isDelta: isDelta, &fbb) - return org_apache_arrow_flatbuf_DictionaryBatch.endDictionaryBatch(&fbb, start: __start) + return org_apache_arrow_flatbuf_DictionaryBatch.endDictionaryBatch( + &fbb, start: __start) } - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { + public static func verify( + _ verifier: inout Verifier, at position: Int, of type: T.Type + ) throws where T: Verifiable { var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.id.p, fieldName: "id", required: false, type: Int64.self) - try _v.visit(field: VTOFFSET.data.p, fieldName: "data", required: false, type: ForwardOffset.self) - try _v.visit(field: VTOFFSET.isDelta.p, fieldName: "isDelta", required: false, type: Bool.self) + try _v.visit( + field: VTOFFSET.id.p, fieldName: "id", required: false, type: Int64.self) + try _v.visit( + field: VTOFFSET.data.p, fieldName: "data", required: false, + type: ForwardOffset.self) + try _v.visit( + field: VTOFFSET.isDelta.p, fieldName: "isDelta", required: false, + type: Bool.self) _v.finish() } } -public struct org_apache_arrow_flatbuf_Message: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Message: FlatBufferTable, + FlatbuffersVectorInitializable, Verifiable +{ - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } + static func validateVersion() { FlatBuffersVersion_25_12_19() } + public var __buffer: ByteBuffer! { _accessor.bb } private var _accessor: Table private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } + public init(_ bb: ByteBuffer, o: Int32) { + _accessor = Table(bb: bb, position: o) + } private enum VTOFFSET: VOffset { case version = 4 @@ -373,20 +539,56 @@ public struct org_apache_arrow_flatbuf_Message: FlatBufferObject, Verifiable { var p: VOffset { self.rawValue } } - public var version: org_apache_arrow_flatbuf_MetadataVersion { let o = _accessor.offset(VTOFFSET.version.v); return o == 0 ? .v1 : org_apache_arrow_flatbuf_MetadataVersion(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .v1 } - public var headerType: org_apache_arrow_flatbuf_MessageHeader { let o = _accessor.offset(VTOFFSET.headerType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_MessageHeader(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ } - public func header(type: T.Type) -> T? { let o = _accessor.offset(VTOFFSET.header.v); return o == 0 ? nil : _accessor.union(o) } - public var bodyLength: Int64 { let o = _accessor.offset(VTOFFSET.bodyLength.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) } - public var hasCustomMetadata: Bool { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? false : true } - public var customMetadataCount: Int32 { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func customMetadata(at index: Int32) -> org_apache_arrow_flatbuf_KeyValue? { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? nil : org_apache_arrow_flatbuf_KeyValue(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } - public static func startMessage(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) } - public static func add(version: org_apache_arrow_flatbuf_MetadataVersion, _ fbb: inout FlatBufferBuilder) { fbb.add(element: version.rawValue, def: 0, at: VTOFFSET.version.p) } - public static func add(headerType: org_apache_arrow_flatbuf_MessageHeader, _ fbb: inout FlatBufferBuilder) { fbb.add(element: headerType.rawValue, def: 0, at: VTOFFSET.headerType.p) } - public static func add(header: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: header, at: VTOFFSET.header.p) } - public static func add(bodyLength: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: bodyLength, def: 0, at: VTOFFSET.bodyLength.p) } - public static func addVectorOf(customMetadata: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: customMetadata, at: VTOFFSET.customMetadata.p) } - public static func endMessage(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } + public var version: org_apache_arrow_flatbuf_MetadataVersion { + let o = _accessor.offset(VTOFFSET.version.v) + return o == 0 + ? .v1 + : org_apache_arrow_flatbuf_MetadataVersion( + rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .v1 + } + public var headerType: org_apache_arrow_flatbuf_MessageHeader { + let o = _accessor.offset(VTOFFSET.headerType.v) + return o == 0 + ? .none_ + : org_apache_arrow_flatbuf_MessageHeader( + rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ + } + public func header(type: T.Type) -> T? { + let o = _accessor.offset(VTOFFSET.header.v) + return o == 0 ? nil : _accessor.union(o) + } + public var bodyLength: Int64 { + let o = _accessor.offset(VTOFFSET.bodyLength.v) + return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) + } + public var customMetadata: FlatbufferVector + { _accessor.vector(at: VTOFFSET.customMetadata.v, byteSize: 4) } + public static func startMessage(_ fbb: inout FlatBufferBuilder) -> UOffset { + fbb.startTable(with: 5) + } + public static func add( + version: org_apache_arrow_flatbuf_MetadataVersion, + _ fbb: inout FlatBufferBuilder + ) { fbb.add(element: version.rawValue, def: 0, at: VTOFFSET.version.p) } + public static func add( + headerType: org_apache_arrow_flatbuf_MessageHeader, + _ fbb: inout FlatBufferBuilder + ) { fbb.add(element: headerType.rawValue, def: 0, at: VTOFFSET.headerType.p) } + public static func add(header: Offset, _ fbb: inout FlatBufferBuilder) { + fbb.add(offset: header, at: VTOFFSET.header.p) + } + public static func add(bodyLength: Int64, _ fbb: inout FlatBufferBuilder) { + fbb.add(element: bodyLength, def: 0, at: VTOFFSET.bodyLength.p) + } + public static func addVectorOf( + customMetadata: Offset, _ fbb: inout FlatBufferBuilder + ) { fbb.add(offset: customMetadata, at: VTOFFSET.customMetadata.p) } + public static func endMessage(_ fbb: inout FlatBufferBuilder, start: UOffset) + -> Offset + { + let end = Offset(offset: fbb.endTable(at: start)) + return end + } public static func createMessage( _ fbb: inout FlatBufferBuilder, version: org_apache_arrow_flatbuf_MetadataVersion = .v1, @@ -400,32 +602,56 @@ public struct org_apache_arrow_flatbuf_Message: FlatBufferObject, Verifiable { org_apache_arrow_flatbuf_Message.add(headerType: headerType, &fbb) org_apache_arrow_flatbuf_Message.add(header: header, &fbb) org_apache_arrow_flatbuf_Message.add(bodyLength: bodyLength, &fbb) - org_apache_arrow_flatbuf_Message.addVectorOf(customMetadata: customMetadata, &fbb) + org_apache_arrow_flatbuf_Message.addVectorOf( + customMetadata: customMetadata, &fbb) return org_apache_arrow_flatbuf_Message.endMessage(&fbb, start: __start) } - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { + public static func verify( + _ verifier: inout Verifier, at position: Int, of type: T.Type + ) throws where T: Verifiable { var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.version.p, fieldName: "version", required: false, type: org_apache_arrow_flatbuf_MetadataVersion.self) - try _v.visit(unionKey: VTOFFSET.headerType.p, unionField: VTOFFSET.header.p, unionKeyName: "headerType", fieldName: "header", required: false, completion: { (verifier, key: org_apache_arrow_flatbuf_MessageHeader, pos) in - switch key { - case .none_: - break // NOTE - SWIFT doesnt support none - case .schema: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Schema.self) - case .dictionarybatch: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_DictionaryBatch.self) - case .recordbatch: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_RecordBatch.self) - case .tensor: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Tensor.self) - case .sparsetensor: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_SparseTensor.self) - } - }) - try _v.visit(field: VTOFFSET.bodyLength.p, fieldName: "bodyLength", required: false, type: Int64.self) - try _v.visit(field: VTOFFSET.customMetadata.p, fieldName: "customMetadata", required: false, type: ForwardOffset, org_apache_arrow_flatbuf_KeyValue>>.self) + try _v.visit( + field: VTOFFSET.version.p, fieldName: "version", required: false, + type: org_apache_arrow_flatbuf_MetadataVersion.self) + try _v.visit( + unionKey: VTOFFSET.headerType.p, unionField: VTOFFSET.header.p, + unionKeyName: "headerType", fieldName: "header", required: false, + completion: { + (verifier, key: org_apache_arrow_flatbuf_MessageHeader, pos) in + switch key { + case .none_: + break // NOTE - SWIFT doesnt support none + case .schema: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Schema.self) + case .dictionarybatch: + try ForwardOffset.verify( + &verifier, at: pos, + of: org_apache_arrow_flatbuf_DictionaryBatch.self) + case .recordbatch: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_RecordBatch.self) + case .tensor: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Tensor.self) + case .sparsetensor: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_SparseTensor.self) + } + }) + try _v.visit( + field: VTOFFSET.bodyLength.p, fieldName: "bodyLength", required: false, + type: Int64.self) + try _v.visit( + field: VTOFFSET.customMetadata.p, fieldName: "customMetadata", + required: false, + type: ForwardOffset< + Vector< + ForwardOffset, + org_apache_arrow_flatbuf_KeyValue + > + >.self) _v.finish() } } - diff --git a/Sources/ArrowIPC/Generated/Schema_generated.swift b/Sources/ArrowIPC/Generated/Schema_generated.swift index 31af7a2..b085074 100644 --- a/Sources/ArrowIPC/Generated/Schema_generated.swift +++ b/Sources/ArrowIPC/Generated/Schema_generated.swift @@ -1,22 +1,15 @@ // swift-format-ignore-file -// Copyright 2025 The Apache Software Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// automatically generated by the FlatBuffers compiler, do not modify +// swiftlint:disable all +// swiftformat:disable all +#if canImport(Common) +import Common +#endif import FlatBuffers -public enum org_apache_arrow_flatbuf_MetadataVersion: Int16, Enum, Verifiable { +public enum org_apache_arrow_flatbuf_MetadataVersion: Int16, FlatbuffersVectorInitializable, Enum, Verifiable { public typealias T = Int16 public static var byteSize: Int { return MemoryLayout.size } public var value: Int16 { return self.rawValue } @@ -59,7 +52,7 @@ public enum org_apache_arrow_flatbuf_MetadataVersion: Int16, Enum, Verifiable { /// Enums added to this list should be assigned power-of-two values /// to facilitate exchanging and comparing bitmaps for supported /// features. -public enum org_apache_arrow_flatbuf_Feature: Int64, Enum, Verifiable { +public enum org_apache_arrow_flatbuf_Feature: Int64, FlatbuffersVectorInitializable, Enum, Verifiable { public typealias T = Int64 public static var byteSize: Int { return MemoryLayout.size } public var value: Int64 { return self.rawValue } @@ -78,7 +71,7 @@ public enum org_apache_arrow_flatbuf_Feature: Int64, Enum, Verifiable { } -public enum org_apache_arrow_flatbuf_UnionMode: Int16, Enum, Verifiable { +public enum org_apache_arrow_flatbuf_UnionMode: Int16, FlatbuffersVectorInitializable, Enum, Verifiable { public typealias T = Int16 public static var byteSize: Int { return MemoryLayout.size } public var value: Int16 { return self.rawValue } @@ -90,7 +83,7 @@ public enum org_apache_arrow_flatbuf_UnionMode: Int16, Enum, Verifiable { } -public enum org_apache_arrow_flatbuf_Precision: Int16, Enum, Verifiable { +public enum org_apache_arrow_flatbuf_Precision: Int16, FlatbuffersVectorInitializable, Enum, Verifiable { public typealias T = Int16 public static var byteSize: Int { return MemoryLayout.size } public var value: Int16 { return self.rawValue } @@ -103,7 +96,7 @@ public enum org_apache_arrow_flatbuf_Precision: Int16, Enum, Verifiable { } -public enum org_apache_arrow_flatbuf_DateUnit: Int16, Enum, Verifiable { +public enum org_apache_arrow_flatbuf_DateUnit: Int16, FlatbuffersVectorInitializable, Enum, Verifiable { public typealias T = Int16 public static var byteSize: Int { return MemoryLayout.size } public var value: Int16 { return self.rawValue } @@ -115,7 +108,7 @@ public enum org_apache_arrow_flatbuf_DateUnit: Int16, Enum, Verifiable { } -public enum org_apache_arrow_flatbuf_TimeUnit: Int16, Enum, Verifiable { +public enum org_apache_arrow_flatbuf_TimeUnit: Int16, FlatbuffersVectorInitializable, Enum, Verifiable { public typealias T = Int16 public static var byteSize: Int { return MemoryLayout.size } public var value: Int16 { return self.rawValue } @@ -129,7 +122,7 @@ public enum org_apache_arrow_flatbuf_TimeUnit: Int16, Enum, Verifiable { } -public enum org_apache_arrow_flatbuf_IntervalUnit: Int16, Enum, Verifiable { +public enum org_apache_arrow_flatbuf_IntervalUnit: Int16, FlatbuffersVectorInitializable, Enum, Verifiable { public typealias T = Int16 public static var byteSize: Int { return MemoryLayout.size } public var value: Int16 { return self.rawValue } @@ -145,7 +138,7 @@ public enum org_apache_arrow_flatbuf_IntervalUnit: Int16, Enum, Verifiable { /// ---------------------------------------------------------------------- /// Top-level Type value, enabling extensible type-specific metadata. We can /// add new logical types to Type without breaking backwards compatibility -public enum org_apache_arrow_flatbuf_Type_: UInt8, UnionEnum { +public enum org_apache_arrow_flatbuf_Type_: UInt8, FlatbuffersVectorInitializable, UnionEnum { public typealias T = UInt8 public init?(value: T) { @@ -192,7 +185,7 @@ public enum org_apache_arrow_flatbuf_Type_: UInt8, UnionEnum { /// Maintained for forwards compatibility, in the future /// Dictionaries might be explicit maps between integers and values /// allowing for non-contiguous index values -public enum org_apache_arrow_flatbuf_DictionaryKind: Int16, Enum, Verifiable { +public enum org_apache_arrow_flatbuf_DictionaryKind: Int16, FlatbuffersVectorInitializable, Enum, Verifiable { public typealias T = Int16 public static var byteSize: Int { return MemoryLayout.size } public var value: Int16 { return self.rawValue } @@ -205,7 +198,7 @@ public enum org_apache_arrow_flatbuf_DictionaryKind: Int16, Enum, Verifiable { /// ---------------------------------------------------------------------- /// Endianness of the platform producing the data -public enum org_apache_arrow_flatbuf_Endianness: Int16, Enum, Verifiable { +public enum org_apache_arrow_flatbuf_Endianness: Int16, FlatbuffersVectorInitializable, Enum, Verifiable { public typealias T = Int16 public static var byteSize: Int { return MemoryLayout.size } public var value: Int16 { return self.rawValue } @@ -219,9 +212,9 @@ public enum org_apache_arrow_flatbuf_Endianness: Int16, Enum, Verifiable { /// ---------------------------------------------------------------------- /// A Buffer represents a single contiguous memory segment -public struct org_apache_arrow_flatbuf_Buffer: NativeStruct, Verifiable, FlatbuffersInitializable { +public struct org_apache_arrow_flatbuf_Buffer: NativeStruct, FlatbuffersVectorInitializable, Verifiable, FlatbuffersInitializable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } /// The relative offset into the shared memory page where the bytes for this /// buffer starts @@ -234,9 +227,7 @@ public struct org_apache_arrow_flatbuf_Buffer: NativeStruct, Verifiable, Flatbuf private var _length: Int64 public init(_ bb: ByteBuffer, o: Int32) { - let _accessor = Struct(bb: bb, position: o) - _offset = _accessor.readBuffer(of: Int64.self, at: 0) - _length = _accessor.readBuffer(of: Int64.self, at: 8) + self = bb.read(def: Self.self, position: Int(o)) } public init(offset: Int64, length: Int64) { @@ -266,9 +257,9 @@ public struct org_apache_arrow_flatbuf_Buffer: NativeStruct, Verifiable, Flatbuf /// ---------------------------------------------------------------------- /// A Buffer represents a single contiguous memory segment -public struct org_apache_arrow_flatbuf_Buffer_Mutable: FlatBufferObject { +public struct org_apache_arrow_flatbuf_Buffer_Mutable: FlatBufferStruct, FlatbuffersVectorInitializable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Struct @@ -279,9 +270,9 @@ public struct org_apache_arrow_flatbuf_Buffer_Mutable: FlatBufferObject { } /// These are stored in the flatbuffer in the Type union below -public struct org_apache_arrow_flatbuf_Null: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Null: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -300,9 +291,9 @@ public struct org_apache_arrow_flatbuf_Null: FlatBufferObject, Verifiable { /// A Struct_ in the flatbuffer metadata is the same as an Arrow Struct /// (according to the physical memory layout). We used Struct_ here as /// Struct is a reserved word in Flatbuffers -public struct org_apache_arrow_flatbuf_Struct_: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Struct_: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -318,9 +309,9 @@ public struct org_apache_arrow_flatbuf_Struct_: FlatBufferObject, Verifiable { } } -public struct org_apache_arrow_flatbuf_List: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_List: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -338,9 +329,9 @@ public struct org_apache_arrow_flatbuf_List: FlatBufferObject, Verifiable { /// Same as List, but with 64-bit offsets, allowing to represent /// extremely large data values. -public struct org_apache_arrow_flatbuf_LargeList: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_LargeList: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -359,9 +350,9 @@ public struct org_apache_arrow_flatbuf_LargeList: FlatBufferObject, Verifiable { /// Represents the same logical types that List can, but contains offsets and /// sizes allowing for writes in any order and sharing of child values among /// list values. -public struct org_apache_arrow_flatbuf_ListView: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_ListView: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -379,9 +370,9 @@ public struct org_apache_arrow_flatbuf_ListView: FlatBufferObject, Verifiable { /// Same as ListView, but with 64-bit offsets and sizes, allowing to represent /// extremely large data values. -public struct org_apache_arrow_flatbuf_LargeListView: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_LargeListView: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -397,9 +388,9 @@ public struct org_apache_arrow_flatbuf_LargeListView: FlatBufferObject, Verifiab } } -public struct org_apache_arrow_flatbuf_FixedSizeList: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_FixedSizeList: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -458,9 +449,9 @@ public struct org_apache_arrow_flatbuf_FixedSizeList: FlatBufferObject, Verifiab /// The metadata is structured so that Arrow systems without special handling /// for Map can make Map an alias for List. The "layout" attribute for the Map /// field must have the same contents as a List. -public struct org_apache_arrow_flatbuf_Map: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Map: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -499,9 +490,9 @@ public struct org_apache_arrow_flatbuf_Map: FlatBufferObject, Verifiable { /// By default ids in the type vector refer to the offsets in the children /// optionally typeIds provides an indirection between the child offset and the type id /// for each child `typeIds[offset]` is the id used in the type vector -public struct org_apache_arrow_flatbuf_Union: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Union: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -516,10 +507,8 @@ public struct org_apache_arrow_flatbuf_Union: FlatBufferObject, Verifiable { } public var mode: org_apache_arrow_flatbuf_UnionMode { let o = _accessor.offset(VTOFFSET.mode.v); return o == 0 ? .sparse : org_apache_arrow_flatbuf_UnionMode(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .sparse } - public var hasTypeIds: Bool { let o = _accessor.offset(VTOFFSET.typeIds.v); return o == 0 ? false : true } - public var typeIdsCount: Int32 { let o = _accessor.offset(VTOFFSET.typeIds.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func typeIds(at index: Int32) -> Int32 { let o = _accessor.offset(VTOFFSET.typeIds.v); return o == 0 ? 0 : _accessor.directRead(of: Int32.self, offset: _accessor.vector(at: o) + index * 4) } - public var typeIds: [Int32] { return _accessor.getVector(at: VTOFFSET.typeIds.v) ?? [] } + public var typeIds: FlatbufferVector { return _accessor.vector(at: VTOFFSET.typeIds.v, byteSize: 4) } + public func withUnsafePointerToTypeIds(_ body: (UnsafeRawBufferPointer, Int) throws -> T) rethrows -> T? { return try _accessor.withUnsafePointerToSlice(at: VTOFFSET.typeIds.v, body: body) } public static func startUnion(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) } public static func add(mode: org_apache_arrow_flatbuf_UnionMode, _ fbb: inout FlatBufferBuilder) { fbb.add(element: mode.rawValue, def: 0, at: VTOFFSET.mode.p) } public static func addVectorOf(typeIds: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: typeIds, at: VTOFFSET.typeIds.p) } @@ -543,9 +532,9 @@ public struct org_apache_arrow_flatbuf_Union: FlatBufferObject, Verifiable { } } -public struct org_apache_arrow_flatbuf_Int: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Int: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -585,9 +574,9 @@ public struct org_apache_arrow_flatbuf_Int: FlatBufferObject, Verifiable { } } -public struct org_apache_arrow_flatbuf_FloatingPoint: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_FloatingPoint: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -621,9 +610,9 @@ public struct org_apache_arrow_flatbuf_FloatingPoint: FlatBufferObject, Verifiab } /// Unicode with UTF-8 encoding -public struct org_apache_arrow_flatbuf_Utf8: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Utf8: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -640,9 +629,9 @@ public struct org_apache_arrow_flatbuf_Utf8: FlatBufferObject, Verifiable { } /// Opaque binary data -public struct org_apache_arrow_flatbuf_Binary: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Binary: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -660,9 +649,9 @@ public struct org_apache_arrow_flatbuf_Binary: FlatBufferObject, Verifiable { /// Same as Utf8, but with 64-bit offsets, allowing to represent /// extremely large data values. -public struct org_apache_arrow_flatbuf_LargeUtf8: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_LargeUtf8: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -680,9 +669,9 @@ public struct org_apache_arrow_flatbuf_LargeUtf8: FlatBufferObject, Verifiable { /// Same as Binary, but with 64-bit offsets, allowing to represent /// extremely large data values. -public struct org_apache_arrow_flatbuf_LargeBinary: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_LargeBinary: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -705,9 +694,9 @@ public struct org_apache_arrow_flatbuf_LargeBinary: FlatBufferObject, Verifiable /// /// Since it uses a variable number of data buffers, each Field with this type /// must have a corresponding entry in `variadicBufferCounts`. -public struct org_apache_arrow_flatbuf_Utf8View: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Utf8View: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -730,9 +719,9 @@ public struct org_apache_arrow_flatbuf_Utf8View: FlatBufferObject, Verifiable { /// /// Since it uses a variable number of data buffers, each Field with this type /// must have a corresponding entry in `variadicBufferCounts`. -public struct org_apache_arrow_flatbuf_BinaryView: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_BinaryView: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -748,9 +737,9 @@ public struct org_apache_arrow_flatbuf_BinaryView: FlatBufferObject, Verifiable } } -public struct org_apache_arrow_flatbuf_FixedSizeBinary: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_FixedSizeBinary: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -784,9 +773,9 @@ public struct org_apache_arrow_flatbuf_FixedSizeBinary: FlatBufferObject, Verifi } } -public struct org_apache_arrow_flatbuf_Bool: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Bool: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -807,9 +796,9 @@ public struct org_apache_arrow_flatbuf_Bool: FlatBufferObject, Verifiable { /// which encodes the indices at which the run with the value in /// each corresponding index in the values child array ends. /// Like list/struct types, the value array can be of any type. -public struct org_apache_arrow_flatbuf_RunEndEncoded: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_RunEndEncoded: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -829,9 +818,9 @@ public struct org_apache_arrow_flatbuf_RunEndEncoded: FlatBufferObject, Verifiab /// complement. Currently 32-bit (4-byte), 64-bit (8-byte), /// 128-bit (16-byte) and 256-bit (32-byte) integers are used. /// The representation uses the endianness indicated in the Schema. -public struct org_apache_arrow_flatbuf_Decimal: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Decimal: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -886,9 +875,9 @@ public struct org_apache_arrow_flatbuf_Decimal: FlatBufferObject, Verifiable { /// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no /// leap seconds), where the values are evenly divisible by 86400000 /// * Days (32 bits) since the UNIX epoch -public struct org_apache_arrow_flatbuf_Date: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Date: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -935,9 +924,9 @@ public struct org_apache_arrow_flatbuf_Date: FlatBufferObject, Verifiable { /// This definition doesn't allow for leap seconds. Time values from /// measurements with leap seconds will need to be corrected when ingesting /// into Arrow (for example by replacing the value 86400 with 86399). -public struct org_apache_arrow_flatbuf_Time: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Time: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -1081,9 +1070,9 @@ public struct org_apache_arrow_flatbuf_Time: FlatBufferObject, Verifiable { /// values should be computed "as if" the timezone of the date-time values /// was UTC; for example, the naive date-time "January 1st 1970, 00h00" would /// be encoded as timestamp value 0. -public struct org_apache_arrow_flatbuf_Timestamp: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Timestamp: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -1133,9 +1122,9 @@ public struct org_apache_arrow_flatbuf_Timestamp: FlatBufferObject, Verifiable { } } -public struct org_apache_arrow_flatbuf_Interval: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Interval: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -1168,9 +1157,9 @@ public struct org_apache_arrow_flatbuf_Interval: FlatBufferObject, Verifiable { } } -public struct org_apache_arrow_flatbuf_Duration: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Duration: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -1206,9 +1195,9 @@ public struct org_apache_arrow_flatbuf_Duration: FlatBufferObject, Verifiable { /// ---------------------------------------------------------------------- /// user defined key value pairs to add custom metadata to arrow /// key namespacing is the responsibility of the user -public struct org_apache_arrow_flatbuf_KeyValue: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_KeyValue: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -1249,9 +1238,9 @@ public struct org_apache_arrow_flatbuf_KeyValue: FlatBufferObject, Verifiable { } } -public struct org_apache_arrow_flatbuf_DictionaryEncoding: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_DictionaryEncoding: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -1318,9 +1307,9 @@ public struct org_apache_arrow_flatbuf_DictionaryEncoding: FlatBufferObject, Ver /// ---------------------------------------------------------------------- /// A field represents a named column in a record / row batch or child of a /// nested type. -public struct org_apache_arrow_flatbuf_Field: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Field: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -1351,13 +1340,9 @@ public struct org_apache_arrow_flatbuf_Field: FlatBufferObject, Verifiable { public var dictionary: org_apache_arrow_flatbuf_DictionaryEncoding? { let o = _accessor.offset(VTOFFSET.dictionary.v); return o == 0 ? nil : org_apache_arrow_flatbuf_DictionaryEncoding(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } /// children apply only to nested data types like Struct, List and Union. For /// primitive types children will have length 0. - public var hasChildren: Bool { let o = _accessor.offset(VTOFFSET.children.v); return o == 0 ? false : true } - public var childrenCount: Int32 { let o = _accessor.offset(VTOFFSET.children.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func children(at index: Int32) -> org_apache_arrow_flatbuf_Field? { let o = _accessor.offset(VTOFFSET.children.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Field(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } + public var children: FlatbufferVector { return _accessor.vector(at: VTOFFSET.children.v, byteSize: 4) } /// User-defined metadata - public var hasCustomMetadata: Bool { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? false : true } - public var customMetadataCount: Int32 { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func customMetadata(at index: Int32) -> org_apache_arrow_flatbuf_KeyValue? { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? nil : org_apache_arrow_flatbuf_KeyValue(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } + public var customMetadata: FlatbufferVector { return _accessor.vector(at: VTOFFSET.customMetadata.v, byteSize: 4) } public static func startField(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 7) } public static func add(name: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: name, at: VTOFFSET.name.p) } public static func add(nullable: Bool, _ fbb: inout FlatBufferBuilder) { fbb.add(element: nullable, def: false, @@ -1460,9 +1445,9 @@ public struct org_apache_arrow_flatbuf_Field: FlatBufferObject, Verifiable { /// ---------------------------------------------------------------------- /// A Schema describes the columns in a row batch -public struct org_apache_arrow_flatbuf_Schema: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Schema: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -1482,16 +1467,10 @@ public struct org_apache_arrow_flatbuf_Schema: FlatBufferObject, Verifiable { /// it is Little Endian by default /// if endianness doesn't match the underlying system then the vectors need to be converted public var endianness: org_apache_arrow_flatbuf_Endianness { let o = _accessor.offset(VTOFFSET.endianness.v); return o == 0 ? .little : org_apache_arrow_flatbuf_Endianness(rawValue: _accessor.readBuffer(of: Int16.self, at: o)) ?? .little } - public var hasFields: Bool { let o = _accessor.offset(VTOFFSET.fields.v); return o == 0 ? false : true } - public var fieldsCount: Int32 { let o = _accessor.offset(VTOFFSET.fields.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func fields(at index: Int32) -> org_apache_arrow_flatbuf_Field? { let o = _accessor.offset(VTOFFSET.fields.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Field(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } - public var hasCustomMetadata: Bool { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? false : true } - public var customMetadataCount: Int32 { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func customMetadata(at index: Int32) -> org_apache_arrow_flatbuf_KeyValue? { let o = _accessor.offset(VTOFFSET.customMetadata.v); return o == 0 ? nil : org_apache_arrow_flatbuf_KeyValue(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } + public var fields: FlatbufferVector { return _accessor.vector(at: VTOFFSET.fields.v, byteSize: 4) } + public var customMetadata: FlatbufferVector { return _accessor.vector(at: VTOFFSET.customMetadata.v, byteSize: 4) } /// Features used in the stream/file. - public var hasFeatures: Bool { let o = _accessor.offset(VTOFFSET.features.v); return o == 0 ? false : true } - public var featuresCount: Int32 { let o = _accessor.offset(VTOFFSET.features.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func features(at index: Int32) -> org_apache_arrow_flatbuf_Feature? { let o = _accessor.offset(VTOFFSET.features.v); return o == 0 ? org_apache_arrow_flatbuf_Feature.unused : org_apache_arrow_flatbuf_Feature(rawValue: _accessor.directRead(of: Int64.self, offset: _accessor.vector(at: o) + index * 8)) } + public var features: FlatbufferVector { return _accessor.vector(at: VTOFFSET.features.v, byteSize: 8) } public static func startSchema(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 4) } public static func add(endianness: org_apache_arrow_flatbuf_Endianness, _ fbb: inout FlatBufferBuilder) { fbb.add(element: endianness.rawValue, def: 0, at: VTOFFSET.endianness.p) } public static func addVectorOf(fields: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: fields, at: VTOFFSET.fields.p) } diff --git a/Sources/ArrowIPC/Generated/SparseTensor_generated.swift b/Sources/ArrowIPC/Generated/SparseTensor_generated.swift index 0a9f544..86dd542 100644 --- a/Sources/ArrowIPC/Generated/SparseTensor_generated.swift +++ b/Sources/ArrowIPC/Generated/SparseTensor_generated.swift @@ -1,22 +1,16 @@ // swift-format-ignore-file -// Copyright 2025 The Apache Software Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// automatically generated by the FlatBuffers compiler, do not modify +// swiftlint:disable all +// swiftformat:disable all +#if canImport(Common) +import Common +#endif + import FlatBuffers -public enum org_apache_arrow_flatbuf_SparseMatrixCompressedAxis: Int16, Enum, Verifiable { +public enum org_apache_arrow_flatbuf_SparseMatrixCompressedAxis: Int16, FlatbuffersVectorInitializable, Enum, Verifiable { public typealias T = Int16 public static var byteSize: Int { return MemoryLayout.size } public var value: Int16 { return self.rawValue } @@ -28,7 +22,7 @@ public enum org_apache_arrow_flatbuf_SparseMatrixCompressedAxis: Int16, Enum, Ve } -public enum org_apache_arrow_flatbuf_SparseTensorIndex: UInt8, UnionEnum { +public enum org_apache_arrow_flatbuf_SparseTensorIndex: UInt8, FlatbuffersVectorInitializable, UnionEnum { public typealias T = UInt8 public init?(value: T) { @@ -79,9 +73,9 @@ public enum org_apache_arrow_flatbuf_SparseTensorIndex: UInt8, UnionEnum { /// When isCanonical is true, the indices is sorted in lexicographical order /// (row-major order), and it does not have duplicated entries. Otherwise, /// the indices may not be sorted, or may have duplicated entries. -public struct org_apache_arrow_flatbuf_SparseTensorIndexCOO: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_SparseTensorIndexCOO: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -101,10 +95,8 @@ public struct org_apache_arrow_flatbuf_SparseTensorIndexCOO: FlatBufferObject, V public var indicesType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indicesType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } /// Non-negative byte offsets to advance one value cell along each dimension /// If omitted, default to row-major order (C-like). - public var hasIndicesStrides: Bool { let o = _accessor.offset(VTOFFSET.indicesStrides.v); return o == 0 ? false : true } - public var indicesStridesCount: Int32 { let o = _accessor.offset(VTOFFSET.indicesStrides.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func indicesStrides(at index: Int32) -> Int64 { let o = _accessor.offset(VTOFFSET.indicesStrides.v); return o == 0 ? 0 : _accessor.directRead(of: Int64.self, offset: _accessor.vector(at: o) + index * 8) } - public var indicesStrides: [Int64] { return _accessor.getVector(at: VTOFFSET.indicesStrides.v) ?? [] } + public var indicesStrides: FlatbufferVector { return _accessor.vector(at: VTOFFSET.indicesStrides.v, byteSize: 8) } + public func withUnsafePointerToIndicesStrides(_ body: (UnsafeRawBufferPointer, Int) throws -> T) rethrows -> T? { return try _accessor.withUnsafePointerToSlice(at: VTOFFSET.indicesStrides.v, body: body) } /// The location and size of the indices matrix's data public var indicesBuffer: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.indicesBuffer.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) } public var mutableIndicesBuffer: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.indicesBuffer.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) } @@ -147,9 +139,9 @@ public struct org_apache_arrow_flatbuf_SparseTensorIndexCOO: FlatBufferObject, V } /// Compressed Sparse format, that is matrix-specific. -public struct org_apache_arrow_flatbuf_SparseMatrixIndexCSX: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_SparseMatrixIndexCSX: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -244,9 +236,9 @@ public struct org_apache_arrow_flatbuf_SparseMatrixIndexCSX: FlatBufferObject, V } /// Compressed Sparse Fiber (CSF) sparse tensor index. -public struct org_apache_arrow_flatbuf_SparseTensorIndexCSF: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_SparseTensorIndexCSF: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -309,10 +301,9 @@ public struct org_apache_arrow_flatbuf_SparseTensorIndexCSF: FlatBufferObject, V /// [0, 2, 4, 5, 8] /// ]. /// ``` - public var hasIndptrBuffers: Bool { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? false : true } - public var indptrBuffersCount: Int32 { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func indptrBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer? { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Buffer.self, offset: _accessor.vector(at: o) + index * 16) } - public func mutableIndptrBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer_Mutable? { let o = _accessor.offset(VTOFFSET.indptrBuffers.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) } + public var indptrBuffers: FlatbufferVector { return _accessor.vector(at: VTOFFSET.indptrBuffers.v, byteSize: 16) } + public var mutableIndptrBuffers: FlatbufferVector { return _accessor.vector(at: VTOFFSET.indptrBuffers.v, byteSize: 16) } + public func withUnsafePointerToIndptrBuffers(_ body: (UnsafeRawBufferPointer, Int) throws -> T) rethrows -> T? { return try _accessor.withUnsafePointerToSlice(at: VTOFFSET.indptrBuffers.v, body: body) } /// The type of values in indicesBuffers public var indicesType: org_apache_arrow_flatbuf_Int! { let o = _accessor.offset(VTOFFSET.indicesType.v); return org_apache_arrow_flatbuf_Int(_accessor.bb, o: _accessor.indirect(o + _accessor.position)) } /// indicesBuffers stores values of nodes. @@ -326,20 +317,17 @@ public struct org_apache_arrow_flatbuf_SparseTensorIndexCSF: FlatBufferObject, V /// [1, 2, 0, 2, 0, 0, 1, 2] /// ]. /// ``` - public var hasIndicesBuffers: Bool { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? false : true } - public var indicesBuffersCount: Int32 { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func indicesBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer? { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? nil : _accessor.directRead(of: org_apache_arrow_flatbuf_Buffer.self, offset: _accessor.vector(at: o) + index * 16) } - public func mutableIndicesBuffers(at index: Int32) -> org_apache_arrow_flatbuf_Buffer_Mutable? { let o = _accessor.offset(VTOFFSET.indicesBuffers.v); return o == 0 ? nil : org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: _accessor.vector(at: o) + index * 16) } + public var indicesBuffers: FlatbufferVector { return _accessor.vector(at: VTOFFSET.indicesBuffers.v, byteSize: 16) } + public var mutableIndicesBuffers: FlatbufferVector { return _accessor.vector(at: VTOFFSET.indicesBuffers.v, byteSize: 16) } + public func withUnsafePointerToIndicesBuffers(_ body: (UnsafeRawBufferPointer, Int) throws -> T) rethrows -> T? { return try _accessor.withUnsafePointerToSlice(at: VTOFFSET.indicesBuffers.v, body: body) } /// axisOrder stores the sequence in which dimensions were traversed to /// produce the prefix tree. /// For example, the axisOrder for the above X is: /// ```text /// axisOrder(X) = [0, 1, 2, 3]. /// ``` - public var hasAxisOrder: Bool { let o = _accessor.offset(VTOFFSET.axisOrder.v); return o == 0 ? false : true } - public var axisOrderCount: Int32 { let o = _accessor.offset(VTOFFSET.axisOrder.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func axisOrder(at index: Int32) -> Int32 { let o = _accessor.offset(VTOFFSET.axisOrder.v); return o == 0 ? 0 : _accessor.directRead(of: Int32.self, offset: _accessor.vector(at: o) + index * 4) } - public var axisOrder: [Int32] { return _accessor.getVector(at: VTOFFSET.axisOrder.v) ?? [] } + public var axisOrder: FlatbufferVector { return _accessor.vector(at: VTOFFSET.axisOrder.v, byteSize: 4) } + public func withUnsafePointerToAxisOrder(_ body: (UnsafeRawBufferPointer, Int) throws -> T) rethrows -> T? { return try _accessor.withUnsafePointerToSlice(at: VTOFFSET.axisOrder.v, body: body) } public static func startSparseTensorIndexCSF(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) } public static func add(indptrType: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indptrType, at: VTOFFSET.indptrType.p) } public static func addVectorOf(indptrBuffers: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: indptrBuffers, at: VTOFFSET.indptrBuffers.p) } @@ -381,9 +369,9 @@ public struct org_apache_arrow_flatbuf_SparseTensorIndexCSF: FlatBufferObject, V } } -public struct org_apache_arrow_flatbuf_SparseTensor: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_SparseTensor: FlatBufferTable, FlatbuffersVectorInitializable, Verifiable { - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table @@ -408,9 +396,7 @@ public struct org_apache_arrow_flatbuf_SparseTensor: FlatBufferObject, Verifiabl /// no strings or nested types. public func type(type: T.Type) -> T! { let o = _accessor.offset(VTOFFSET.type.v); return _accessor.union(o) } /// The dimensions of the tensor, optionally named. - public var hasShape: Bool { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? false : true } - public var shapeCount: Int32 { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func shape(at index: Int32) -> org_apache_arrow_flatbuf_TensorDim? { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? nil : org_apache_arrow_flatbuf_TensorDim(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } + public var shape: FlatbufferVector { return _accessor.vector(at: VTOFFSET.shape.v, byteSize: 4) } /// The number of non-zero values in a sparse tensor. public var nonZeroLength: Int64 { let o = _accessor.offset(VTOFFSET.nonZeroLength.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) } public var sparseIndexType: org_apache_arrow_flatbuf_SparseTensorIndex { let o = _accessor.offset(VTOFFSET.sparseIndexType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_SparseTensorIndex(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ } diff --git a/Sources/ArrowIPC/Generated/Tensor_generated.swift b/Sources/ArrowIPC/Generated/Tensor_generated.swift index 4e6d61f..219edc1 100644 --- a/Sources/ArrowIPC/Generated/Tensor_generated.swift +++ b/Sources/ArrowIPC/Generated/Tensor_generated.swift @@ -1,32 +1,29 @@ // swift-format-ignore-file -// Copyright 2025 The Apache Software Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - +// automatically generated by the FlatBuffers compiler, do not modify +// swiftlint:disable all +// swiftformat:disable all import FlatBuffers +#if canImport(Common) +import Common +#endif + /// ---------------------------------------------------------------------- /// Data structures for dense tensors /// Shape data for a single axis in a tensor -public struct org_apache_arrow_flatbuf_TensorDim: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_TensorDim: FlatBufferTable, + FlatbuffersVectorInitializable, Verifiable +{ - static func validateVersion() { FlatBuffersVersion_25_2_10() } + static func validateVersion() { FlatBuffersVersion_25_12_19() } public var __buffer: ByteBuffer! { return _accessor.bb } private var _accessor: Table private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } + public init(_ bb: ByteBuffer, o: Int32) { + _accessor = Table(bb: bb, position: o) + } private enum VTOFFSET: VOffset { case size = 4 @@ -36,14 +33,33 @@ public struct org_apache_arrow_flatbuf_TensorDim: FlatBufferObject, Verifiable { } /// Length of dimension - public var size: Int64 { let o = _accessor.offset(VTOFFSET.size.v); return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) } + public var size: Int64 { + let o = _accessor.offset(VTOFFSET.size.v) + return o == 0 ? 0 : _accessor.readBuffer(of: Int64.self, at: o) + } /// Name of the dimension, optional - public var name: String? { let o = _accessor.offset(VTOFFSET.name.v); return o == 0 ? nil : _accessor.string(at: o) } - public var nameSegmentArray: [UInt8]? { return _accessor.getVector(at: VTOFFSET.name.v) } - public static func startTensorDim(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 2) } - public static func add(size: Int64, _ fbb: inout FlatBufferBuilder) { fbb.add(element: size, def: 0, at: VTOFFSET.size.p) } - public static func add(name: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: name, at: VTOFFSET.name.p) } - public static func endTensorDim(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); return end } + public var name: String? { + let o = _accessor.offset(VTOFFSET.name.v) + return o == 0 ? nil : _accessor.string(at: o) + } + public var nameSegmentArray: [UInt8]? { + _accessor.getVector(at: VTOFFSET.name.v) + } + public static func startTensorDim(_ fbb: inout FlatBufferBuilder) -> UOffset { + fbb.startTable(with: 2) + } + public static func add(size: Int64, _ fbb: inout FlatBufferBuilder) { + fbb.add(element: size, def: 0, at: VTOFFSET.size.p) + } + public static func add(name: Offset, _ fbb: inout FlatBufferBuilder) { + fbb.add(offset: name, at: VTOFFSET.name.p) + } + public static func endTensorDim( + _ fbb: inout FlatBufferBuilder, start: UOffset + ) -> Offset { + let end = Offset(offset: fbb.endTable(at: start)) + return end + } public static func createTensorDim( _ fbb: inout FlatBufferBuilder, size: Int64 = 0, @@ -55,22 +71,32 @@ public struct org_apache_arrow_flatbuf_TensorDim: FlatBufferObject, Verifiable { return org_apache_arrow_flatbuf_TensorDim.endTensorDim(&fbb, start: __start) } - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { + public static func verify( + _ verifier: inout Verifier, at position: Int, of type: T.Type + ) throws where T: Verifiable { var _v = try verifier.visitTable(at: position) - try _v.visit(field: VTOFFSET.size.p, fieldName: "size", required: false, type: Int64.self) - try _v.visit(field: VTOFFSET.name.p, fieldName: "name", required: false, type: ForwardOffset.self) + try _v.visit( + field: VTOFFSET.size.p, fieldName: "size", required: false, + type: Int64.self) + try _v.visit( + field: VTOFFSET.name.p, fieldName: "name", required: false, + type: ForwardOffset.self) _v.finish() } } -public struct org_apache_arrow_flatbuf_Tensor: FlatBufferObject, Verifiable { +public struct org_apache_arrow_flatbuf_Tensor: FlatBufferTable, + FlatbuffersVectorInitializable, Verifiable +{ - static func validateVersion() { FlatBuffersVersion_25_2_10() } - public var __buffer: ByteBuffer! { return _accessor.bb } + static func validateVersion() { FlatBuffersVersion_25_12_19() } + public var __buffer: ByteBuffer! { _accessor.bb } private var _accessor: Table private init(_ t: Table) { _accessor = t } - public init(_ bb: ByteBuffer, o: Int32) { _accessor = Table(bb: bb, position: o) } + public init(_ bb: ByteBuffer, o: Int32) { + _accessor = Table(bb: bb, position: o) + } private enum VTOFFSET: VOffset { case typeType = 4 @@ -82,30 +108,70 @@ public struct org_apache_arrow_flatbuf_Tensor: FlatBufferObject, Verifiable { var p: VOffset { self.rawValue } } - public var typeType: org_apache_arrow_flatbuf_Type_ { let o = _accessor.offset(VTOFFSET.typeType.v); return o == 0 ? .none_ : org_apache_arrow_flatbuf_Type_(rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ } + public var typeType: org_apache_arrow_flatbuf_Type_ { + let o = _accessor.offset(VTOFFSET.typeType.v) + return o == 0 + ? .none_ + : org_apache_arrow_flatbuf_Type_( + rawValue: _accessor.readBuffer(of: UInt8.self, at: o)) ?? .none_ + } /// The type of data contained in a value cell. Currently only fixed-width /// value types are supported, no strings or nested types - public func type(type: T.Type) -> T! { let o = _accessor.offset(VTOFFSET.type.v); return _accessor.union(o) } + public func type(type: T.Type) -> T! { + let o = _accessor.offset(VTOFFSET.type.v) + return _accessor.union(o) + } /// The dimensions of the tensor, optionally named - public var hasShape: Bool { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? false : true } - public var shapeCount: Int32 { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func shape(at index: Int32) -> org_apache_arrow_flatbuf_TensorDim? { let o = _accessor.offset(VTOFFSET.shape.v); return o == 0 ? nil : org_apache_arrow_flatbuf_TensorDim(_accessor.bb, o: _accessor.indirect(_accessor.vector(at: o) + index * 4)) } + public var shape: FlatbufferVector { + _accessor.vector(at: VTOFFSET.shape.v, byteSize: 4) + } /// Non-negative byte offsets to advance one value cell along each dimension /// If omitted, default to row-major order (C-like). - public var hasStrides: Bool { let o = _accessor.offset(VTOFFSET.strides.v); return o == 0 ? false : true } - public var stridesCount: Int32 { let o = _accessor.offset(VTOFFSET.strides.v); return o == 0 ? 0 : _accessor.vector(count: o) } - public func strides(at index: Int32) -> Int64 { let o = _accessor.offset(VTOFFSET.strides.v); return o == 0 ? 0 : _accessor.directRead(of: Int64.self, offset: _accessor.vector(at: o) + index * 8) } - public var strides: [Int64] { return _accessor.getVector(at: VTOFFSET.strides.v) ?? [] } + public var strides: FlatbufferVector { + _accessor.vector(at: VTOFFSET.strides.v, byteSize: 8) + } + public func withUnsafePointerToStrides( + _ body: (UnsafeRawBufferPointer, Int) throws -> T + ) rethrows -> T? { + try _accessor.withUnsafePointerToSlice(at: VTOFFSET.strides.v, body: body) + } /// The location and size of the tensor's data - public var data: org_apache_arrow_flatbuf_Buffer! { let o = _accessor.offset(VTOFFSET.data.v); return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) } - public var mutableData: org_apache_arrow_flatbuf_Buffer_Mutable! { let o = _accessor.offset(VTOFFSET.data.v); return org_apache_arrow_flatbuf_Buffer_Mutable(_accessor.bb, o: o + _accessor.position) } - public static func startTensor(_ fbb: inout FlatBufferBuilder) -> UOffset { fbb.startTable(with: 5) } - public static func add(typeType: org_apache_arrow_flatbuf_Type_, _ fbb: inout FlatBufferBuilder) { fbb.add(element: typeType.rawValue, def: 0, at: VTOFFSET.typeType.p) } - public static func add(type: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: type, at: VTOFFSET.type.p) } - public static func addVectorOf(shape: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: shape, at: VTOFFSET.shape.p) } - public static func addVectorOf(strides: Offset, _ fbb: inout FlatBufferBuilder) { fbb.add(offset: strides, at: VTOFFSET.strides.p) } - public static func add(data: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder) { guard let data = data else { return }; fbb.create(struct: data, position: VTOFFSET.data.p) } - public static func endTensor(_ fbb: inout FlatBufferBuilder, start: UOffset) -> Offset { let end = Offset(offset: fbb.endTable(at: start)); fbb.require(table: end, fields: [6, 8, 12]); return end } + public var data: org_apache_arrow_flatbuf_Buffer! { + let o = _accessor.offset(VTOFFSET.data.v) + return _accessor.readBuffer(of: org_apache_arrow_flatbuf_Buffer.self, at: o) + } + public var mutableData: org_apache_arrow_flatbuf_Buffer_Mutable! { + let o = _accessor.offset(VTOFFSET.data.v) + return org_apache_arrow_flatbuf_Buffer_Mutable( + _accessor.bb, o: o + _accessor.position) + } + public static func startTensor(_ fbb: inout FlatBufferBuilder) -> UOffset { + fbb.startTable(with: 5) + } + public static func add( + typeType: org_apache_arrow_flatbuf_Type_, _ fbb: inout FlatBufferBuilder + ) { fbb.add(element: typeType.rawValue, def: 0, at: VTOFFSET.typeType.p) } + public static func add(type: Offset, _ fbb: inout FlatBufferBuilder) { + fbb.add(offset: type, at: VTOFFSET.type.p) + } + public static func addVectorOf(shape: Offset, _ fbb: inout FlatBufferBuilder) + { fbb.add(offset: shape, at: VTOFFSET.shape.p) } + public static func addVectorOf( + strides: Offset, _ fbb: inout FlatBufferBuilder + ) { fbb.add(offset: strides, at: VTOFFSET.strides.p) } + public static func add( + data: org_apache_arrow_flatbuf_Buffer?, _ fbb: inout FlatBufferBuilder + ) { + guard let data = data else { return } + fbb.create(struct: data, position: VTOFFSET.data.p) + } + public static func endTensor(_ fbb: inout FlatBufferBuilder, start: UOffset) + -> Offset + { + let end = Offset(offset: fbb.endTable(at: start)) + fbb.require(table: end, fields: [6, 8, 12]) + return end + } public static func createTensor( _ fbb: inout FlatBufferBuilder, typeType: org_apache_arrow_flatbuf_Type_ = .none_, @@ -123,70 +189,112 @@ public struct org_apache_arrow_flatbuf_Tensor: FlatBufferObject, Verifiable { return org_apache_arrow_flatbuf_Tensor.endTensor(&fbb, start: __start) } - public static func verify(_ verifier: inout Verifier, at position: Int, of type: T.Type) throws where T: Verifiable { + public static func verify( + _ verifier: inout Verifier, at position: Int, of type: T.Type + ) throws where T: Verifiable { var _v = try verifier.visitTable(at: position) - try _v.visit(unionKey: VTOFFSET.typeType.p, unionField: VTOFFSET.type.p, unionKeyName: "typeType", fieldName: "type", required: true, completion: { (verifier, key: org_apache_arrow_flatbuf_Type_, pos) in - switch key { - case .none_: - break // NOTE - SWIFT doesnt support none - case .null: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Null.self) - case .int: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Int.self) - case .floatingpoint: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FloatingPoint.self) - case .binary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Binary.self) - case .utf8: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8.self) - case .bool: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Bool.self) - case .decimal: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Decimal.self) - case .date: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Date.self) - case .time: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Time.self) - case .timestamp: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Timestamp.self) - case .interval: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Interval.self) - case .list: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_List.self) - case .struct_: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Struct_.self) - case .union: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Union.self) - case .fixedsizebinary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeBinary.self) - case .fixedsizelist: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeList.self) - case .map: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Map.self) - case .duration: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Duration.self) - case .largebinary: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeBinary.self) - case .largeutf8: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeUtf8.self) - case .largelist: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeList.self) - case .runendencoded: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_RunEndEncoded.self) - case .binaryview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_BinaryView.self) - case .utf8view: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8View.self) - case .listview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_ListView.self) - case .largelistview: - try ForwardOffset.verify(&verifier, at: pos, of: org_apache_arrow_flatbuf_LargeListView.self) - } - }) - try _v.visit(field: VTOFFSET.shape.p, fieldName: "shape", required: true, type: ForwardOffset, org_apache_arrow_flatbuf_TensorDim>>.self) - try _v.visit(field: VTOFFSET.strides.p, fieldName: "strides", required: false, type: ForwardOffset>.self) - try _v.visit(field: VTOFFSET.data.p, fieldName: "data", required: true, type: org_apache_arrow_flatbuf_Buffer.self) + try _v.visit( + unionKey: VTOFFSET.typeType.p, unionField: VTOFFSET.type.p, + unionKeyName: "typeType", fieldName: "type", required: true, + completion: { (verifier, key: org_apache_arrow_flatbuf_Type_, pos) in + switch key { + case .none_: + break // NOTE - SWIFT doesnt support none + case .null: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Null.self) + case .int: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Int.self) + case .floatingpoint: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_FloatingPoint.self) + case .binary: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Binary.self) + case .utf8: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8.self) + case .bool: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Bool.self) + case .decimal: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Decimal.self) + case .date: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Date.self) + case .time: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Time.self) + case .timestamp: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Timestamp.self) + case .interval: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Interval.self) + case .list: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_List.self) + case .struct_: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Struct_.self) + case .union: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Union.self) + case .fixedsizebinary: + try ForwardOffset.verify( + &verifier, at: pos, + of: org_apache_arrow_flatbuf_FixedSizeBinary.self) + case .fixedsizelist: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_FixedSizeList.self) + case .map: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Map.self) + case .duration: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Duration.self) + case .largebinary: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_LargeBinary.self) + case .largeutf8: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_LargeUtf8.self) + case .largelist: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_LargeList.self) + case .runendencoded: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_RunEndEncoded.self) + case .binaryview: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_BinaryView.self) + case .utf8view: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_Utf8View.self) + case .listview: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_ListView.self) + case .largelistview: + try ForwardOffset.verify( + &verifier, at: pos, of: org_apache_arrow_flatbuf_LargeListView.self) + } + }) + try _v.visit( + field: VTOFFSET.shape.p, fieldName: "shape", required: true, + type: ForwardOffset< + Vector< + ForwardOffset, + org_apache_arrow_flatbuf_TensorDim + > + >.self) + try _v.visit( + field: VTOFFSET.strides.p, fieldName: "strides", required: false, + type: ForwardOffset>.self) + try _v.visit( + field: VTOFFSET.data.p, fieldName: "data", required: true, + type: org_apache_arrow_flatbuf_Buffer.self) _v.finish() } } - diff --git a/Sources/ArrowIPC/MessageHeader.swift b/Sources/ArrowIPC/MessageHeader.swift new file mode 100644 index 0000000..dcda2af --- /dev/null +++ b/Sources/ArrowIPC/MessageHeader.swift @@ -0,0 +1,30 @@ +// Copyright 2025 The Apache Software Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Arrow +import BinaryParsing + +struct MessageHeader { + let metadataLength: UInt32 + var isEndOfStream: Bool { metadataLength == 0 } + + @inlinable + init(parsing input: inout ParserSpan) throws { + let continuation = try UInt32(parsingLittleEndian: &input) + guard continuation == continuationMarker else { + throw ArrowError(.invalid("Missing continuation marker.")) + } + self.metadataLength = try UInt32(parsingLittleEndian: &input) + } +} diff --git a/Sources/Arrow/ProtoUtil.swift b/Sources/ArrowIPC/ProtoUtil.swift similarity index 89% rename from Sources/Arrow/ProtoUtil.swift rename to Sources/ArrowIPC/ProtoUtil.swift index 8674c00..1cd78d9 100644 --- a/Sources/Arrow/ProtoUtil.swift +++ b/Sources/ArrowIPC/ProtoUtil.swift @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import Arrow import Foundation func fromProto( @@ -92,20 +93,15 @@ func fromProto( arrowType = .timestamp(arrowUnit, timestampType.timezone) case .struct_: var children: [ArrowField] = [] - for index in 0.. 0 { - switch field.type { - case .list(let listField), .map(let listField, _): - guard let listArray = array as? ListArrayProtocol else { - throw .init(.invalid("Expected list array.")) - } - let childColumn = try encodeColumn( - array: listArray.values, field: listField) - children = [childColumn] - // List arrays point to child arrays therefore have nil data buffers. - data = nil - case .fixedSizeList(let listField, _): - guard let listArray = array as? ListArrayProtocol else { - throw .init(.invalid("Expected fixed-size list array.")) - } + switch field.type { + case .list(let listField), + .largeList(let listField), + .map(let listField, _), + .fixedSizeList(let listField, _): + guard let listArray = array as? ListArrayProtocol else { + throw .init(.invalid("Expected list array.")) + } + let childColumn = try encodeColumn( + array: listArray.values, + field: listField + ) + children = [childColumn] + // List arrays point to child arrays therefore have nil data buffers. + data = nil + case .strct(let arrowFields): + guard let structArray = array as? ArrowStructArray else { + throw .init(.invalid("Expected struct array.")) + } + children = [] + for (arrowField, (_, array)) in zip(arrowFields, structArray.fields) { let childColumn = try encodeColumn( - array: listArray.values, field: listField) - children = [childColumn] - data = nil - case .strct(let arrowFields): - guard let structArray = array as? ArrowStructArray else { - throw .init(.invalid("Expected struct array.")) - } - children = [] - for (arrowField, (_, array)) in zip(arrowFields, structArray.fields) { - let childColumn = try encodeColumn( - array: array, field: arrowField) - children?.append(childColumn) - data = nil - } - children = try arrowFields.enumerated().map { - index, arrowField throws(ArrowError) in - try encodeColumn(array: structArray.fields[index].1, field: arrowField) - } - data = nil - case .boolean: - data = try extractBoolData(from: array) - case .int8: - data = try extractIntData(from: array, expectedType: Int8.self) - case .int16: - data = try extractIntData(from: array, expectedType: Int16.self) - case .int32: - data = try extractIntData(from: array, expectedType: Int32.self) - case .int64: - data = try extractIntData(from: array, expectedType: Int64.self) - case .uint8: - data = try extractIntData(from: array, expectedType: UInt8.self) - case .uint16: - data = try extractIntData(from: array, expectedType: UInt16.self) - case .uint32: - data = try extractIntData(from: array, expectedType: UInt32.self) - case .uint64: - data = try extractIntData(from: array, expectedType: UInt64.self) - case .float16: - data = try extractFloatData(from: array, expectedType: Float16.self) - case .float32: - data = try extractFloatData(from: array, expectedType: Float32.self) - case .float64: - data = try extractFloatData(from: array, expectedType: Float64.self) - case .time32(_): - data = try extractIntData(from: array, expectedType: UInt32.self) - case .date32: - data = try extractIntData(from: array, expectedType: Int32.self) - case .date64: - data = try extractIntData(from: array, expectedType: Int64.self) - case .time64(_): - data = try extractIntData(from: array, expectedType: UInt64.self) - case .timestamp(_, _): - data = try extractIntData(from: array, expectedType: Int64.self) - case .duration(_): - data = try extractIntData(from: array, expectedType: Int64.self) - case .interval(.yearMonth): - data = try extractIntData(from: array, expectedType: Int32.self) - case .interval(.dayTime): - data = try extractIntData(from: array, expectedType: Int64.self) - case .interval(.monthDayNano): - // This is tricky - 128 bits (4 + 4 + 8 bytes) - // Might need special handling or extract as raw bytes - data = try extractIntData(from: array, expectedType: Int64.self) - case .binary: - try extractBinaryData(from: array, into: &data) - case .fixedSizeBinary(_): - try extractBinaryData(from: array, into: &data) - case .utf8: - try extractUtf8Data(from: array, into: &data) - case .binaryView, .utf8View: - try extractBinaryViewData( - from: array, - into: &views, - variadicDataBuffers: &variadicDataBuffers + array: array, + field: arrowField ) - default: - throw .init( - .invalid("Encoder did not handle a field type: \(field.type)")) + children?.append(childColumn) + data = nil } + children = try arrowFields.enumerated().map { + index, arrowField throws(ArrowError) in + try encodeColumn(array: structArray.fields[index].1, field: arrowField) + } + data = nil + case .boolean: + data = try extractBoolData(from: array) + case .int8: + data = try extractIntData(from: array, expectedType: Int8.self) + case .int16: + data = try extractIntData(from: array, expectedType: Int16.self) + case .int32: + data = try extractIntData(from: array, expectedType: Int32.self) + case .int64: + data = try extractIntData(from: array, expectedType: Int64.self) + case .uint8: + data = try extractIntData(from: array, expectedType: UInt8.self) + case .uint16: + data = try extractIntData(from: array, expectedType: UInt16.self) + case .uint32: + data = try extractIntData(from: array, expectedType: UInt32.self) + case .uint64: + data = try extractIntData(from: array, expectedType: UInt64.self) + case .float16: + data = try extractFloatData(from: array, expectedType: Float16.self) + case .float32: + data = try extractFloatData(from: array, expectedType: Float32.self) + case .float64: + data = try extractFloatData(from: array, expectedType: Float64.self) + case .time32(_): + data = try extractIntData(from: array, expectedType: UInt32.self) + case .date32: + data = try extractIntData(from: array, expectedType: Int32.self) + case .date64: + data = try extractIntData(from: array, expectedType: Int64.self) + case .time64(_): + data = try extractIntData(from: array, expectedType: UInt64.self) + case .timestamp(_, _): + data = try extractIntData(from: array, expectedType: Int64.self) + case .duration(_): + data = try extractIntData(from: array, expectedType: Int64.self) + case .interval(.yearMonth): + data = try extractIntData(from: array, expectedType: Int32.self) + case .interval(.dayTime): + data = try extractIntData(from: array, expectedType: Int64.self) + case .interval(.monthDayNano): + // This is tricky - 128 bits (4 + 4 + 8 bytes) + // Might need special handling or extract as raw bytes + data = try extractIntData(from: array, expectedType: Int64.self) + case .binary: + try extractBinaryData(from: array, into: &data) + case .largeBinary: + try extractBinaryData(from: array, into: &data) + case .fixedSizeBinary(_): + try extractBinaryData(from: array, into: &data) + case .utf8: + try extractUtf8Data(from: array, into: &data) + case .largeUtf8: + try extractUtf8Data(from: array, into: &data) + case .binaryView, .utf8View: + try extractBinaryViewData( + from: array, + into: &views, + variadicDataBuffers: &variadicDataBuffers + ) + default: + throw .init( + .invalid("Encoder did not handle a field type: \(field.type)")) } return .init( name: field.name, diff --git a/Tests/ArrowIPCTests/ArrowTestingGold.swift b/Tests/ArrowIPCTests/ArrowTestingGold.swift index 495f6b4..108f6d4 100644 --- a/Tests/ArrowIPCTests/ArrowTestingGold.swift +++ b/Tests/ArrowIPCTests/ArrowTestingGold.swift @@ -30,14 +30,17 @@ struct ArrowTestingGold { static let testCases: [String] = [ "generated_binary", - "generated_binary_view", "generated_binary_no_batches", + "generated_binary_view", "generated_binary_zerolength", "generated_custom_metadata", + "generated_datetime", + "generated_duplicate_fieldnames", "generated_nested", + "generated_nested_large_offsets", "generated_recursive_nested", + "generated_large_binary", "generated_map", - "generated_datetime", "generated_duration", "generated_primitive", "generated_primitive_no_batches", @@ -79,23 +82,50 @@ struct ArrowTestingGold { "generated_union", ] - @Test(arguments: testCases) - func read(name: String) throws { - let resourceURL = try loadTestResource( - name: name, - withExtension: "json.lz4", - subdirectory: "integration/cpp-21.0.0" + @Test + func readFile() throws { + + let name = "generated_nested_large_offsets" + let (testFile, testCase) = try loadTestCase( + name: name, fileExtension: "arrow_file") + let arrowReader = try ArrowReader(url: testFile) + let (arrowSchema, recordBatches) = try arrowReader.read() + #expect(testCase.batches.count == recordBatches.count) + try validateReadResults( + testCase: testCase, recordBatches: recordBatches, arrowSchema: arrowSchema ) - let lz4Data = try Data(contentsOf: resourceURL) - let lz4 = try LZ4(parsing: lz4Data) - let testCase = try JSONDecoder().decode(ArrowGold.self, from: lz4.data) - let testFile = try loadTestResource( - name: name, - withExtension: "arrow_file", - subdirectory: "integration/cpp-21.0.0" + } + + @Test(arguments: testCases) + func readStream(name: String) throws { + let (testFile, testCase) = try loadTestCase( + name: name, fileExtension: "stream") + let arrowReader = ArrowStreamReader() + let data = try Data(contentsOf: testFile) + let recordBatches = try arrowReader.read(data: data) + let arrowSchema = arrowReader.arrowSchema! + try validateReadResults( + testCase: testCase, recordBatches: recordBatches, arrowSchema: arrowSchema ) + } + + @Test(arguments: testCases) + func read(name: String) throws { + let (testFile, testCase) = try loadTestCase( + name: name, fileExtension: "arrow_file") let arrowReader = try ArrowReader(url: testFile) let (arrowSchema, recordBatches) = try arrowReader.read() + #expect(testCase.batches.count == recordBatches.count) + try validateReadResults( + testCase: testCase, recordBatches: recordBatches, arrowSchema: arrowSchema + ) + } + + private func validateReadResults( + testCase: ArrowGold, + recordBatches: [RecordBatch], + arrowSchema: ArrowSchema + ) throws { #expect(testCase.batches.count == recordBatches.count) // Strip placeholder values. let expectedBatches = testCase.batches.map { batch in diff --git a/Tests/ArrowIPCTests/ArrowType+validation.swift b/Tests/ArrowIPCTests/ArrowType+validation.swift index a89e45c..617b822 100644 --- a/Tests/ArrowIPCTests/ArrowType+validation.swift +++ b/Tests/ArrowIPCTests/ArrowType+validation.swift @@ -227,8 +227,12 @@ extension ArrowType { name = "bool" case .binary: name = "binary" + case .largeBinary: + name = "largebinary" case .utf8: name = "utf8" + case .largeUtf8: + name = "largeutf8" case .binaryView: name = "binaryview" case .utf8View: diff --git a/Tests/ArrowIPCTests/Subprocess.swift b/Tests/ArrowIPCTests/Subprocess.swift index 2df011d..6fef205 100644 --- a/Tests/ArrowIPCTests/Subprocess.swift +++ b/Tests/ArrowIPCTests/Subprocess.swift @@ -28,15 +28,14 @@ func pyArrowRoundTrip(ipcData: Data) async throws -> Data { arguments: [ "-c", """ - import pyarrow as pa import sys from io import BytesIO + import pyarrow as pa + input_data = sys.stdin.buffer.read() buffer = BytesIO(input_data) - reader = pa.ipc.open_file(buffer) - writer = pa.ipc.new_file(sys.stdout.buffer, reader.schema) for i in range(reader.num_record_batches): batch = reader.get_batch(i) diff --git a/Tests/ArrowIPCTests/TestSupport.swift b/Tests/ArrowIPCTests/TestSupport.swift index 5bb0c9f..1d66fc5 100644 --- a/Tests/ArrowIPCTests/TestSupport.swift +++ b/Tests/ArrowIPCTests/TestSupport.swift @@ -43,6 +43,27 @@ func loadTestResource( } } +func loadTestCase( + name: String, + fileExtension: String +) throws -> (URL, ArrowGold) { + + let resourceURL = try loadTestResource( + name: name, + withExtension: "json.lz4", + subdirectory: "integration/cpp-21.0.0" + ) + let lz4Data = try Data(contentsOf: resourceURL) + let lz4 = try LZ4(parsing: lz4Data) + let testCase = try JSONDecoder().decode(ArrowGold.self, from: lz4.data) + let testFile = try loadTestResource( + name: name, + withExtension: fileExtension, + subdirectory: "integration/cpp-21.0.0" + ) + return (testFile, testCase) +} + func checkBoolRecordBatch(recordBatch: RecordBatch) { #expect(recordBatch.length == 5)