diff --git a/.github/workflows/prune-stale-images.yml b/.github/workflows/prune-stale-images.yml new file mode 100644 index 00000000..e72bc663 --- /dev/null +++ b/.github/workflows/prune-stale-images.yml @@ -0,0 +1,25 @@ +name: prune-stale-images + +on: + schedule: + - cron: '37 4 * * *' + workflow_dispatch: + +jobs: + prune: + runs-on: ubuntu-latest + permissions: + packages: write + + steps: + - name: Prune old temporary FIT performer images + uses: vlaurin/action-ghcr-prune@v0.6.0 + with: + token: ${{ github.token }} + organization: ${{ github.repository_owner }} + container: ruby-fit-performer + keep-younger-than: 7 # days + prune-untagged: true + prune-tags-regexes: | + ^refs- + ^sha- diff --git a/.github/workflows/publish-fit-performer.yml b/.github/workflows/publish-fit-performer.yml new file mode 100644 index 00000000..bb83f8f1 --- /dev/null +++ b/.github/workflows/publish-fit-performer.yml @@ -0,0 +1,47 @@ +name: publish-fit-performer +permissions: {} + +run-name: ${{ inputs.ref }} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.ref }} + cancel-in-progress: true + +on: + push: + branches: + - main + - release* + + tags: + - '*' + + paths-ignore: + - '.gitignore' + - '**/*.md' + - '**/.rubocop.yml' + - '**/.rubocop_todo.yml' + - 'examples/**' + - 'LICENSE.txt' + + workflow_dispatch: + inputs: + ref: + description: "Performer version. Specify a branch, tag, full commit hash, GitHub PR like 'refs/pull/37/merge', or Gerrit change like 'refs/changes/58/240858/6'" + required: true + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - uses: couchbaselabs/sdk-docker-build-action@v1 + with: + sdk: ruby + ref: ${{ inputs.ref }} + dockerfile: ./fit-performer/Dockerfile + env: + DOCKER_BUILD_SUMMARY: false diff --git a/.github/workflows/validate-fit-performer.yml b/.github/workflows/validate-fit-performer.yml new file mode 100644 index 00000000..1d951966 --- /dev/null +++ b/.github/workflows/validate-fit-performer.yml @@ -0,0 +1,27 @@ +name: validate-fit-performer +permissions: {} + +on: + pull_request: + branches: + - main + - release* + + paths: + - 'fit-performer/**' + - '.github/workflows/validate-fit-performer.yml' + +jobs: + build: + runs-on: ubuntu-latest + if: github.event_name != 'push' + + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + fetch-depth: 0 + fetch-tags: true + + - run: | + docker build -f fit-performer/Dockerfile . diff --git a/.gitignore b/.gitignore index 5347f1af..ef0383a1 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,7 @@ Gemfile.lock Makefile mkmf.log + +# Generated FIT performer protobuf code +fit-performer/lib/fit/protocol/**/* /release-* diff --git a/.rubocop.yml b/.rubocop.yml index fcda3591..d6b4fc0f 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -14,6 +14,7 @@ AllCops: - 'vendor/**/*' - 'ext/**/*' - 'lib/couchbase/protostellar/generated/**/*' + - 'fit-performer/lib/fit/protocol/**/*' # Copied from Rails v6.1.0-38-gbf165e3e19 - 'test/active_support/behaviors/*.rb' - 'test/active_support/behaviors.rb' diff --git a/Gemfile b/Gemfile index 0516a110..0f672b49 100644 --- a/Gemfile +++ b/Gemfile @@ -22,6 +22,14 @@ gemspec path: "couchbase-opentelemetry" gem "rake" +group :fit_performer, :development do + gem "grpc-tools", "~> 1.59" + gem 'opentelemetry-exporter-otlp', '~> 0.31.1' + gem "opentelemetry-exporter-otlp-metrics", '~> 0.6.1' + gem "opentelemetry-metrics-sdk", "~> 0.11.2" + gem "opentelemetry-sdk", "~> 1.10" +end + group :development do gem "activesupport", "~> 7.0.3" gem "drb" @@ -29,7 +37,6 @@ group :development do gem "flay" gem "flog" gem "gem-compiler" - gem "grpc-tools", "~> 1.59" gem "heckle" gem "irb" # TODO(SA): review https://minite.st/docs/History_rdoc.html#label-6.0.0+-2F+2025-12-17 @@ -37,8 +44,6 @@ group :development do gem "minitest-mock" gem "minitest-reporters" gem "mutex_m" - gem "opentelemetry-metrics-sdk", "~> 0.11.2" - gem "opentelemetry-sdk", "~> 1.10" gem "rack" gem "reek" gem "rubocop", require: false diff --git a/bin/update-fit-protocol b/bin/update-fit-protocol new file mode 100755 index 00000000..34459f18 --- /dev/null +++ b/bin/update-fit-protocol @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +rm -rf fit-performer/proto +mkdir -p fit-performer/proto +curl --location --fail https://github.com/couchbaselabs/fit-protocol/archive/refs/heads/main.tar.gz \ + | tar -xz --strip-components=2 -C fit-performer/proto fit-protocol-main/operational diff --git a/fit-performer/.rubocop.yml b/fit-performer/.rubocop.yml new file mode 100644 index 00000000..fc2019d4 --- /dev/null +++ b/fit-performer/.rubocop.yml @@ -0,0 +1 @@ +inherit_from: ../.rubocop.yml diff --git a/fit-performer/Dockerfile b/fit-performer/Dockerfile new file mode 100644 index 00000000..cde46e05 --- /dev/null +++ b/fit-performer/Dockerfile @@ -0,0 +1,15 @@ +FROM ruby:4.0-trixie + +WORKDIR /app +COPY ./ ./ + +RUN apt-get update && apt-get install -y cmake build-essential libssl-dev protobuf-compiler + +RUN bundle install +RUN bundle exec rake compile + +WORKDIR /app/fit-performer + +RUN bundle exec rake generate + +ENTRYPOINT ["bundle", "exec", "ruby", "server.rb"] diff --git a/fit-performer/README.md b/fit-performer/README.md new file mode 100644 index 00000000..78f9a6f1 --- /dev/null +++ b/fit-performer/README.md @@ -0,0 +1,35 @@ +# Couchbase Ruby FIT performer + +This is the FIT performer for the Couchbase Ruby Client. + +## Setup + +### Generated code + +To generate the Ruby code corresponding to the Protocol Buffers and gRPC definitions: + +```shell +bundle exec rake generate +``` + +### Updating the protocol + +The FIT protocol is hosted in https://github.com/couchbaselabs/fit-protocol. There is a copy of the protocol in this repository (in `/fit-performer/proto`) the performer is built against. + +The latest version of the protocol can be fetched by running: +```shell +bundle exec rake update_protocol +``` + +## Starting the performer + +```shell +bundle exec ruby server.rb +``` + +## Docker + +We provide a dockerfile to build and run the performer with Docker. You can build the image from the root of this repository as follows: +```shell +docker build -f fit-performer/Dockerfile . +``` diff --git a/fit-performer/Rakefile b/fit-performer/Rakefile new file mode 100644 index 00000000..f5b48c7d --- /dev/null +++ b/fit-performer/Rakefile @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "rubocop/rake_task" + +require "mkmf" + +RuboCop::RakeTask.new + +desc "Re-generate gRPC server implementation" +task :generate_grpc_server do + protoc_binary = find_executable("grpc_tools_ruby_protoc") + grpc_dir = File.expand_path("proto", __dir__) + proto_files = Dir["#{grpc_dir}/*.proto"] + output_dir = File.expand_path("lib/fit/protocol", __dir__) + rm_rf(output_dir) + mkdir(output_dir) + sh(protoc_binary, + "--proto_path=#{grpc_dir.shellescape}", + "--grpc_out=#{output_dir.shellescape}", + "--ruby_out=#{output_dir.shellescape}", + *proto_files) + Dir["#{output_dir}/*_pb.rb"].each do |file| + content = File.read(file) + .gsub(/^require ["']([\w.]+)_pb['"]/, "require_relative '\\1_pb'") + File.write(file, content) + end +end + +desc "Re-generate gRPC server implementation and reformat using Rubocop" +task generate: %w[generate_grpc_server rubocop:autocorrect_all] + +desc "Fetch the latest FIT protocol" +task :update_protocol do + root_dir = File.expand_path("..", __dir__) + sh(File.join(root_dir, "bin", "update-fit-protocol"), chdir: root_dir) +end diff --git a/fit-performer/lib/fit.rb b/fit-performer/lib/fit.rb new file mode 100644 index 00000000..d929d089 --- /dev/null +++ b/fit-performer/lib/fit.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module FIT +end diff --git a/fit-performer/lib/fit/performer.rb b/fit-performer/lib/fit/performer.rb new file mode 100644 index 00000000..17c8ee88 --- /dev/null +++ b/fit-performer/lib/fit/performer.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module FIT + module Performer + end +end diff --git a/fit-performer/lib/fit/performer/commands.rb b/fit-performer/lib/fit/performer/commands.rb new file mode 100644 index 00000000..605fe1da --- /dev/null +++ b/fit-performer/lib/fit/performer/commands.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'google/protobuf/timestamp_pb' + +require_relative 'commands/key_value' +require_relative 'commands/query_index_manager' +require_relative 'commands/query' +require_relative 'commands/search_index_manager' +require_relative 'commands/search' +require_relative 'commands/collection_manager' +require_relative 'commands/bucket_manager' +require_relative 'commands/misc/update_authenticator_command' + +module FIT + module Performer + module Commands + def self.build_command(raw_command, connection, span_owner, executed_cmd_count) + cluster = connection.cluster + cmd_type = raw_command.command + kv_cmd_types = [:insert, :get, :remove, :replace, :upsert, :range_scan] + + current_time = Time.now.to_f + seconds = current_time.truncate + nanos = ((current_time - seconds) * (10**9)).round + + cmd_kwargs = { + initiated: Google::Protobuf::Timestamp.new(seconds: seconds, nanos: nanos), + return_result: raw_command.return_result, + get_span_fn: lambda { |span_id| span_owner.get_span(span_id) }, + } + + if cmd_type == :cluster_command + cluster_cmd = raw_command.cluster_command + cluster_cmd_type = cluster_cmd.command + case cluster_cmd_type + when :query_index_manager + Commands::QueryIndexManager.build_cluster_level_command(cluster_cmd.query_index_manager, cluster, cmd_kwargs) + when :query + Commands::Query.build_cluster_level_command(cluster_cmd.query, cluster, cmd_kwargs) + when :search_index_manager + Commands::SearchIndexManager.build_cluster_level_command(cluster_cmd.search_index_manager, cluster, cmd_kwargs) + when :bucket_manager + Commands::BucketManager.build_command(cluster_cmd.bucket_manager, cluster, cmd_kwargs) + when :search + Commands::Search.build_cluster_level_command(cluster_cmd.search, cluster, cmd_kwargs) + when :search_v2 + Commands::Search.build_cluster_level_command(cluster_cmd.search_v2, cluster, cmd_kwargs) + when :authenticator + Commands::Misc::UpdateAuthenticatorCommand.create_command( + authenticator: connection.create_authenticator(cluster_cmd.authenticator), + cluster: cluster, + **cmd_kwargs, + ) + else + raise PerformerError, "Cluster-level command `#{cluster_cmd_type}` not implemented" + end + + elsif cmd_type == :bucket_command + bucket_cmd = raw_command.bucket_command + bucket_cmd_type = bucket_cmd.command + bucket = cluster.bucket(bucket_cmd.bucket_name) + case bucket_cmd_type + when :collection_manager + Commands::CollectionManager.build_command(bucket_cmd.collection_manager, bucket, cmd_kwargs) + else + raise PerformerError, "Bucket-level command `#{bucket_cmd_type}` not implemented" + end + + elsif cmd_type == :scope_command + scope_cmd = raw_command.scope_command + scope = scope_cmd.scope.then do |s| + cluster.bucket(s.bucket_name).scope(s.scope_name) + end + scope_cmd_type = scope_cmd.command + case scope_cmd_type + when :query + Commands::Query.build_scope_level_command(scope_cmd.query, scope, cmd_kwargs) + when :search + Commands::Search.build_scope_level_command(scope_cmd.search, scope, cmd_kwargs) + when :search_v2 + Commands::Search.build_scope_level_command(scope_cmd.search_v2, scope, cmd_kwargs) + when :search_index_manager + Commands::SearchIndexManager.build_scope_level_command(scope_cmd.search_index_manager, scope, cmd_kwargs) + else + raise PerformerError, "Scope-level command `#{scope_cmd_type}` not implemented" + end + + elsif cmd_type == :collection_command + collection_cmd = raw_command.collection_command + collection = nil + if collection_cmd.has_collection? + collection = collection_cmd.collection.then do |c| + cluster.bucket(c.bucket_name).scope(c.scope_name).collection(c.collection_name) + end + end + collection_cmd_type = collection_cmd.command + case collection_cmd_type + when :query_index_manager + Commands::QueryIndexManager.build_collection_level_command(collection_cmd.query_index_manager, collection, cmd_kwargs) + when *Commands::KeyValue::SUPPORTED_COMMANDS + kv_cmd = collection_cmd.public_send(collection_cmd_type) + Commands::KeyValue.build_command(kv_cmd, collection_cmd_type, cluster, executed_cmd_count, cmd_kwargs) + else + raise PerformerError, "Collection-level command `#{collection_cmd_type}` not implemented" + end + + elsif kv_cmd_types.include?(cmd_type) + raw_kv_cmd = raw_command.public_send(cmd_type) + Commands::KeyValue.build_command(raw_kv_cmd, cmd_type, cluster, executed_cmd_count, cmd_kwargs) + + else + raise PerformerError, "Command type `#{cmd_type}` not recognised" + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/bucket_manager.rb b/fit-performer/lib/fit/performer/commands/bucket_manager.rb new file mode 100644 index 00000000..d11f1208 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/bucket_manager.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'bucket_manager/get_bucket_command' +require_relative 'bucket_manager/get_all_buckets_command' +require_relative 'bucket_manager/create_bucket_command' +require_relative 'bucket_manager/drop_bucket_command' +require_relative 'bucket_manager/flush_bucket_command' +require_relative 'bucket_manager/update_bucket_command' + +module FIT + module Performer + module Commands + module BucketManager + def self.build_command(raw_cmd, cluster, cmd_kwargs) + cmd_type = raw_cmd.command + cmd = raw_cmd.public_send(cmd_type) + + cmd_kwargs[:manager] = cluster.buckets + cmd_kwargs[:raw_options] = cmd.options if cmd.has_options? + + case cmd_type + when :get_bucket + cmd_kwargs[:bucket_name] = cmd.bucket_name + GetBucketCommand.create_command(**cmd_kwargs) + when :get_all_buckets + GetAllBucketsCommand.create_command(**cmd_kwargs) + when :create_bucket + cmd_kwargs[:settings] = get_create_bucket_settings(cmd) + CreateBucketCommand.create_command(**cmd_kwargs) + when :drop_bucket + cmd_kwargs[:bucket_name] = cmd.bucket_name + DropBucketCommand.create_command(**cmd_kwargs) + when :flush_bucket + cmd_kwargs[:bucket_name] = cmd.bucket_name + FlushBucketCommand.create_command(**cmd_kwargs) + when :update_bucket + cmd_kwargs[:settings] = get_bucket_settings(cmd) + UpdateBucketCommand.create_command(**cmd_kwargs) + else + raise PerformerError, "Bucket management command `#{cmd_type}` not supported" + end + end + + def self.get_bucket_settings(raw_mgmt_cmd) + proto_settings = raw_mgmt_cmd.settings + + Couchbase::Management::BucketSettings.new do |s| + s.name = proto_settings.name + s.flush_enabled = proto_settings.flush_enabled if proto_settings.has_flush_enabled? + s.ram_quota_mb = proto_settings.ram_quota_MB + s.num_replicas = proto_settings.num_replicas if proto_settings.has_num_replicas? + s.replica_indexes = proto_settings.replica_indexes if proto_settings.has_replica_indexes? + s.bucket_type = proto_settings.bucket_type.downcase if proto_settings.has_bucket_type? + s.eviction_policy = proto_settings.eviction_policy.downcase if proto_settings.has_eviction_policy? + s.max_expiry = proto_settings.max_expiry_seconds if proto_settings.has_max_expiry_seconds? + s.compression_mode = proto_settings.compression_mode.downcase if proto_settings.has_compression_mode? + s.minimum_durability_level = proto_settings.minimum_durability_level.downcase if proto_settings.has_minimum_durability_level? + s.storage_backend = proto_settings.storage_backend.downcase if proto_settings.has_storage_backend? + if proto_settings.has_history_retention_collection_default? + s.history_retention_collection_default = proto_settings.history_retention_collection_default + end + s.history_retention_duration = proto_settings.history_retention_seconds if proto_settings.has_history_retention_seconds? + s.history_retention_bytes = proto_settings.history_retention_bytes if proto_settings.has_history_retention_bytes? + s.num_vbuckets = proto_settings.num_vbuckets if proto_settings.has_num_vbuckets? + end + end + + def self.get_create_bucket_settings(raw_mgmt_cmd) + proto_create_settings = raw_mgmt_cmd.settings + settings = get_bucket_settings(proto_create_settings) + if proto_create_settings.has_conflict_resolution_type? + settings.conflict_resolution_type = proto_create_settings.conflict_resolution_type.downcase + end + settings + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/bucket_manager/create_bucket_command.rb b/fit-performer/lib/fit/performer/commands/bucket_manager/create_bucket_command.rb new file mode 100644 index 00000000..f0b199dc --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/bucket_manager/create_bucket_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module BucketManager + class CreateBucketCommand + def initialize(manager:, settings:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @cmd_args = [settings] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Bucket::CreateBucket.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.create_bucket(*@cmd_args) + end + end + + def self.create_command(...) + cmd = CreateBucketCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/bucket_manager/drop_bucket_command.rb b/fit-performer/lib/fit/performer/commands/bucket_manager/drop_bucket_command.rb new file mode 100644 index 00000000..666c4417 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/bucket_manager/drop_bucket_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module BucketManager + class DropBucketCommand + def initialize(manager:, bucket_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @cmd_args = [bucket_name] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Bucket::DropBucket.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.drop_bucket(*@cmd_args) + end + end + + def self.create_command(...) + cmd = DropBucketCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/bucket_manager/flush_bucket_command.rb b/fit-performer/lib/fit/performer/commands/bucket_manager/flush_bucket_command.rb new file mode 100644 index 00000000..7cbea9b7 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/bucket_manager/flush_bucket_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module BucketManager + class FlushBucketCommand + def initialize(manager:, bucket_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @cmd_args = [bucket_name] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Bucket::FlushBucket.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.flush_bucket(*@cmd_args) + end + end + + def self.create_command(...) + cmd = FlushBucketCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/bucket_manager/get_all_buckets_command.rb b/fit-performer/lib/fit/performer/commands/bucket_manager/get_all_buckets_command.rb new file mode 100644 index 00000000..0086ced7 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/bucket_manager/get_all_buckets_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module BucketManager + class GetAllBucketsCommand + def initialize(manager:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @cmd_args = [] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Bucket::GetAllBuckets.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_get_all_buckets_result(initiated: @initiated, return_result: @return_result) do + @manager.get_all_buckets(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetAllBucketsCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/bucket_manager/get_bucket_command.rb b/fit-performer/lib/fit/performer/commands/bucket_manager/get_bucket_command.rb new file mode 100644 index 00000000..10e10a9f --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/bucket_manager/get_bucket_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module BucketManager + class GetBucketCommand + def initialize(manager:, bucket_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @cmd_args = [bucket_name] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Bucket::GetBucket.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_bucket_settings(initiated: @initiated, return_result: @return_result) do + @manager.get_bucket(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetBucketCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/bucket_manager/options_builder.rb b/fit-performer/lib/fit/performer/commands/bucket_manager/options_builder.rb new file mode 100644 index 00000000..a190967c --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/bucket_manager/options_builder.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative '../options_builder_base' + +module FIT + module Performer + module Commands + module BucketManager + class OptionsBuilder < OptionsBuilderBase + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/bucket_manager/results.rb b/fit-performer/lib/fit/performer/commands/bucket_manager/results.rb new file mode 100644 index 00000000..ff76110e --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/bucket_manager/results.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative '../shared_results' + +module FIT + module Performer + module Commands + module BucketManager + class Results < SharedResults + def self.as_get_all_buckets_result(return_result:, initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + if return_result + create_run_result( + sdk_result: Protocol::SDK::Result.new( + bucket_manager_result: Protocol::SDK::Cluster::BucketManager::Result.new( + get_all_buckets_result: to_get_all_buckets_result(result: result), + ), + ), + initiated: initiated, + elapsed_nanos: nanos, + ) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.as_bucket_settings(return_result:, initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + if return_result + create_run_result( + sdk_result: Protocol::SDK::Result.new( + bucket_manager_result: Protocol::SDK::Cluster::BucketManager::Result.new( + bucket_settings: to_bucket_settings(result: result), + ), + ), + initiated: initiated, + elapsed_nanos: nanos, + ) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.to_get_all_buckets_result(result:) + proto_result = Protocol::SDK::Cluster::BucketManager::GetAllBucketsResult.new + result.each do |settings| + proto_result.result[settings.name] = to_bucket_settings(result: settings) + end + proto_result + end + + def self.to_bucket_settings(result:) + proto_result = Protocol::SDK::Cluster::BucketManager::BucketSettings.new( + name: result.name, ram_quota_MB: result.ram_quota_mb, + ) + proto_result.flush_enabled = result.flush_enabled unless result.flush_enabled.nil? + proto_result.num_replicas = result.num_replicas unless result.num_replicas.nil? + proto_result.replica_indexes = result.replica_indexes unless result.replica_indexes.nil? + proto_result.bucket_type = result.bucket_type.upcase unless result.bucket_type.nil? + proto_result.eviction_policy = result.eviction_policy.upcase unless result.eviction_policy.nil? + proto_result.max_expiry_seconds = result.max_expiry unless result.max_expiry.nil? + proto_result.compression_mode = result.compression_mode.upcase unless result.compression_mode.nil? + proto_result.minimum_durability_level = result.minimum_durability_level.upcase unless result.minimum_durability_level.nil? + proto_result.storage_backend = result.storage_backend.upcase unless result.storage_backend.nil? + unless result.history_retention_collection_default.nil? + proto_result.history_retention_collection_default = result.history_retention_collection_default + end + proto_result.history_retention_seconds = result.history_retention_duration unless result.history_retention_duration.nil? + proto_result.history_retention_bytes = result.history_retention_bytes unless result.history_retention_bytes.nil? + proto_result.num_vbuckets = result.num_vbuckets unless result.num_vbuckets.nil? + proto_result + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/bucket_manager/update_bucket_command.rb b/fit-performer/lib/fit/performer/commands/bucket_manager/update_bucket_command.rb new file mode 100644 index 00000000..6421af5c --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/bucket_manager/update_bucket_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module BucketManager + class UpdateBucketCommand + def initialize(manager:, settings:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @cmd_args = [settings] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Bucket::UpdateBucket.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.update_bucket(*@cmd_args) + end + end + + def self.create_command(...) + cmd = UpdateBucketCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/collection_manager.rb b/fit-performer/lib/fit/performer/commands/collection_manager.rb new file mode 100644 index 00000000..765402a6 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/collection_manager.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fit/protocol/sdk.bucket.collection_manager_pb' + +require_relative 'collection_manager/get_all_scopes_command' +require_relative 'collection_manager/create_scope_command' +require_relative 'collection_manager/drop_scope_command' +require_relative 'collection_manager/create_collection_command' +require_relative 'collection_manager/update_collection_command' +require_relative 'collection_manager/drop_collection_command' + +module FIT + module Performer + module Commands + module CollectionManager + def self.build_command(raw_cmd, bucket, cmd_kwargs) + cmd_type = raw_cmd.command + cmd = raw_cmd.public_send(cmd_type) + + cmd_kwargs[:manager] = bucket.collections + cmd_kwargs[:raw_options] = cmd.options if cmd.has_options? + + case cmd_type + when :get_all_scopes + GetAllScopesCommand.create_command(**cmd_kwargs) + when :create_scope + cmd_kwargs[:scope_name] = cmd.name + CreateScopeCommand.create_command(**cmd_kwargs) + when :drop_scope + cmd_kwargs[:scope_name] = cmd.name + DropScopeCommand.create_command(**cmd_kwargs) + when :create_collection + cmd_kwargs[:scope_name] = cmd.scope_name + cmd_kwargs[:collection_name] = cmd.name + cmd_kwargs[:settings] = get_create_collection_settings(cmd) + CreateCollectionCommand.create_command(**cmd_kwargs) + when :update_collection + cmd_kwargs[:scope_name] = cmd.scope_name + cmd_kwargs[:collection_name] = cmd.name + cmd_kwargs[:settings] = get_update_collection_settings(cmd) + UpdateCollectionCommand.create_command(**cmd_kwargs) + when :drop_collection + cmd_kwargs[:scope_name] = cmd.scope_name + cmd_kwargs[:collection_name] = cmd.name + DropCollectionCommand.create_command(**cmd_kwargs) + else + raise PerformerError, "Collection management command `#{cmd_type}` not supported" + end + end + + def self.get_create_collection_settings(raw_mgmt_cmd) + return nil unless raw_mgmt_cmd.has_settings? + + Couchbase::Management::CreateCollectionSettings.new do |settings| + settings.max_expiry = raw_mgmt_cmd.settings.expiry_secs if raw_mgmt_cmd.settings.has_expiry_secs? + settings.history = raw_mgmt_cmd.settings.history if raw_mgmt_cmd.settings.has_history? + end + end + + def self.get_update_collection_settings(raw_mgmt_cmd) + Couchbase::Management::UpdateCollectionSettings.new do |settings| + settings.max_expiry = raw_mgmt_cmd.settings.expiry_secs if raw_mgmt_cmd.settings.has_expiry_secs? + settings.history = raw_mgmt_cmd.settings.history if raw_mgmt_cmd.settings.has_history? + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/collection_manager/create_collection_command.rb b/fit-performer/lib/fit/performer/commands/collection_manager/create_collection_command.rb new file mode 100644 index 00000000..0a1d43f0 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/collection_manager/create_collection_command.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/management/collection_manager' + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module CollectionManager + class CreateCollectionCommand + def initialize(manager:, scope_name:, collection_name:, initiated:, get_span_fn:, settings: nil, return_result: true, + raw_options: nil) + @manager = manager + @cmd_args = [scope_name, collection_name, settings || Couchbase::Management::CreateCollectionSettings::DEFAULT] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Collection::CreateCollection.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.create_collection(*@cmd_args) + end + end + + def self.create_command(...) + cmd = CreateCollectionCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/collection_manager/create_scope_command.rb b/fit-performer/lib/fit/performer/commands/collection_manager/create_scope_command.rb new file mode 100644 index 00000000..63ded4e2 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/collection_manager/create_scope_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module CollectionManager + class CreateScopeCommand + def initialize(manager:, scope_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @cmd_args = [scope_name] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Collection::CreateScope.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.create_scope(*@cmd_args) + end + end + + def self.create_command(...) + cmd = CreateScopeCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/collection_manager/drop_collection_command.rb b/fit-performer/lib/fit/performer/commands/collection_manager/drop_collection_command.rb new file mode 100644 index 00000000..2857adcf --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/collection_manager/drop_collection_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module CollectionManager + class DropCollectionCommand + def initialize(manager:, scope_name:, collection_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @cmd_args = [scope_name, collection_name] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Collection::DropCollection.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.drop_collection(*@cmd_args) + end + end + + def self.create_command(...) + cmd = DropCollectionCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/collection_manager/drop_scope_command.rb b/fit-performer/lib/fit/performer/commands/collection_manager/drop_scope_command.rb new file mode 100644 index 00000000..b8096386 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/collection_manager/drop_scope_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module CollectionManager + class DropScopeCommand + def initialize(manager:, scope_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @cmd_args = [scope_name] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Collection::DropScope.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.drop_scope(*@cmd_args) + end + end + + def self.create_command(...) + cmd = DropScopeCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/collection_manager/get_all_scopes_command.rb b/fit-performer/lib/fit/performer/commands/collection_manager/get_all_scopes_command.rb new file mode 100644 index 00000000..fb115879 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/collection_manager/get_all_scopes_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module CollectionManager + class GetAllScopesCommand + def initialize(manager:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @cmd_args = [] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Collection::GetAllScopes.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_get_all_scopes_result(return_result: @return_result, initiated: @initiated) do + @manager.get_all_scopes(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetAllScopesCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/collection_manager/options_builder.rb b/fit-performer/lib/fit/performer/commands/collection_manager/options_builder.rb new file mode 100644 index 00000000..a1fa7941 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/collection_manager/options_builder.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative '../options_builder_base' + +module FIT + module Performer + module Commands + module CollectionManager + class OptionsBuilder < OptionsBuilderBase + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/collection_manager/results.rb b/fit-performer/lib/fit/performer/commands/collection_manager/results.rb new file mode 100644 index 00000000..671de6d2 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/collection_manager/results.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fit/protocol/sdk.bucket.collection_manager_pb' + +require_relative '../shared_results' + +module FIT + module Performer + module Commands + module CollectionManager + class Results < SharedResults + def self.as_get_all_scopes_result(return_result:, initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + if return_result + to_get_all_scopes_result(result: result, initiated: initiated, elapsed_nanos: nanos) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.to_get_all_scopes_result(result:, initiated:, elapsed_nanos:) + proto_result = Protocol::SDK::Bucket::CollectionManager::GetAllScopesResult.new + result.each do |scope_spec| + proto_scope_spec = Protocol::SDK::Bucket::CollectionManager::ScopeSpec.new(name: scope_spec.name) + scope_spec.collections.each do |coll_spec| + proto_coll_spec = Protocol::SDK::Bucket::CollectionManager::CollectionSpec.new( + name: coll_spec.name, + scope_name: coll_spec.scope_name, + ) + proto_coll_spec.expiry_secs = coll_spec.max_expiry unless coll_spec.max_expiry.nil? + proto_coll_spec.history = coll_spec.history unless coll_spec.history.nil? + + proto_scope_spec.collections << proto_coll_spec + end + proto_result.result << proto_scope_spec + end + + create_run_result( + sdk_result: Protocol::SDK::Result.new( + collection_manager_result: Protocol::SDK::Bucket::CollectionManager::Result.new( + get_all_scopes_result: proto_result, + ), + ), + initiated: initiated, + elapsed_nanos: elapsed_nanos, + ) + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/collection_manager/update_collection_command.rb b/fit-performer/lib/fit/performer/commands/collection_manager/update_collection_command.rb new file mode 100644 index 00000000..cb7b00d3 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/collection_manager/update_collection_command.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module CollectionManager + class UpdateCollectionCommand + def initialize(manager:, scope_name:, collection_name:, settings:, initiated:, get_span_fn:, return_result: true, + raw_options: nil) + @manager = manager + @cmd_args = [scope_name, collection_name, settings] + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Collection::UpdateCollection.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.update_collection(*@cmd_args) + end + end + + def self.create_command(...) + cmd = UpdateCollectionCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value.rb b/fit-performer/lib/fit/performer/commands/key_value.rb new file mode 100644 index 00000000..4337cdad --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value.rb @@ -0,0 +1,304 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/subdoc' + +require 'securerandom' + +require_relative 'key_value/get_command' +require_relative 'key_value/insert_command' +require_relative 'key_value/remove_command' +require_relative 'key_value/replace_command' +require_relative 'key_value/upsert_command' +require_relative 'key_value/get_and_lock_command' +require_relative 'key_value/get_and_touch_command' +require_relative 'key_value/exists_command' +require_relative 'key_value/touch_command' +require_relative 'key_value/unlock_command' +require_relative 'key_value/scan_command' +require_relative 'key_value/lookup_in_command' +require_relative 'key_value/lookup_in_any_replica_command' +require_relative 'key_value/lookup_in_all_replicas_command' +require_relative 'key_value/mutate_in_command' +require_relative 'key_value/append_command' +require_relative 'key_value/prepend_command' +require_relative 'key_value/increment_command' +require_relative 'key_value/decrement_command' +require_relative 'key_value/get_any_replica_command' +require_relative 'key_value/get_all_replicas_command' + +module FIT + module Performer + module Commands + module KeyValue # rubocop:disable Metrics/ModuleLength + SUPPORTED_COMMANDS = [ + :lookup_in, :lookup_in_all_replicas, :lookup_in_any_replica, :get_and_lock, :unlock, :get_and_touch, + :exists, :touch, :binary, :mutate_in, :get_any_replica, :get_all_replicas + ].freeze + + def self.build_command(raw_kv_cmd, kv_cmd_type, cluster, executed_cmd_count, cmd_kwargs) + return build_binary_command(raw_kv_cmd, cluster, executed_cmd_count, cmd_kwargs) if kv_cmd_type == :binary + + cmd_kwargs[:collection] = get_collection(raw_kv_cmd, cluster) + cmd_kwargs[:raw_options] = raw_kv_cmd.options if raw_kv_cmd.has_options? + + cmd_kwargs[:doc_id] = get_doc_id(raw_kv_cmd.location, executed_cmd_count) if kv_cmd_type != :range_scan + + case kv_cmd_type + when :insert + cmd_kwargs[:content] = get_content(raw_kv_cmd.content) + InsertCommand.create_command(**cmd_kwargs) + when :get + cmd_kwargs[:content_as] = raw_kv_cmd.content_as + GetCommand.create_command(**cmd_kwargs) + when :remove + RemoveCommand.create_command(**cmd_kwargs) + when :replace + cmd_kwargs[:content] = get_content(raw_kv_cmd.content) + ReplaceCommand.create_command(**cmd_kwargs) + when :upsert + cmd_kwargs[:content] = get_content(raw_kv_cmd.content) + UpsertCommand.create_command(**cmd_kwargs) + when :range_scan + cmd_kwargs.update({ + raw_scan_type: raw_kv_cmd.scan_type, + stream_config: raw_kv_cmd.stream_config, + }) + cmd_kwargs[:content_as] = raw_kv_cmd.content_as if raw_kv_cmd.has_content_as? + ScanCommand.create_command(**cmd_kwargs) + when :lookup_in + cmd_kwargs[:specs] = get_lookup_in_specs(raw_kv_cmd) + cmd_kwargs[:raw_specs] = raw_kv_cmd.spec # Used for per-spec content_as + LookupInCommand.create_command(**cmd_kwargs) + when :lookup_in_any_replica + cmd_kwargs[:specs] = get_lookup_in_specs(raw_kv_cmd) + cmd_kwargs[:raw_specs] = raw_kv_cmd.spec # Used for per-spec content_as + LookupInAnyReplicaCommand.create_command(**cmd_kwargs) + when :lookup_in_all_replicas + cmd_kwargs.update({ + specs: get_lookup_in_specs(raw_kv_cmd), + raw_specs: raw_kv_cmd.spec, # Used for per-spec content_as + stream_config: raw_kv_cmd.stream_config, + }) + LookupInAllReplicasCommand.create_command(**cmd_kwargs) + when :mutate_in + cmd_kwargs[:specs] = get_mutate_in_specs(raw_kv_cmd) + cmd_kwargs[:raw_specs] = raw_kv_cmd.spec + MutateInCommand.create_command(**cmd_kwargs) + when :get_and_lock + cmd_kwargs[:lock_time] = raw_kv_cmd.duration.seconds + cmd_kwargs[:content_as] = raw_kv_cmd.content_as + GetAndLockCommand.create_command(**cmd_kwargs) + when :get_and_touch + cmd_kwargs[:expiry] = get_expiry(raw_kv_cmd) + cmd_kwargs[:content_as] = raw_kv_cmd.content_as + GetAndTouchCommand.create_command(**cmd_kwargs) + when :exists + ExistsCommand.create_command(**cmd_kwargs) + when :touch + cmd_kwargs[:expiry] = get_expiry(raw_kv_cmd) + TouchCommand.create_command(**cmd_kwargs) + when :unlock + cmd_kwargs[:cas] = raw_kv_cmd.cas + UnlockCommand.create_command(**cmd_kwargs) + when :get_any_replica + cmd_kwargs[:content_as] = raw_kv_cmd.content_as + GetAnyReplicaCommand.create_command(**cmd_kwargs) + when :get_all_replicas + cmd_kwargs.update({ + content_as: raw_kv_cmd.content_as, + stream_config: raw_kv_cmd.stream_config, + }) + GetAllReplicasCommand.create_command(**cmd_kwargs) + else + raise PerformerError, "KV command type `#{kv_cmd_type}` not supported" + end + end + + def self.build_binary_command(raw_cmd, cluster, executed_cmd_count, cmd_kwargs) + binary_cmd_type = raw_cmd.command + raw_binary_cmd = raw_cmd.public_send(binary_cmd_type) + + cmd_kwargs[:binary_collection] = get_collection(raw_binary_cmd, cluster).binary + cmd_kwargs[:doc_id] = get_doc_id(raw_binary_cmd.location, executed_cmd_count) + cmd_kwargs[:raw_options] = raw_binary_cmd.options if raw_binary_cmd.has_options? + + case binary_cmd_type + when :append + cmd_kwargs[:content] = raw_binary_cmd.content + AppendCommand.create_command(**cmd_kwargs) + when :prepend + cmd_kwargs[:content] = raw_binary_cmd.content + PrependCommand.create_command(**cmd_kwargs) + when :increment + IncrementCommand.create_command(**cmd_kwargs) + when :decrement + DecrementCommand.create_command(**cmd_kwargs) + else + raise PerformerError, "Binary KV command type `#{binary_cmd_type}` not supported" + end + end + + def self.get_doc_id(raw_doc_location, executed_cmd_count) + location_type = raw_doc_location.location + case location_type + when :specific + raw_doc_location.specific.id + when :uuid + SecureRandom.uuid + when :pool + preface = raw_doc_location.pool.id_preface + pool_size = raw_doc_location.pool.pool_size + selection_strategy = raw_doc_location.pool.poolSelectionStrategy + case selection_strategy + when :random + distribution = raw_doc_location.pool.random.distribution + case distribution + when :RANDOM_DISTRIBUTION_UNIFORM + preface + rand(pool_size).to_s + else + raise PerformerError, "Random distribution `#{distribution}` not supported for random pool selection strategy" + end + when :counter + preface + (executed_cmd_count % pool_size).to_s + else + raise PerformerError, "Pool selection strategy `#{selection_strategy}` not supported" + end + else + raise PerformerError, "Document location type `#{raw_doc_location}` not supported" + end + end + + def self.get_collection(raw_kv_cmd, cluster) + raw_coll = + if raw_kv_cmd.respond_to?(:location) + raw_doc_location = raw_kv_cmd.location + raw_doc_location.public_send(raw_doc_location.location).collection + else + raw_kv_cmd.collection + end + cluster.bucket(raw_coll.bucket_name).scope(raw_coll.scope_name).collection(raw_coll.collection_name) + end + + def self.get_content(raw_content) + content_type = raw_content.content + case content_type + when :passthrough_string + raw_content.passthrough_string + when :convert_to_json + JSON.parse(raw_content.convert_to_json) + when :null + nil + when :byte_array + raw_content.byte_array + else + raise PerformerError, "Content type `#{content_type}` not supported" + end + end + + def self.get_expiry(raw_kv_cmd) + expiry_type = raw_kv_cmd.expiry.expiryType + case expiry_type + when :relativeSecs + raw_kv_cmd.expiry.relativeSecs + when :absoluteEpochSecs + Time.at(raw_kv_cmd.expiry.absoluteEpochSecs) + else + raise PerformerError, "Expiry type `#{expiry_type}` not supported" + end + end + + def self.get_lookup_in_specs(raw_kv_cmd) + raw_kv_cmd.spec.map do |raw_spec| + spec = + case raw_spec.operation + when :exists + Couchbase::LookupInSpec.exists(raw_spec.exists.path) + when :get + Couchbase::LookupInSpec.get(raw_spec.get.path) + when :count + Couchbase::LookupInSpec.count(raw_spec.count.path) + else + raise PerformerError, "LookupIn spec type `#{raw_spec.operation}` not supported" + end + op = raw_spec.public_send(raw_spec.operation) + spec.xattr if op.has_xattr? && op.xattr + spec + end + end + + def self.get_mutate_in_specs(raw_kv_cmd) + raw_kv_cmd.spec.map do |raw_spec| + op = raw_spec.public_send(raw_spec.operation) + spec = + case raw_spec.operation + when :upsert + Couchbase::MutateInSpec.upsert(op.path, get_mutate_in_value(op.content)) + when :insert + Couchbase::MutateInSpec.insert(op.path, get_mutate_in_value(op.content)) + when :replace + Couchbase::MutateInSpec.replace(op.path, get_mutate_in_value(op.content)) + when :remove + Couchbase::MutateInSpec.remove(op.path) + when :array_append + Couchbase::MutateInSpec.array_append(op.path, get_mutate_in_values(op.content)) + when :array_prepend + Couchbase::MutateInSpec.array_prepend(op.path, get_mutate_in_values(op.content)) + when :array_insert + Couchbase::MutateInSpec.array_insert(op.path, get_mutate_in_values(op.content)) + when :array_add_unique + Couchbase::MutateInSpec.array_add_unique(op.path, get_mutate_in_value(op.content)) + when :increment + Couchbase::MutateInSpec.increment(op.path, op.delta) + when :decrement + Couchbase::MutateInSpec.decrement(op.path, op.delta) + else + raise PerformerError, "MutateIn spec type `#{raw_spec.operation}` not supported" + end + spec.xattr if op.has_xattr? && op.xattr + spec.create_path if op.respond_to?(:create_path) && op.has_create_path? && op.create_path + spec + end + end + + def self.get_mutate_in_value(proto_content) + type = proto_content.content_or_macro + case type + when :content + get_content(proto_content.content) + when :macro + case proto_content.macro + when :CAS + :cas + when :SEQ_NO + :seq_no + when :VALUE_CRC_32C + :value_crc32c + else + raise PerformerError, "Unexpected MutateInMacro `#{content_or_macro.macro}" + end + else + raise PerformerError, "Unexpected ContentOrMacro type `#{type}`" + end + end + + def self.get_mutate_in_values(proto_contents) + proto_contents.map { |proto_content| get_mutate_in_value(proto_content) } + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/append_command.rb b/fit-performer/lib/fit/performer/commands/key_value/append_command.rb new file mode 100644 index 00000000..eac54456 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/append_command.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module KeyValue + class AppendCommand + def initialize(binary_collection:, doc_id:, content:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @binary_collection = binary_collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id, content] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Append.new, raw_options: @raw_options) + builder.set_timeout + .set_durability + .set_cas + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_mutation_result(return_result: @return_result, initiated: @initiated) do + @binary_collection.append(*@cmd_args) + end + end + + def self.create_command(...) + cmd = AppendCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/decrement_command.rb b/fit-performer/lib/fit/performer/commands/key_value/decrement_command.rb new file mode 100644 index 00000000..1e16154a --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/decrement_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module KeyValue + class DecrementCommand + def initialize(binary_collection:, doc_id:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @binary_collection = binary_collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Decrement.new, raw_options: @raw_options) + builder.set_timeout + .set_expiry + .set_delta + .set_initial + .set_durability + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_counter_result(return_result: @return_result, initiated: @initiated) do + @binary_collection.decrement(*@cmd_args) + end + end + + def self.create_command(...) + cmd = DecrementCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/exists_command.rb b/fit-performer/lib/fit/performer/commands/key_value/exists_command.rb new file mode 100644 index 00000000..dfc36b91 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/exists_command.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class ExistsCommand + def initialize(collection:, doc_id:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Exists.new, raw_options: @raw_options) + builder.set_timeout + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_exists_result(return_result: @return_result, initiated: @initiated) do + @collection.exists(*@cmd_args) + end + end + + def self.create_command(...) + cmd = ExistsCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/get_all_replicas_command.rb b/fit-performer/lib/fit/performer/commands/key_value/get_all_replicas_command.rb new file mode 100644 index 00000000..1713ba0e --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/get_all_replicas_command.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/options' + +require 'logger' + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class GetAllReplicasCommand + attr_reader :stream_config + + def initialize(collection:, doc_id:, initiated:, return_result:, content_as:, stream_config:, get_span_fn:, raw_options: nil) + @logger = Logger.new($stdout) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id] + @content_as = content_as + @stream_config = stream_config + @get_span_fn = get_span_fn + end + + def stream_type + :STREAM_KV_GET_ALL_REPLICAS + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::GetAllReplicas.new, raw_options: @raw_options) + builder.set_timeout + .set_transcoder + .set_read_preference + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_get_all_replicas_result_stream(initiated: @initiated, content_as: @content_as, + stream_id: @stream_config.stream_id) do + @collection.get_all_replicas(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetAllReplicasCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/get_and_lock_command.rb b/fit-performer/lib/fit/performer/commands/key_value/get_and_lock_command.rb new file mode 100644 index 00000000..71362679 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/get_and_lock_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'logger' + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class GetAndLockCommand + def initialize(collection:, doc_id:, lock_time:, initiated:, return_result:, content_as:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id, lock_time] + @content_as = content_as + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::GetAndLock.new, raw_options: @raw_options) + builder.set_timeout + .set_transcoder + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_get_result(return_result: @return_result, initiated: @initiated, content_as: @content_as) do + @collection.get_and_lock(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetAndLockCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/get_and_touch_command.rb b/fit-performer/lib/fit/performer/commands/key_value/get_and_touch_command.rb new file mode 100644 index 00000000..a698e8c2 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/get_and_touch_command.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class GetAndTouchCommand + def initialize(collection:, doc_id:, expiry:, initiated:, return_result:, content_as:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id, expiry] + @content_as = content_as + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::GetAndTouch.new, raw_options: @raw_options) + builder.set_timeout + .set_transcoder + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_get_result(return_result: @return_result, initiated: @initiated, content_as: @content_as) do + @collection.get_and_touch(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetAndTouchCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/get_any_replica_command.rb b/fit-performer/lib/fit/performer/commands/key_value/get_any_replica_command.rb new file mode 100644 index 00000000..7a5e1139 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/get_any_replica_command.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/options' + +require 'logger' + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class GetAnyReplicaCommand + def initialize(collection:, doc_id:, initiated:, return_result:, content_as:, get_span_fn:, raw_options: nil) + @logger = Logger.new($stdout) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id] + @content_as = content_as + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::GetAnyReplica.new, raw_options: @raw_options) + builder.set_timeout + .set_transcoder + .set_read_preference + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_get_replica_result(return_result: @return_result, initiated: @initiated, content_as: @content_as) do + @collection.get_any_replica(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetAnyReplicaCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/get_command.rb b/fit-performer/lib/fit/performer/commands/key_value/get_command.rb new file mode 100644 index 00000000..739d115e --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/get_command.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/options' + +require 'logger' + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class GetCommand + def initialize(collection:, doc_id:, initiated:, return_result:, content_as:, get_span_fn:, raw_options: nil) + @logger = Logger.new($stdout) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id] + @content_as = content_as + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Get.new, raw_options: @raw_options) + builder.set_timeout + .set_with_expiry + .set_projection + .set_transcoder + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_get_result(return_result: @return_result, initiated: @initiated, content_as: @content_as) do + @collection.get(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/increment_command.rb b/fit-performer/lib/fit/performer/commands/key_value/increment_command.rb new file mode 100644 index 00000000..90c9ea8b --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/increment_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module KeyValue + class IncrementCommand + def initialize(binary_collection:, doc_id:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @binary_collection = binary_collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Increment.new, raw_options: @raw_options) + builder.set_timeout + .set_expiry + .set_delta + .set_initial + .set_durability + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_counter_result(return_result: @return_result, initiated: @initiated) do + @binary_collection.increment(*@cmd_args) + end + end + + def self.create_command(...) + cmd = IncrementCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/insert_command.rb b/fit-performer/lib/fit/performer/commands/key_value/insert_command.rb new file mode 100644 index 00000000..c0ba507b --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/insert_command.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/options' + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module KeyValue + class InsertCommand + def initialize(collection:, doc_id:, content:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id, content] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Insert.new, raw_options: @raw_options) + builder.set_timeout + .set_durability + .set_expiry + .set_transcoder + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_mutation_result(return_result: @return_result, initiated: @initiated) do + @collection.insert(*@cmd_args) + end + end + + def self.create_command(...) + cmd = InsertCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/lookup_in_all_replicas_command.rb b/fit-performer/lib/fit/performer/commands/key_value/lookup_in_all_replicas_command.rb new file mode 100644 index 00000000..a6927b27 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/lookup_in_all_replicas_command.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/options' + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class LookupInAllReplicasCommand + attr_reader :stream_config + + def initialize(collection:, doc_id:, specs:, raw_specs:, initiated:, return_result:, stream_config:, get_span_fn:, + raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @raw_specs = raw_specs + @cmd_args = [doc_id, specs] + @stream_config = stream_config + @get_span_fn = get_span_fn + end + + def stream_type + :STREAM_LOOKUP_IN_ALL_REPLICAS + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::LookupInAllReplicas.new, raw_options: @raw_options) + builder.set_timeout + .set_read_preference + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_lookup_in_all_replicas_result_stream(initiated: @initiated, raw_specs: @raw_specs, + stream_id: @stream_config.stream_id) do + @collection.lookup_in_all_replicas(*@cmd_args) + end + end + + def self.create_command(...) + cmd = LookupInAllReplicasCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/lookup_in_any_replica_command.rb b/fit-performer/lib/fit/performer/commands/key_value/lookup_in_any_replica_command.rb new file mode 100644 index 00000000..ed43ac28 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/lookup_in_any_replica_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/options' + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class LookupInAnyReplicaCommand + def initialize(collection:, doc_id:, specs:, raw_specs:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @raw_specs = raw_specs + @cmd_args = [doc_id, specs] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::LookupInAnyReplica.new, raw_options: @raw_options) + builder.set_timeout + .set_read_preference + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_lookup_in_replica_result(return_result: @return_result, initiated: @initiated, raw_specs: @raw_specs) do + @collection.lookup_in_any_replica(*@cmd_args) + end + end + + def self.create_command(...) + cmd = LookupInAnyReplicaCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/lookup_in_command.rb b/fit-performer/lib/fit/performer/commands/key_value/lookup_in_command.rb new file mode 100644 index 00000000..41c9d2f6 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/lookup_in_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/options' + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class LookupInCommand + def initialize(collection:, doc_id:, specs:, raw_specs:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @raw_specs = raw_specs + @cmd_args = [doc_id, specs] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::LookupIn.new, raw_options: @raw_options) + builder.set_timeout + .set_access_deleted + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_lookup_in_result(return_result: @return_result, initiated: @initiated, raw_specs: @raw_specs) do + @collection.lookup_in(*@cmd_args) + end + end + + def self.create_command(...) + cmd = LookupInCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/mutate_in_command.rb b/fit-performer/lib/fit/performer/commands/key_value/mutate_in_command.rb new file mode 100644 index 00000000..867dc3fa --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/mutate_in_command.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class MutateInCommand + def initialize(collection:, doc_id:, specs:, raw_specs:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @raw_specs = raw_specs + @cmd_args = [doc_id, specs] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::MutateIn.new, raw_options: @raw_options) + builder.set_timeout + .set_expiry + .set_cas + .set_durability + .set_store_semantics + .set_access_deleted + .set_preserve_expiry + .set_create_as_deleted + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_mutate_in_result(return_result: @return_result, initiated: @initiated, raw_specs: @raw_specs) do + @collection.mutate_in(*@cmd_args) + end + end + + def self.create_command(...) + cmd = MutateInCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/options_builder.rb b/fit-performer/lib/fit/performer/commands/key_value/options_builder.rb new file mode 100644 index 00000000..a771b49a --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/options_builder.rb @@ -0,0 +1,155 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/json_transcoder' +require 'couchbase/raw_binary_transcoder' +require 'couchbase/raw_json_transcoder' +require 'couchbase/raw_string_transcoder' + +require 'fit/performer/performer_error' +require 'fit/performer/commands/options_builder_base' + +module FIT + module Performer + module Commands + module KeyValue + class OptionsBuilder < OptionsBuilderBase + def set_with_expiry + @options.with_expiry = @raw_options.with_expiry if @raw_options.has_with_expiry? + self + end + + def set_projection + @options.project(*@raw_options.projection) unless @raw_options.projection.empty? + self + end + + def set_transcoder + return self unless @raw_options.has_transcoder? + + transcoder_type = @raw_options.transcoder.transcoder + @options.transcoder = + case transcoder_type + when :json + Couchbase::JsonTranscoder.new + when :raw_string + begin + Couchbase::RawStringTranscoder.new # Added in 3.4.3 + rescue NameError + nil + end + when :raw_binary + begin + Couchbase::RawBinaryTranscoder.new # Added in 3.4.3 + rescue NameError + nil + end + when :raw_json + begin + Couchbase::RawJsonTranscoder.new # Added in 3.4.3 + rescue NameError + nil + end + else + raise PerformerError, "Transcoder type `#{transcoder_type}` not supported" + end + self + end + + def set_durability + return self unless @raw_options.has_durability? + + durability_type = @raw_options.durability.durability + case durability_type + when :durabilityLevel + @options.durability_level = @raw_options.durability.durabilityLevel.downcase + when :observe + obs = @raw_options.durability.observe + @options.persist_to = obs.persistTo.to_s.delete_prefix('PERSIST_TO_').to_sym.downcase + @options.replicate_to = obs.replicateTo.to_s.delete_prefix('REPLICATE_TO_').to_sym.downcase + else + raise PerformerError, "Durability type `#{durability_type}` not supported" + end + + self + end + + def set_cas + @options.cas = @raw_options.cas if @raw_options.has_cas? + self + end + + def set_expiry + return self unless @raw_options.has_expiry? + + @options.expiry = KeyValue.get_expiry(@raw_options) + self + end + + def set_batch_byte_limit + @options.batch_byte_limit = @raw_options.batch_byte_limit if @raw_options.has_batch_byte_limit? + self + end + + def set_batch_item_limit + @options.batch_item_limit = @raw_options.batch_item_limit if @raw_options.has_batch_item_limit? + self + end + + def set_concurrency + @options.concurrency = @raw_options.concurrency if @raw_options.has_concurrency? + self + end + + def set_ids_only + @options.ids_only = @raw_options.ids_only if @raw_options.has_ids_only? + self + end + + def set_access_deleted + @options.access_deleted = @raw_options.access_deleted if @raw_options.has_access_deleted? + self + end + + def set_delta + @options.delta = @raw_options.delta if @raw_options.has_delta? + self + end + + def set_initial + @options.initial = @raw_options.initial if @raw_options.has_initial? + self + end + + def set_store_semantics + @options.store_semantics = @raw_options.store_semantics.downcase if @raw_options.has_store_semantics? + self + end + + def set_create_as_deleted + @options.create_as_deleted = @raw_options.create_as_deleted if @raw_options.has_create_as_deleted? + self + end + + def set_read_preference + @options.read_preference = @raw_options.read_preference.downcase if @raw_options.has_read_preference? + self + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/prepend_command.rb b/fit-performer/lib/fit/performer/commands/key_value/prepend_command.rb new file mode 100644 index 00000000..ed5453a6 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/prepend_command.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module KeyValue + class PrependCommand + def initialize(binary_collection:, doc_id:, content:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @binary_collection = binary_collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id, content] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Prepend.new, raw_options: @raw_options) + builder.set_timeout + .set_durability + .set_cas + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_mutation_result(return_result: @return_result, initiated: @initiated) do + @binary_collection.prepend(*@cmd_args) + end + end + + def self.create_command(...) + cmd = PrependCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/remove_command.rb b/fit-performer/lib/fit/performer/commands/key_value/remove_command.rb new file mode 100644 index 00000000..c97afe01 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/remove_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/options' + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module KeyValue + class RemoveCommand + def initialize(collection:, doc_id:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Remove.new, raw_options: @raw_options) + builder.set_timeout + .set_durability + .set_cas + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_mutation_result(return_result: @return_result, initiated: @initiated) do + @collection.remove(*@cmd_args) + end + end + + def self.create_command(...) + cmd = RemoveCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/replace_command.rb b/fit-performer/lib/fit/performer/commands/key_value/replace_command.rb new file mode 100644 index 00000000..6301f295 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/replace_command.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/options' + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module KeyValue + class ReplaceCommand + def initialize(collection:, doc_id:, content:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id, content] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Replace.new, raw_options: @raw_options) + builder.set_timeout + .set_durability + .set_expiry + .set_preserve_expiry + .set_cas + .set_transcoder + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_mutation_result(return_result: @return_result, initiated: @initiated) do + @collection.replace(*@cmd_args) + end + end + + def self.create_command(...) + cmd = ReplaceCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/results.rb b/fit-performer/lib/fit/performer/commands/key_value/results.rb new file mode 100644 index 00000000..aa897c26 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/results.rb @@ -0,0 +1,442 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'json' + +require 'fit/protocol/sdk.workload_pb' +require 'fit/protocol/sdk.kv.commands_pb' +require 'fit/protocol/sdk.kv.rangescan.top_level_pb' +require 'fit/protocol/shared.basic_pb' +require 'fit/protocol/shared.content_pb' + +require_relative '../shared_results' + +module FIT + module Performer + module Commands + module KeyValue + class Results < SharedResults + def self.as_get_result(return_result:, initiated:, content_as:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + if return_result + to_get_result(result: result, initiated: initiated, elapsed_nanos: nanos, content_as: content_as) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.as_get_replica_result(return_result:, initiated:, content_as:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + if return_result + to_get_replica_result(result: result, initiated: initiated, content_as: content_as, elapsed_nanos: nanos) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.as_get_all_replicas_result_stream(initiated:, stream_id:, content_as:) + begin + results = yield + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + + to_get_all_replicas_result_enumerator( + enumerator: results.each, initiated: initiated, stream_id: stream_id, content_as: content_as, + ) + end + + def self.as_mutation_result(return_result:, initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + if return_result + to_mutation_result(result: result, initiated: initiated, elapsed_nanos: nanos) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.as_exists_result(return_result:, initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + if return_result + to_exists_result(result: result, initiated: initiated, elapsed_nanos: nanos) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.as_counter_result(return_result:, initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + if return_result + to_counter_result(result: result, initiated: initiated, elapsed_nanos: nanos) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.as_scan_result_stream(initiated:, stream_id:, content_as:) + begin + results = yield + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + + to_scan_result_enumerator( + enumerator: results.each, initiated: initiated, stream_id: stream_id, content_as: content_as, + ) + end + + def self.as_lookup_in_result(return_result:, initiated:, raw_specs:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + + nanos = ((end_time - start_time) * (10**9)).round + if return_result + to_lookup_in_result(result: result, initiated: initiated, elapsed_nanos: nanos, raw_specs: raw_specs) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.as_lookup_in_replica_result(return_result:, initiated:, raw_specs:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + + nanos = ((end_time - start_time) * (10**9)).round + if return_result + to_lookup_in_result( + result: result, initiated: initiated, elapsed_nanos: nanos, raw_specs: raw_specs, from_replica: true, + ) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.as_lookup_in_all_replicas_result_stream(initiated:, stream_id:, raw_specs:) + begin + results = yield + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + + to_lookup_in_all_replicas_result_enumerator( + enumerator: results.each, initiated: initiated, stream_id: stream_id, raw_specs: raw_specs, + ) + end + + def self.as_mutate_in_result(return_result:, initiated:, raw_specs:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + + nanos = ((end_time - start_time) * (10**9)).round + if return_result + to_mutate_in_result(result: result, initiated: initiated, elapsed_nanos: nanos, raw_specs: raw_specs) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.to_get_replica_result(result:, initiated:, content_as:, elapsed_nanos: 0) + get_result = FIT::Protocol::SDK::KV::GetReplicaResult.new( + cas: result.cas, + content: get_content(content: result.content, content_as: content_as), + is_replica: result.is_replica, + ) + + get_result.expiry_time = result.expiry_time.to_i unless result.expiry_time.nil? + + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(get_replica_result: get_result), + initiated: initiated, elapsed_nanos: elapsed_nanos + ) + end + + def self.to_get_result(result:, elapsed_nanos:, initiated:, content_as:) + get_result = FIT::Protocol::SDK::KV::GetResult.new( + cas: result.cas, + content: get_content(content: result.content, content_as: content_as), + ) + get_result.expiry_time = result.expiry_time.to_i unless result.expiry_time.nil? + + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(get_result: get_result), + initiated: initiated, elapsed_nanos: elapsed_nanos + ) + end + + def self.to_get_all_replicas_result_enumerator(enumerator:, initiated:, stream_id:, content_as:) + Enumerator.new do |y| + loop do + result = enumerator.next + y << to_get_all_replicas_result( + result: result, initiated: initiated, stream_id: stream_id, content_as: content_as, + ) + rescue StopIteration + break + rescue StandardError => e + y << to_exception(error: e, initiated: initiated, unwrapped: true) + break + end + end + end + + def self.to_get_all_replicas_result(result:, initiated:, stream_id:, content_as:) + get_all_replicas_result = FIT::Protocol::SDK::KV::GetReplicaResult.new( + cas: result.cas, + content: get_content(content: result.content, content_as: content_as), + is_replica: result.is_replica, + stream_id: stream_id, + ) + + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(get_replica_result: get_all_replicas_result), + initiated: initiated, + ) + end + + def self.to_mutation_result(result:, elapsed_nanos:, initiated:) + mut_result = FIT::Protocol::SDK::KV::MutationResult.new(cas: result.cas) + mut_result.mutation_token = extract_mutation_token(result) unless result.mutation_token.nil? + + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(mutation_result: mut_result), + initiated: initiated, elapsed_nanos: elapsed_nanos + ) + end + + def self.to_exists_result(result:, elapsed_nanos:, initiated:) + exists_result = FIT::Protocol::SDK::KV::ExistsResult.new(cas: result.cas, exists: result.exists?) + + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(exists_result: exists_result), + initiated: initiated, elapsed_nanos: elapsed_nanos + ) + end + + def self.to_counter_result(result:, elapsed_nanos:, initiated:) + counter_result = FIT::Protocol::SDK::KV::CounterResult.new(cas: result.cas, content: result.content) + counter_result.mutation_token = extract_mutation_token(result) unless result.mutation_token.nil? + + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(counter_result: counter_result), + initiated: initiated, elapsed_nanos: elapsed_nanos + ) + end + + def self.to_scan_result_enumerator(enumerator:, initiated:, stream_id:, content_as:) + Enumerator.new do |y| + loop do + result = enumerator.next + y << to_scan_result( + result: result, initiated: initiated, stream_id: stream_id, content_as: content_as, + ) + rescue StopIteration + break + rescue StandardError => e + y << to_exception(error: e, initiated: initiated, unwrapped: true) + break + end + end + end + + def self.to_scan_result(result:, initiated:, stream_id:, content_as:) + scan_result = + if result.id_only + FIT::Protocol::SDK::KV::RangeScan::ScanResult.new( + id: result.id, id_only: result.id_only, stream_id: stream_id, + ) + else + FIT::Protocol::SDK::KV::RangeScan::ScanResult.new( + id: result.id, + id_only: result.id_only, + cas: result.cas, + content: get_content(content: result.content, content_as: content_as), + expiry_time: result.expiry, + stream_id: stream_id, + ) + end + + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(range_scan_result: scan_result), + initiated: initiated, + ) + end + + def self.to_lookup_in_result(result:, initiated:, raw_specs:, from_replica: false, elapsed_nanos: 0, unwrapped: false) + lookup_in_result = + if from_replica + FIT::Protocol::SDK::KV::LookupIn::LookupInReplicaResult.new + else + FIT::Protocol::SDK::KV::LookupIn::LookupInResult.new + end + + raw_specs.each_with_index do |raw_spec, idx| + content_or_error = FIT::Protocol::Shared::ContentOrError.new + begin + content_or_error.content = get_content(content: result.content(idx), content_as: raw_spec.content_as) + rescue PerformerError => e + raise e + rescue StandardError => e + content_or_error.exception = to_exception(error: e, unwrapped: true) + end + + exists_or_error = FIT::Protocol::SDK::KV::LookupIn::BooleanOrError.new + begin + exists_or_error.value = result.exists?(idx) + rescue StandardError => e + exists_or_error.exception = to_exception(error: e, unwrapped: true) + end + + lookup_in_result.results << FIT::Protocol::SDK::KV::LookupIn::LookupInSpecResult.new( + content_as_result: content_or_error, + exists_result: exists_or_error, + ) + end + + lookup_in_result.is_replica = result.replica? if from_replica + + return lookup_in_result if unwrapped + + if from_replica + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(lookup_in_any_replica_result: lookup_in_result), + initiated: initiated, elapsed_nanos: elapsed_nanos + ) + else + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(lookup_in_result: lookup_in_result), + initiated: initiated, elapsed_nanos: elapsed_nanos + ) + end + end + + def self.to_lookup_in_all_replicas_result_enumerator(enumerator:, initiated:, stream_id:, raw_specs:) + Enumerator.new do |y| + loop do + result = enumerator.next + y << to_lookup_in_all_replicas_result( + result: result, initiated: initiated, stream_id: stream_id, raw_specs: raw_specs, + ) + rescue StopIteration + break + rescue StandardError => e + y << to_exception(error: e, initiated: initiated, unwrapped: true) + break + end + end + end + + def self.to_lookup_in_all_replicas_result(result:, initiated:, stream_id:, raw_specs:) + lookup_in_replica_result = to_lookup_in_result( + result: result, initiated: initiated, raw_specs: raw_specs, from_replica: true, unwrapped: true, + ) + lookup_in_all_replicas_result = FIT::Protocol::SDK::KV::LookupIn::LookupInAllReplicasResult.new( + lookup_in_replica_result: lookup_in_replica_result, + stream_id: stream_id, + ) + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(lookup_in_all_replicas_result: lookup_in_all_replicas_result), + initiated: initiated, + ) + end + + def self.to_mutate_in_result(result:, initiated:, elapsed_nanos:, raw_specs:) + mutate_in_result = FIT::Protocol::SDK::KV::MutateIn::MutateInResult.new(cas: result.cas) + mutate_in_result.mutation_token = extract_mutation_token(result) unless result.mutation_token.nil? + + raw_specs.each_with_index do |raw_spec, idx| + content_or_error = FIT::Protocol::Shared::ContentOrError.new + begin + content_or_error.content = get_content(content: result.content(idx), content_as: raw_spec.content_as) + rescue PerformerError => e + raise e + rescue StandardError => e + content_or_error.exception = to_exception(error: e, unwrapped: true) + end + + mutate_in_result.results << FIT::Protocol::SDK::KV::MutateIn::MutateInSpecResult.new( + content_as_result: content_or_error, + ) + end + + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(mutate_in_result: mutate_in_result), + initiated: initiated, elapsed_nanos: elapsed_nanos + ) + end + + def self.extract_mutation_token(sdk_result) + FIT::Protocol::Shared::MutationToken.new( + partition_id: sdk_result.mutation_token.partition_id, + partition_uuid: sdk_result.mutation_token.partition_uuid, + sequence_number: sdk_result.mutation_token.sequence_number, + bucket_name: sdk_result.mutation_token.bucket_name, + ) + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/scan_command.rb b/fit-performer/lib/fit/performer/commands/key_value/scan_command.rb new file mode 100644 index 00000000..4ee8e43b --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/scan_command.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/options' + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module KeyValue + class ScanCommand + attr_reader :stream_config + + def initialize( + collection:, raw_scan_type:, initiated:, return_result:, + stream_config:, get_span_fn:, content_as: nil, raw_options: nil + ) + @collection = collection + @raw_scan_type = raw_scan_type + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @stream_config = stream_config + @content_as = content_as + @cmd_args = [] + @get_span_fn = get_span_fn + end + + def stream_type + :STREAM_KV_RANGE_SCAN + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Scan.new, raw_options: @raw_options) + builder.set_timeout + .set_batch_byte_limit + .set_batch_item_limit + .set_concurrency + .set_ids_only + .set_consistent_with + .set_transcoder + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def set_scan_type + scan_type = + case @raw_scan_type.type + when :range + proto_range_scan = @raw_scan_type.range + case proto_range_scan.range + when :from_to + from_choice = proto_range_scan.from_to.from + to_choice = proto_range_scan.from_to.to + from = + case from_choice.choice + when :term + proto_term = from_choice.term + if proto_term.has_exclusive? + Couchbase::ScanTerm.new(proto_term.as_string, exclusive: proto_term.exclusive) + else + Couchbase::ScanTerm.new(proto_term.as_string) + end + when :default + nil + else + raise PerformerError, "Scan term type #{from_choice.choice} not supported" + end + to = + case to_choice.choice + when :term + proto_term = to_choice.term + if proto_term.has_exclusive? + Couchbase::ScanTerm.new(proto_term.as_string, exclusive: proto_term.exclusive) + else + Couchbase::ScanTerm.new(proto_term.as_string) + end + when :default + nil + else + raise PerformerError, "Scan term type #{to_choice.choice} not supported" + end + Couchbase::RangeScan.new(from: from, to: to) + when :doc_id_prefix + Couchbase::PrefixScan.new(proto_range_scan.doc_id_prefix) + else + raise PerformerError, "Range scan range definition with `#{proto_range_scan.range}` not supported" + end + when :sampling + proto_sampling_scan = @raw_scan_type.sampling + if proto_sampling_scan.has_seed? + Couchbase::SamplingScan.new(proto_sampling_scan.limit, proto_sampling_scan.seed) + else + Couchbase::SamplingScan.new(proto_sampling_scan.limit) + end + else + raise PerformerError, "Scan type `#{@raw_scan_type.type}` not recognised" + end + + @cmd_args.append(scan_type) + end + + def execute_command + Results.as_scan_result_stream( + initiated: @initiated, stream_id: @stream_config.stream_id, content_as: @content_as, + ) do + @collection.scan(*@cmd_args) + end + end + + def self.create_command(...) + cmd = ScanCommand.new(...) + cmd.set_scan_type + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/touch_command.rb b/fit-performer/lib/fit/performer/commands/key_value/touch_command.rb new file mode 100644 index 00000000..1dd1ad2e --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/touch_command.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class TouchCommand + def initialize(collection:, doc_id:, expiry:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id, expiry] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Touch.new, raw_options: @raw_options) + builder.set_timeout + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_mutation_result(return_result: @return_result, initiated: @initiated) do + @collection.touch(*@cmd_args) + end + end + + def self.create_command(...) + cmd = TouchCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/unlock_command.rb b/fit-performer/lib/fit/performer/commands/key_value/unlock_command.rb new file mode 100644 index 00000000..22a9d9ab --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/unlock_command.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module KeyValue + class UnlockCommand + def initialize(collection:, doc_id:, cas:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id, cas] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Unlock.new, raw_options: @raw_options) + builder.set_timeout + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @collection.unlock(*@cmd_args) + end + end + + def self.create_command(...) + cmd = UnlockCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/key_value/upsert_command.rb b/fit-performer/lib/fit/performer/commands/key_value/upsert_command.rb new file mode 100644 index 00000000..fb72d509 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/key_value/upsert_command.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/options' + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module KeyValue + class UpsertCommand + def initialize(collection:, doc_id:, content:, initiated:, return_result:, get_span_fn:, raw_options: nil) + @collection = collection + @initiated = initiated + @return_result = return_result + @raw_options = raw_options + @cmd_args = [doc_id, content] + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Upsert.new, raw_options: @raw_options) + builder.set_timeout + .set_durability + .set_expiry + .set_preserve_expiry + .set_transcoder + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_mutation_result(return_result: @return_result, initiated: @initiated) do + @collection.upsert(*@cmd_args) + end + end + + def self.create_command(...) + cmd = UpsertCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/misc/update_authenticator_command.rb b/fit-performer/lib/fit/performer/commands/misc/update_authenticator_command.rb new file mode 100644 index 00000000..951fd791 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/misc/update_authenticator_command.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fit/performer/commands/shared_results' + +module FIT + module Performer + module Commands + module Misc + class UpdateAuthenticatorCommand + def initialize(authenticator:, cluster:, initiated:, return_result:, get_span_fn:) + @authenticator = authenticator + @cluster = cluster + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def execute_command + SharedResults.as_success(initiated: @initiated) do + @cluster.update_authenticator(@authenticator) + end + end + + def self.create_command(...) + new(...) + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/options_builder_base.rb b/fit-performer/lib/fit/performer/commands/options_builder_base.rb new file mode 100644 index 00000000..7a2d5167 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/options_builder_base.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module FIT + module Performer + module Commands + class OptionsBuilderBase + attr_reader :options + + def initialize(options:, raw_options:) + @options = options + @raw_options = raw_options + end + + def set_timeout + if @raw_options.respond_to?(:has_timeout_millis?) + @options.timeout = @raw_options.timeout_millis if @raw_options.has_timeout_millis? + elsif @raw_options.has_timeout_msecs? + @options.timeout = @raw_options.timeout_msecs + end + self + end + + def set_parent_span(get_span_fn) + return self unless @raw_options.has_parent_span_id? + + @options.parent_span = get_span_fn.call(@raw_options.parent_span_id) + self + end + + def set_consistent_with + return self unless @raw_options.has_consistent_with? + + mutation_state = Couchbase::MutationState.new + @raw_options.consistent_with.tokens.each do |proto_token| + mutation_state.add( + Couchbase::MutationToken.new do |t| + t.partition_id = proto_token.partition_id + t.partition_uuid = proto_token.partition_uuid + t.sequence_number = proto_token.sequence_number + t.bucket_name = proto_token.bucket_name + end, + ) + end + @options.consistent_with(mutation_state) + self + end + + def set_preserve_expiry + @options.preserve_expiry = @raw_options.preserve_expiry if @raw_options.has_preserve_expiry? + self + end + + class CustomJsonTranscoder + def initialize + @json_transcoder = Couchbase::JsonTranscoder.new + end + + def encode(document) + document["Serialized"] = true + @json_transcoder.encode(document) + end + + def decode(blob, flags) + document = @json_transcoder.decode(blob, flags) + document["Serialized"] = false + document + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query.rb b/fit-performer/lib/fit/performer/commands/query.rb new file mode 100644 index 00000000..6eb1a0b2 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'query/query_command' + +module FIT + module Performer + module Commands + module Query + def self.build_cluster_level_command(raw_cmd, cluster, cmd_kwargs) + cmd_kwargs[:cluster] = cluster + build_command(raw_cmd, cmd_kwargs) + end + + def self.build_scope_level_command(raw_cmd, scope, cmd_kwargs) + cmd_kwargs[:scope] = scope + build_command(raw_cmd, cmd_kwargs) + end + + def self.build_command(raw_cmd, cmd_kwargs) + cmd_kwargs[:statement] = raw_cmd.statement + cmd_kwargs[:raw_options] = raw_cmd.options if raw_cmd.has_options? + cmd_kwargs[:content_as] = raw_cmd.content_as + QueryCommand.create_command(**cmd_kwargs) + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query/options_builder.rb b/fit-performer/lib/fit/performer/commands/query/options_builder.rb new file mode 100644 index 00000000..8cd6d6e7 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query/options_builder.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fit/performer/commands/options_builder_base' + +module FIT + module Performer + module Commands + module Query + class OptionsBuilder < OptionsBuilderBase + def set_scan_consistency + @options.scan_consistency = @raw_options.scan_consistency.downcase if @raw_options.has_scan_consistency? + self + end + + def set_raw + @raw_options.raw.each do |k, v| + @options.raw_parameters[k] = v + end + self + end + + def set_adhoc + @options.adhoc = @raw_options.adhoc if @raw_options.has_adhoc? + self + end + + def set_profile + @options.profile = @raw_options.profile.to_sym if @raw_options.has_profile? + self + end + + def set_readonly + @options.readonly = @raw_options.readonly if @raw_options.has_readonly? + self + end + + def set_positional_parameters + @options.positional_parameters(@raw_options.parameters_positional.to_a) unless @raw_options.parameters_positional.empty? + self + end + + def set_named_parameters + # rubocop:disable Style/ZeroLengthPredicate + # Protobuf maps have no #empty? + @options.named_parameters(@raw_options.parameters_named.to_h) unless @raw_options.parameters_named.size.zero? + # rubocop:enable Style/ZeroLengthPredicate + self + end + + def set_flex_index + @options.flex_index = @raw_options.flex_index if @raw_options.has_flex_index? + self + end + + def set_pipeline_cap + @options.pipeline_cap = @raw_options.pipeline_cap if @raw_options.has_pipeline_cap? + self + end + + def set_pipeline_batch + @options.pipeline_batch = @raw_options.pipeline_batch if @raw_options.has_pipeline_batch? + self + end + + def set_scan_cap + @options.scan_cap = @raw_options.scan_cap if @raw_options.has_scan_cap? + self + end + + def set_scan_wait + @options.scan_wait = @raw_options.scan_wait_millis if @raw_options.has_scan_wait_millis? + self + end + + def set_timeout + @options.timeout = @raw_options.timeout_millis if @raw_options.has_timeout_millis? + self + end + + def set_max_parallelism + @options.max_parallelism = @raw_options.max_parallelism if @raw_options.has_max_parallelism? + self + end + + def set_metrics + @options.metrics = @raw_options.metrics if @raw_options.has_metrics? + self + end + + def set_use_replica + @options.use_replica = @raw_options.use_replica if @raw_options.has_use_replica? + self + end + + def set_client_context_id + @options.client_context_id = @raw_options.client_context_id if @raw_options.has_client_context_id? + self + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query/query_command.rb b/fit-performer/lib/fit/performer/commands/query/query_command.rb new file mode 100644 index 00000000..206c44f8 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query/query_command.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module Query + class QueryCommand + def initialize( + statement:, initiated:, return_result:, content_as:, get_span_fn:, + cluster: nil, scope: nil, raw_options: nil + ) + @cmd_args = [statement] + @cluster = cluster + @scope = scope + @raw_options = raw_options + @initiated = initiated + @return_result = return_result + @content_as = content_as + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Query.new, raw_options: @raw_options) + builder.set_timeout + .set_adhoc + .set_flex_index + .set_max_parallelism + .set_metrics + .set_named_parameters + .set_pipeline_batch + .set_pipeline_cap + .set_positional_parameters + .set_profile + .set_raw + .set_readonly + .set_scan_cap + .set_scan_consistency + .set_scan_wait + .set_use_replica + .set_client_context_id + .set_preserve_expiry + .set_consistent_with + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_query_result(return_result: @return_result, initiated: @initiated, content_as: @content_as) do + if !@scope.nil? + @scope.query(*@cmd_args) + elsif !@cluster.nil? + @cluster.query(*@cmd_args) + else + raise PerformerError, "Either cluster or scope should be provided for query command" + end + end + end + + def self.create_command(...) + cmd = QueryCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query/results.rb b/fit-performer/lib/fit/performer/commands/query/results.rb new file mode 100644 index 00000000..1098e70f --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query/results.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module FIT + module Performer + module Commands + module Query + class Results < SharedResults + def self.as_query_result(return_result:, initiated:, content_as:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + if return_result + to_query_result(result: result, initiated: initiated, elapsed_nanos: nanos, content_as: content_as) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.to_query_result(result:, initiated:, elapsed_nanos:, content_as:) + query_meta_data_pb = FIT::Protocol::SDK::Query::QueryMetaData.new( + request_id: result.meta_data.request_id, + client_context_id: result.meta_data.client_context_id, + status: result.meta_data.status.upcase, + ) + query_meta_data_pb.signature = result.meta_data.signature.to_json.b unless result.meta_data.signature.nil? + query_meta_data_pb.profile = result.meta_data.profile.to_json.b unless result.meta_data.profile.nil? + + unless result.meta_data.metrics.elapsed_time.nil? + query_meta_data_pb.metrics = FIT::Protocol::SDK::Query::QueryMetrics.new( + elapsed_time: result.meta_data.metrics.elapsed_time, + execution_time: result.meta_data.metrics.execution_time, + sort_count: result.meta_data.metrics.sort_count, + result_count: result.meta_data.metrics.result_count, + result_size: result.meta_data.metrics.result_size, + mutation_count: result.meta_data.metrics.mutation_count, + error_count: result.meta_data.metrics.error_count, + warning_count: result.meta_data.metrics.warning_count, + ) + end + + result.meta_data&.warnings&.each do |warning| + query_meta_data_pb.warnings << FIT::Protocol::SDK::Query::QueryWarning( + code: warning.code, message: warning.message, + ) + end + + query_result_pb = FIT::Protocol::SDK::Query::QueryResult.new(meta_data: query_meta_data_pb) + result.rows.each do |r| + query_result_pb.content << get_content(content: r, content_as: content_as) + end + + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(query_result: query_result_pb), + initiated: initiated, + elapsed_nanos: elapsed_nanos, + ) + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query_index_manager.rb b/fit-performer/lib/fit/performer/commands/query_index_manager.rb new file mode 100644 index 00000000..b9dbe67d --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query_index_manager.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fit/performer/performer_error' +require_relative 'query_index_manager/build_deferred_indexes_command' +require_relative 'query_index_manager/create_index_command' +require_relative 'query_index_manager/create_primary_index_command' +require_relative 'query_index_manager/drop_index_command' +require_relative 'query_index_manager/drop_primary_index_command' +require_relative 'query_index_manager/get_all_indexes_command' +require_relative 'query_index_manager/watch_indexes_command' + +module FIT + module Performer + module Commands + module QueryIndexManager + def self.build_cluster_level_command(raw_cmd, cluster, cmd_kwargs) + cmd_kwargs[:manager] = cluster.query_indexes + cmd_kwargs[:bucket_name] = raw_cmd.bucket_name + build_shared_command(raw_cmd.shared, cmd_kwargs) + end + + def self.build_collection_level_command(raw_cmd, collection, cmd_kwargs) + cmd_kwargs[:manager] = collection.query_indexes + build_shared_command(raw_cmd.shared, cmd_kwargs) + end + + def self.build_shared_command(raw_cmd, cmd_kwargs) + cmd_type = raw_cmd.command + case cmd_type + when :create_primary_index + cmd = raw_cmd.create_primary_index + cmd_kwargs[:raw_options] = cmd.options if cmd.has_options? + CreatePrimaryIndexCommand.create_command(**cmd_kwargs) + when :create_index + cmd = raw_cmd.create_index + cmd_kwargs[:raw_options] = cmd.options if cmd.has_options? + cmd_kwargs[:index_name] = cmd.index_name + cmd_kwargs[:fields] = cmd.fields.to_a + CreateIndexCommand.create_command(**cmd_kwargs) + when :get_all_indexes + cmd = raw_cmd.get_all_indexes + cmd_kwargs[:raw_options] = cmd.options if cmd.has_options? + GetAllIndexesCommand.create_command(**cmd_kwargs) + when :drop_primary_index + cmd = raw_cmd.drop_primary_index + cmd_kwargs[:raw_options] = cmd.options if cmd.has_options? + DropPrimaryIndexCommand.create_command(**cmd_kwargs) + when :drop_index + cmd = raw_cmd.drop_index + cmd_kwargs[:raw_options] = cmd.options if cmd.has_options? + cmd_kwargs[:index_name] = cmd.index_name + DropIndexCommand.create_command(**cmd_kwargs) + when :watch_indexes + cmd = raw_cmd.watch_indexes + cmd_kwargs[:raw_options] = cmd.options if cmd.has_options? + cmd_kwargs[:index_names] = cmd.index_names.to_a + cmd_kwargs[:timeout] = cmd.timeout_msecs + WatchIndexesCommand.create_command(**cmd_kwargs) + when :build_deferred_indexes + cmd = raw_cmd.build_deferred_indexes + cmd_kwargs[:raw_options] = cmd.options if cmd.has_options? + BuildDeferredIndexesCommand.create_command(**cmd_kwargs) + else + raise PerformerError, "Query index manager command `#{cmd_type}` not implemented" + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query_index_manager/build_deferred_indexes_command.rb b/fit-performer/lib/fit/performer/commands/query_index_manager/build_deferred_indexes_command.rb new file mode 100644 index 00000000..e0254d0a --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query_index_manager/build_deferred_indexes_command.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/management/query_index_manager' + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module QueryIndexManager + class BuildDeferredIndexesCommand + def initialize(manager:, initiated:, get_span_fn:, return_result: true, raw_options: nil, bucket_name: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = bucket_name.nil? ? [] : [bucket_name] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Query::BuildDeferredIndexes.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_scope_name + .set_collection_name + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.build_deferred_indexes(*@cmd_args) + end + end + + def self.create_command(...) + cmd = BuildDeferredIndexesCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query_index_manager/create_index_command.rb b/fit-performer/lib/fit/performer/commands/query_index_manager/create_index_command.rb new file mode 100644 index 00000000..cd47e855 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query_index_manager/create_index_command.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/management/query_index_manager' + +require_relative 'options_builder' +require_relative 'results' + +module FIT + module Performer + module Commands + module QueryIndexManager + class CreateIndexCommand + def initialize(manager:, index_name:, fields:, initiated:, get_span_fn:, return_result: true, raw_options: nil, bucket_name: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = bucket_name.nil? ? [] : [bucket_name] + @cmd_args.append(index_name, fields) + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Query::CreateIndex.new, + raw_options: @raw_options, + ) + builder.set_ignore_if_exists + .set_num_replicas + .set_deferred + .set_scope_name + .set_collection_name + .set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.create_index(*@cmd_args) + end + end + + def self.create_command(...) + cmd = CreateIndexCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query_index_manager/create_primary_index_command.rb b/fit-performer/lib/fit/performer/commands/query_index_manager/create_primary_index_command.rb new file mode 100644 index 00000000..6d8d403e --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query_index_manager/create_primary_index_command.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/management/query_index_manager' + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module QueryIndexManager + class CreatePrimaryIndexCommand + def initialize(manager:, initiated:, get_span_fn:, return_result: true, raw_options: nil, bucket_name: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = bucket_name.nil? ? [] : [bucket_name] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Query::CreatePrimaryIndex.new, + raw_options: @raw_options, + ) + builder.set_ignore_if_exists + .set_num_replicas + .set_deferred + .set_index_name + .set_scope_name + .set_collection_name + .set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.create_primary_index(*@cmd_args) + end + end + + def self.create_command(...) + cmd = CreatePrimaryIndexCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query_index_manager/drop_index_command.rb b/fit-performer/lib/fit/performer/commands/query_index_manager/drop_index_command.rb new file mode 100644 index 00000000..381c33e2 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query_index_manager/drop_index_command.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/management/query_index_manager' + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module QueryIndexManager + class DropIndexCommand + def initialize(manager:, index_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil, bucket_name: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = bucket_name.nil? ? [] : [bucket_name] + @cmd_args.append(index_name) + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Query::DropIndex.new, + raw_options: @raw_options, + ) + builder.set_ignore_if_does_not_exist + .set_scope_name + .set_collection_name + .set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.drop_index(*@cmd_args) + end + end + + def self.create_command(...) + cmd = DropIndexCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query_index_manager/drop_primary_index_command.rb b/fit-performer/lib/fit/performer/commands/query_index_manager/drop_primary_index_command.rb new file mode 100644 index 00000000..42df7d6b --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query_index_manager/drop_primary_index_command.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/management/query_index_manager' + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module QueryIndexManager + class DropPrimaryIndexCommand + def initialize(manager:, initiated:, get_span_fn:, return_result: true, raw_options: nil, bucket_name: nil) + @manager = manager + @cmd_args = bucket_name.nil? ? [] : [bucket_name] + @raw_options = raw_options + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Query::DropPrimaryIndex.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_collection_name + .set_scope_name + .set_ignore_if_does_not_exist + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.drop_primary_index(*@cmd_args) + end + end + + def self.create_command(...) + cmd = DropPrimaryIndexCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query_index_manager/get_all_indexes_command.rb b/fit-performer/lib/fit/performer/commands/query_index_manager/get_all_indexes_command.rb new file mode 100644 index 00000000..c841ee7d --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query_index_manager/get_all_indexes_command.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/management/query_index_manager' + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module QueryIndexManager + class GetAllIndexesCommand + def initialize(manager:, initiated:, get_span_fn:, return_result: true, raw_options: nil, bucket_name: nil) + @manager = manager + @cmd_args = bucket_name.nil? ? [] : [bucket_name] + @raw_options = raw_options + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Query::GetAllIndexes.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_scope_name + .set_collection_name + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_query_indexes(initiated: @initiated, return_result: @return_result) do + @manager.get_all_indexes(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetAllIndexesCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query_index_manager/options_builder.rb b/fit-performer/lib/fit/performer/commands/query_index_manager/options_builder.rb new file mode 100644 index 00000000..9fd0b1bc --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query_index_manager/options_builder.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fit/performer/commands/options_builder_base' + +module FIT + module Performer + module Commands + module QueryIndexManager + class OptionsBuilder < OptionsBuilderBase + def set_scope_name + @options.scope_name = @raw_options.scope_name if @raw_options.has_scope_name? + self + end + + def set_collection_name + @options.collection_name = @raw_options.collection_name if @raw_options.has_collection_name? + self + end + + def set_ignore_if_exists + @options.ignore_if_exists = @raw_options.ignore_if_exists if @raw_options.has_ignore_if_exists? + self + end + + def set_ignore_if_does_not_exist + @options.ignore_if_does_not_exist = @raw_options.ignore_if_not_exists if @raw_options.has_ignore_if_not_exists? + self + end + + def set_num_replicas + @options.num_replicas = @raw_options.num_replicas if @raw_options.has_num_replicas? + self + end + + def set_deferred + @options.deferred = @raw_options.deferred if @raw_options.has_deferred? + self + end + + def set_index_name + @options.index_name = @raw_options.index_name if @raw_options.has_index_name? + self + end + + def set_watch_primary + @options.watch_primary = @raw_options.watch_primary if @raw_options.has_watch_primary? + self + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query_index_manager/results.rb b/fit-performer/lib/fit/performer/commands/query_index_manager/results.rb new file mode 100644 index 00000000..a33739e1 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query_index_manager/results.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fit/protocol/sdk.query.index_manager_pb' +require 'fit/protocol/sdk.workload_pb' +require 'fit/protocol/shared.basic_pb' + +require_relative '../shared_results' + +module FIT + module Performer + module Commands + module QueryIndexManager + class Results < SharedResults + def self.as_query_indexes(return_result:, initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + if return_result + to_query_indexes(result: result, initiated: initiated, elapsed_nanos: nanos) + else + to_success(elapsed_nanos: nanos, initiated: initiated) + end + end + + def self.to_query_indexes(result:, initiated:, elapsed_nanos:) + indexes_pb = FIT::Protocol::SDK::Query::IndexManager::QueryIndexes.new( + indexes: result.map do |index| + # TODO: Keyspace is missing from SDK result + kwargs = { + name: index.name, + is_primary: index.is_primary, + type: index.type.upcase, + state: index.state.to_s, + index_key: index.index_key, + bucket_name: index.bucket, + } + kwargs[:condition] = index.condition unless index.condition.nil? + kwargs[:partition] = index.partition unless index.partition.nil? + kwargs[:scope_name] = index.scope unless index.scope.nil? + kwargs[:collection_name] = index.collection unless index.collection.nil? + + FIT::Protocol::SDK::Query::IndexManager::QueryIndex.new(kwargs) + end, + ) + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(query_indexes: indexes_pb), + initiated: initiated, + elapsed_nanos: elapsed_nanos, + ) + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/query_index_manager/watch_indexes_command.rb b/fit-performer/lib/fit/performer/commands/query_index_manager/watch_indexes_command.rb new file mode 100644 index 00000000..83b6bdca --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/query_index_manager/watch_indexes_command.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/management/query_index_manager' + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module QueryIndexManager + class WatchIndexesCommand + def initialize(manager:, index_names:, timeout:, initiated:, get_span_fn:, return_result: true, raw_options: nil, + bucket_name: nil) + @manager = manager + @cmd_args = bucket_name.nil? ? [] : [bucket_name] + @cmd_args.append(index_names, timeout) + @raw_options = raw_options + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::Options::Query::WatchIndexes.new, + raw_options: @raw_options, + ) + builder.set_watch_primary + .set_scope_name + .set_collection_name + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.watch_indexes(*@cmd_args) + end + end + + def self.create_command(...) + cmd = WatchIndexesCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search.rb b/fit-performer/lib/fit/performer/commands/search.rb new file mode 100644 index 00000000..361551e0 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search.rb @@ -0,0 +1,262 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'search/search_query_command' +require_relative 'search/search_command' +require_relative 'search/options_builder' + +module FIT + module Performer + module Commands + module Search # rubocop:disable Metrics/ModuleLength + MATCH_OPERATOR_MAP = { + :SEARCH_MATCH_OPERATOR_OR => :or, + :SEARCH_MATCH_OPERATOR_AND => :and, + }.freeze + + def self.build_cluster_level_command(raw_cmd, cluster, cmd_kwargs) + cmd_kwargs[:cluster] = cluster + build_command(raw_cmd, cmd_kwargs) + end + + def self.build_scope_level_command(raw_cmd, scope, cmd_kwargs) + cmd_kwargs[:scope] = scope + build_command(raw_cmd, cmd_kwargs) + end + + def self.build_command(raw_cmd, cmd_kwargs) + case raw_cmd + when Protocol::SDK::Search::Search + cmd_kwargs.update({ + index_name: raw_cmd.indexName, + search_query: get_search_query(raw_cmd.query), + stream_config: raw_cmd.stream_config, + }) + cmd_kwargs[:raw_options] = raw_cmd.options if raw_cmd.has_options? + cmd_kwargs[:fields_as] = raw_cmd.fields_as if raw_cmd.has_fields_as? + SearchQueryCommand.create_command(**cmd_kwargs) + when Protocol::SDK::Search::SearchWrapper + cmd_kwargs.update({ + index_name: raw_cmd.search.indexName, + search_request: get_search_request(raw_cmd.search.request), + stream_config: raw_cmd.stream_config, + }) + cmd_kwargs[:raw_options] = raw_cmd.search.options if raw_cmd.search.has_options? + cmd_kwargs[:fields_as] = raw_cmd.fields_as if raw_cmd.has_fields_as? + SearchCommand.create_command(**cmd_kwargs) + else + raise PerformerError, "#{raw_cmd.class} is not a valid search command type" + end + end + + def self.get_search_query(raw_search_query) + query = raw_search_query.public_send(raw_search_query.query) + case raw_search_query.query + when :match + Couchbase::Cluster::SearchQuery.match(query.match) do |q| + q.field = query.field if query.has_field? + q.analyzer = query.analyzer if query.has_analyzer? + q.prefix_length = query.prefix_length if query.has_prefix_length? + q.fuzziness = query.fuzziness if query.has_fuzziness? + q.boost = query.boost if query.has_boost? + q.operator = MATCH_OPERATOR_MAP[query.operator] if query.has_operator? + end + when :match_phrase + Couchbase::Cluster::SearchQuery.match_phrase(query.match_phrase) do |q| + q.field = query.field if query.has_field? + q.analyzer = query.analyzer if query.has_analyzer? + q.boost = query.boost if query.has_boost? + end + when :regexp + Couchbase::Cluster::SearchQuery.regexp(query.regexp) do |q| + q.field = query.field if query.has_field? + q.boost = query.boost if query.has_boost? + end + when :query_string + Couchbase::Cluster::SearchQuery.query_string(query.query) do |q| + q.boost = query.boost if query.has_boost? + end + when :wildcard + Couchbase::Cluster::SearchQuery.wildcard(query.wildcard) do |q| + q.field = query.field if query.has_field? + q.boost = query.boost if query.has_boost? + end + when :doc_id + Couchbase::Cluster::SearchQuery.doc_id(*query.ids) do |q| + q.boost = query.boost if query.has_boost? + end + when :search_boolean_field + Couchbase::Cluster::SearchQuery.boolean_field(query.bool) do |q| + q.field = query.field if query.has_field? + q.boost = query.boost if query.has_boost? + end + when :date_range + Couchbase::Cluster::SearchQuery.date_range do |q| + if query.has_start? + if query.has_inclusive_start? + q.start_time(query.start, query.inclusive_start) + else + q.start_time(query.start) + end + end + if query.has_end? + if query.has_inclusive_end? + q.end_time(query.end, query.inclusive_end) + else + q.end_time(query.end) + end + end + q.date_time_parser = query.datetime_parser if query.has_datetime_parser? + q.field = query.field if query.has_field? + q.boost = query.boost if query.has_boost? + end + when :numeric_range + Couchbase::Cluster::SearchQuery.numeric_range do |q| + if query.has_min? + if query.has_inclusive_min? + q.min(query.min, query.inclusive_min) + else + q.min(query.min) + end + end + if query.has_max? + if query.has_inclusive_max? + q.max(query.max, query.inclusive_max) + else + q.max(query.max) + end + end + q.field = query.field if query.has_field? + q.boost = query.boost if query.has_boost? + end + when :term_range + Couchbase::Cluster::SearchQuery.term_range do |q| + if query.has_min? + if query.has_inclusive_min? + q.min(query.min, query.inclusive_min) + else + q.min(query.min) + end + end + if query.has_max? + if query.has_inclusive_max? + q.max(query.max, query.inclusive_max) + else + q.max(query.max) + end + end + q.field = query.field if query.has_field? + q.boost = query.boost if query.has_boost? + end + when :geo_distance + Couchbase::Cluster::SearchQuery.geo_distance(query.location.lon, query.location.lat, query.distance) do |q| + q.field = query.field if query.has_field? + q.boost = query.boost if query.has_boost? + end + when :geo_bounding_box + Couchbase::Cluster::SearchQuery.geo_bounding_box(query.top_left.lon, query.top_left.lat, query.bottom_right.lon, + query.bottom_right.lat) do |q| + q.field = query.field if query.has_field? + q.boost = query.boost if query.has_boost? + end + when :conjunction + queries = query.conjuncts.map { |c| get_search_query(c) } + Couchbase::Cluster::SearchQuery.conjuncts(*queries) do |q| + q.boost = query.boost if query.has_boost? + end + when :disjunction + queries = query.disjuncts.map { |d| get_search_query(d) } + Couchbase::Cluster::SearchQuery.disjuncts(*queries) do |q| + q.min = query.min if query.has_min? + q.boost = query.boost if query.has_boost? + end + when :boolean + Couchbase::Cluster::SearchQuery.booleans do |q| + q.must(*query.must.map { |proto_query| get_search_query(proto_query) }) unless query.must.empty? + q.should(*query.should.map { |proto_query| get_search_query(proto_query) }) unless query.should.empty? + q.must_not(*query.must_not.map { |proto_query| get_search_query(proto_query) }) unless query.must_not.empty? + q.should_min(query.should_min) if query.has_should_min? + q.boost = query.boost if query.has_boost? + end + when :term + Couchbase::Cluster::SearchQuery.term(query.term) do |q| + q.field = query.field if query.has_field? + q.fuzziness = query.fuzziness if query.has_fuzziness? + q.prefix_length = query.prefix_length if query.has_prefix_length? + q.boost = query.boost if query.has_boost? + end + when :prefix + Couchbase::Cluster::SearchQuery.prefix(query.prefix) do |q| + q.field = query.field if query.has_field? + q.boost = query.boost if query.has_boost? + end + when :phrase + Couchbase::Cluster::SearchQuery.phrase(*query.terms) do |q| + q.field = query.field if query.has_field? + q.boost = query.boost if query.has_boost? + end + when :match_all + Couchbase::Cluster::SearchQuery.match_all + when :match_none + Couchbase::Cluster::SearchQuery.match_none + else + raise PerformerError, "Query type #{raw_search_query.query} not supported" + end + end + + def self.get_search_request(raw_search_request) + if raw_search_request.has_vector_search? + vector = get_vector_search(raw_search_request.vector_search) + if raw_search_request.has_search_query? + Couchbase::SearchRequest.new(vector).search_query(get_search_query(raw_search_request.search_query)) + else + Couchbase::SearchRequest.new(vector) + end + elsif raw_search_request.has_search_query? + Couchbase::SearchRequest.new(get_search_query(raw_search_request.search_query)) + else + raise PerformerError, "The SDK does not support creating an empty SearchRequest" + end + end + + def self.get_vector_search(raw_vector_search) + queries = raw_vector_search.vector_query.map do |raw_query| + query_args = [ + raw_query.vector_field_name, + raw_query.has_base64_vector_query? ? raw_query.base64_vector_query : raw_query.vector_query.to_a, + ] + + Couchbase::VectorQuery.new(*query_args) do |q| + if raw_query.has_options? + q.num_candidates = raw_query.options.num_candidates if raw_query.options.has_num_candidates? + q.boost = raw_query.options.boost if raw_query.options.has_boost? + q.prefilter = get_search_query(raw_query.options.prefilter) if raw_query.options.has_prefilter? + end + end + end + + if raw_vector_search.has_options? + builder = OptionsBuilder.new(options: Couchbase::Options::VectorSearch.new, raw_options: raw_vector_search.options) + builder.set_vector_query_combination + Couchbase::VectorSearch.new(queries, builder.options) + else + Couchbase::VectorSearch.new(queries) + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search/options_builder.rb b/fit-performer/lib/fit/performer/commands/search/options_builder.rb new file mode 100644 index 00000000..fd066048 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search/options_builder.rb @@ -0,0 +1,185 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fit/performer/commands/options_builder_base' + +require 'google/protobuf/well_known_types' + +module FIT + module Performer + module Commands + module Search + class OptionsBuilder < OptionsBuilderBase + HIGHLIGHT_STYLE_MAP = { + :HIGHLIGHT_STYLE_HTML => :html, + :HIGHLIGHT_STYLE_ANSI => :ansi, + }.freeze + + GEO_DISTANCE_UNIT_MAP = { + :SEARCH_GEO_DISTANCE_UNITS_METERS => :meters, + :SEARCH_GEO_DISTANCE_UNITS_MILES => :miles, + :SEARCH_GEO_DISTANCE_UNITS_CENTIMETERS => :centimeters, + :SEARCH_GEO_DISTANCE_UNITS_MILLIMETERS => :millimeters, + :SEARCH_GEO_DISTANCE_UNITS_NAUTICAL_MILES => :nauticalmiles, + :SEARCH_GEO_DISTANCE_UNITS_KILOMETERS => :kilometers, + :SEARCH_GEO_DISTANCE_UNITS_FEET => :feet, + :SEARCH_GEO_DISTANCE_UNITS_YARDS => :yards, + :SEARCH_GEO_DISTANCE_UNITS_INCHES => :inch, + }.freeze + + SCAN_CONSISTENCY_MAP = { + :SEARCH_SCAN_CONSISTENCY_NOT_BOUNDED => :not_bounded, + }.freeze + + def set_limit + @options.limit = @raw_options.limit if @raw_options.has_limit? + self + end + + def set_skip + @options.skip = @raw_options.skip if @raw_options.has_skip? + self + end + + def set_explain + @options.explain = @raw_options.explain if @raw_options.has_explain? + self + end + + def set_highlight + if @raw_options.has_highlight? + @options.highlight_style = HIGHLIGHT_STYLE_MAP[@raw_options.highlight.style] if @raw_options.highlight.has_style? + @options.highlight_fields = @raw_options.highlight.fields.to_a unless @raw_options.highlight.fields.empty? + end + self + end + + def set_fields + @options.fields = @raw_options.fields.to_a unless @raw_options.fields.empty? + self + end + + def set_scan_consistency + @options.scan_consistency = SCAN_CONSISTENCY_MAP[@raw_options.scan_consistency] if @raw_options.has_scan_consistency? + self + end + + def set_sort + return self if @raw_options.sort.empty? + + sort = @raw_options.sort.map do |proto_sort| + case proto_sort.sort + when :score + Couchbase::Cluster::SearchSort.score do |s| + s.desc = proto_sort.score.desc if proto_sort.score.has_desc? + end + when :id + Couchbase::Cluster::SearchSort.id do |s| + s.desc = proto_sort.id.desc if proto_sort.id.has_desc? + end + when :field + Couchbase::Cluster::SearchSort.field(proto_sort.field.field) do |s| + s.desc = proto_sort.field.desc if proto_sort.field.has_desc? + s.type = proto_sort.field.type.to_sym if proto_sort.field.has_type? + s.mode = proto_sort.field.mode.to_sym if proto_sort.field.has_mode? + s.missing = proto_sort.field.missing.to_sym if proto_sort.field.has_missing? + end + when :geo_distance + Couchbase::Cluster::SearchSort.geo_distance( + proto_sort.geo_distance.field, + proto_sort.geo_distance.location.lon, + proto_sort.geo_distance.location.lat, + ) do |s| + s.unit = GEO_DISTANCE_UNIT_MAP[proto_sort.geo_distance.unit] if proto_sort.geo_distance.has_unit? + s.desc = proto_sort.geo_distance.sort if proto_sort.geo_distance.has_sort? + end + when :raw + proto_sort.raw + else + raise PerformerError, "Search sort type #{proto_sort.sort} not supported" + end + end + + @options.sort = sort + self + end + + def set_facets + return self unless @raw_options.facets.size.positive? + + facets = {} + @raw_options.facets.each do |name, proto_facet| + facets[name] = + case proto_facet.facet + when :term + Couchbase::Cluster::SearchFacet.term(proto_facet.term.field) do |f| + f.size = proto_facet.term.size if proto_facet.term.has_size? + end + when :numeric_range + Couchbase::Cluster::SearchFacet.numeric_range(proto_facet.numeric_range.name) do |f| + f.size = proto_facet.numeric_range.size if proto_facet.numeric_range.has_size? + proto_facet.numeric_range.numeric_ranges.each do |proto_range| + min = proto_range.has_min? ? proto_range.min : nil + max = proto_range.has_max? ? proto_range.max : nil + f.add(proto_range.name, min, max) + end + end + when :date_range + Couchbase::Cluster::SearchFacet.date_range(proto_facet.date_range) do |f| + f.size = proto_facet.date_range.size if proto_facet.date_range.has_size? + proto_facet.date_range.date_ranges.each do |proto_range| + start_time = proto_range.has_start? ? proto_range.start.to_time : nil + end_time = proto_range.has_end? ? proto_range.end.to_time : nil + f.add(proto_range.name, start_time, end_time) + end + end + else + raise PerformerError, "Search facet type #{proto_facet.facet} not supported" + end + end + @options.facets = facets + self + end + + def set_raw + # TODO: Complete this if/when support for raw is added to the SDK + raise PerformerError, "`raw` search option not supported in the Ruby SDK" if @raw_options.raw.size.positive? + + self + end + + def set_include_locations + @options.include_locations = @raw_options.include_locations if @raw_options.has_include_locations? + self + end + + def set_vector_query_combination + @options.vector_query_combination = @raw_options.vector_query_combination.downcase if @raw_options.has_vector_query_combination? + self + end + + def set_serializer + return self unless @raw_options.has_serialize? + + @options.transcoder = CustomJsonTranscoder.new if @raw_options.serialize.serializer == :custom_serializer + + self + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search/results.rb b/fit-performer/lib/fit/performer/commands/search/results.rb new file mode 100644 index 00000000..5dfe5f64 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search/results.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module FIT + module Performer + module Commands + module Search + class Results < SharedResults + def self.as_search_result_stream(initiated:, stream_id:, fields_as: nil) + begin + result = yield + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + + to_search_result_enumerator(result: result, initiated: initiated, stream_id: stream_id, fields_as: fields_as) + end + + def self.to_search_result_enumerator(result:, initiated:, stream_id:, fields_as: nil) + kwargs = {initiated: initiated, stream_id: stream_id} + Enumerator.new do |y| + result.rows.each do |row| + y << to_search_result(to_search_row(row, fields_as: fields_as), **kwargs) + end + y << to_search_result(to_search_facets(result.facets), **kwargs) unless result.facets.nil? + y << to_search_result(to_search_meta_data(result.meta_data), **kwargs) unless result.meta_data.nil? + end + end + + def self.to_search_result(item, initiated:, stream_id:) + res = FIT::Protocol::SDK::Search::StreamingSearchResult.new(stream_id: stream_id) + case item + when FIT::Protocol::SDK::Search::SearchRow + res.row = item + when FIT::Protocol::SDK::Search::SearchFacets + res.facets = item + when FIT::Protocol::SDK::Search::SearchMetaData + res.meta_data = item + else + raise PerformerError, "#{item.class} is not a valid search stream item" + end + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new(search_streaming_result: res), + initiated: initiated, + ) + end + + def self.to_search_row(row, fields_as: nil) + proto_row = FIT::Protocol::SDK::Search::SearchRow.new( + index: row.index, + id: row.id, + score: row.score, + fields: get_content(content: row.fields, content_as: fields_as), + ) + row.locations&.get_all&.each do |loc| + proto_row.locations << to_search_row_location(loc) + end + proto_row + end + + def self.to_search_row_location(loc) + FIT::Protocol::SDK::Search::SearchRowLocation.new( + field: loc.field, + term: loc.term, + position: loc.position, + start: loc.start_offset, + end: loc.end_offset, + array_positions: loc.array_positions, + ) + end + + def self.to_search_facet_result(facet_res) + FIT::Protocol::SDK::Search::SearchFacetResult.new( + name: facet_res.name, + field: facet_res.field, + total: facet_res.total, + missing: facet_res.missing, + other: facet_res.other, + ) + end + + def self.to_search_facets(facets) + proto_facets = FIT::Protocol::SDK::Search::SearchFacets.new + facets.each do |name, facet| + proto_facets.facets[name] = to_search_facet_result(facet) + end + proto_facets + end + + def self.to_search_meta_data(meta) + proto_meta = FIT::Protocol::SDK::Search::SearchMetaData.new(metrics: to_search_metrics(meta.metrics)) + meta.errors&.each do |k, v| + proto_meta.errors[k] = v + end + proto_meta + end + + def self.to_search_metrics(metrics) + FIT::Protocol::SDK::Search::SearchMetrics.new( + took_msec: metrics.took, + total_rows: metrics.total_rows, + max_score: metrics.max_score, + total_partition_count: metrics.total_partition_count, + success_partition_count: metrics.success_partition_count, + error_partition_count: metrics.error_partition_count, + ) + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search/search_command.rb b/fit-performer/lib/fit/performer/commands/search/search_command.rb new file mode 100644 index 00000000..4bfb358c --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search/search_command.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module FIT + module Performer + module Commands + module Search + class SearchCommand + attr_reader :stream_config + + def initialize( + index_name:, search_request:, return_result:, initiated:, stream_config:, get_span_fn:, + cluster: nil, scope: nil, raw_options: nil, fields_as: nil + ) + @cmd_args = [index_name, search_request] + @receiver = scope || cluster + @raw_options = raw_options + @return_result = return_result + @initiated = initiated + @stream_config = stream_config + @fields_as = fields_as + @get_span_fn = get_span_fn + end + + def stream_type + :STREAM_FULL_TEXT_SEARCH + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Search.new, raw_options: @raw_options) + builder.set_timeout + .set_limit + .set_skip + .set_explain + .set_highlight + .set_fields + .set_scan_consistency + .set_consistent_with + .set_sort + .set_facets + .set_raw + .set_include_locations + .set_serializer + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_search_result_stream(initiated: @initiated, stream_id: @stream_config.stream_id, fields_as: @fields_as) do + @receiver.search(*@cmd_args) + end + end + + def self.create_command(...) + cmd = SearchCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search/search_query_command.rb b/fit-performer/lib/fit/performer/commands/search/search_query_command.rb new file mode 100644 index 00000000..31acb9e0 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search/search_query_command.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module Search + class SearchQueryCommand + attr_reader :stream_config + + def initialize( + index_name:, search_query:, return_result:, initiated:, stream_config:, get_span_fn:, + cluster: nil, scope: nil, raw_options: nil, fields_as: nil + ) + @cmd_args = [index_name, search_query] + @receiver = cluster || scope + @raw_options = raw_options + @return_result = return_result + @initiated = initiated + @stream_config = stream_config + @fields_as = fields_as + @get_span_fn = get_span_fn + end + + def stream_type + :STREAM_FULL_TEXT_SEARCH + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new(options: Couchbase::Options::Search.new, raw_options: @raw_options) + builder.set_timeout + .set_limit + .set_skip + .set_explain + .set_highlight + .set_fields + .set_scan_consistency + .set_consistent_with + .set_sort + .set_facets + .set_raw + .set_include_locations + .set_parent_span(@get_span_fn) + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_search_result_stream(initiated: @initiated, stream_id: @stream_config.stream_id, fields_as: @fields_as) do + @receiver.search_query(*@cmd_args) + end + end + + def self.create_command(...) + cmd = SearchQueryCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager.rb b/fit-performer/lib/fit/performer/commands/search_index_manager.rb new file mode 100644 index 00000000..398042c7 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'search_index_manager/allow_querying_command' +require_relative 'search_index_manager/analyze_document_command' +require_relative 'search_index_manager/disallow_querying_command' +require_relative 'search_index_manager/drop_index_command' +require_relative 'search_index_manager/freeze_plan_command' +require_relative 'search_index_manager/get_all_indexes_command' +require_relative 'search_index_manager/get_index_command' +require_relative 'search_index_manager/get_indexed_documents_count_command' +require_relative 'search_index_manager/pause_ingest_command' +require_relative 'search_index_manager/resume_ingest_command' +require_relative 'search_index_manager/unfreeze_plan_command' +require_relative 'search_index_manager/upsert_index_command' + +module FIT + module Performer + module Commands + module SearchIndexManager + def self.get_index_definition(raw_cmd) + raw_defn = JSON.parse(raw_cmd.index_definition) + Couchbase::Management::SearchIndex.new do |index| + index.name = raw_defn['name'] if raw_defn.key?('name') + index.type = raw_defn['type'] if raw_defn.key?('type') + index.uuid = raw_defn['uuid'] if raw_defn.key?('uuid') + index.params = raw_defn['params'] if raw_defn.key?('params') + index.source_name = raw_defn['sourceName'] if raw_defn.key?('sourceName') + index.source_type = raw_defn['sourceType'] if raw_defn.key?('sourceType') + index.source_uuid = raw_defn['sourceUUID'] if raw_defn.key?('sourceUUID') + index.source_params = raw_defn['sourceParams'] if raw_defn.key?('sourceParams') + index.plan_params = raw_defn['planParams'] if raw_defn.key?('planParams') + end + end + + def self.build_cluster_level_command(raw_cmd, cluster, cmd_kwargs) + cmd_kwargs[:manager] = cluster.search_indexes + build_shared_command(raw_cmd.shared, cmd_kwargs) + end + + def self.build_scope_level_command(raw_cmd, scope, cmd_kwargs) + cmd_kwargs[:manager] = scope.search_indexes + build_shared_command(raw_cmd.shared, cmd_kwargs) + end + + def self.build_shared_command(raw_cmd, cmd_kwargs) + cmd_type = raw_cmd.command + begin + cmd = raw_cmd.public_send(cmd_type) + rescue NoMethodError + raise PerformerError, "Search index management command `#{cmd_type}` not implemented" + end + + cmd_kwargs[:raw_options] = cmd.options + + case cmd_type + when :get_index + cmd_kwargs[:index_name] = cmd.index_name + GetIndexCommand.create_command(**cmd_kwargs) + when :get_all_indexes + GetAllIndexesCommand.create_command(**cmd_kwargs) + when :upsert_index + cmd_kwargs[:index_definition] = get_index_definition(cmd) + UpsertIndexCommand.create_command(**cmd_kwargs) + when :drop_index + cmd_kwargs[:index_name] = cmd.index_name + DropIndexCommand.create_command(**cmd_kwargs) + when :get_indexed_documents_count + cmd_kwargs[:index_name] = cmd.index_name + GetIndexedDocumentsCountCommand.create_command(**cmd_kwargs) + when :pause_ingest + cmd_kwargs[:index_name] = cmd.index_name + PauseIngestCommand.create_command(**cmd_kwargs) + when :resume_ingest + cmd_kwargs[:index_name] = cmd.index_name + ResumeIngestCommand.create_command(**cmd_kwargs) + when :allow_querying + cmd_kwargs[:index_name] = cmd.index_name + AllowQueryingCommand.create_command(**cmd_kwargs) + when :disallow_querying + cmd_kwargs[:index_name] = cmd.index_name + DisallowQueryingCommand.create_command(**cmd_kwargs) + when :freeze_plan + cmd_kwargs[:index_name] = cmd.index_name + FreezePlanCommand.create_command(**cmd_kwargs) + when :unfreeze_plan + cmd_kwargs[:index_name] = cmd.index_name + UnfreezePlanCommand.create_command(**cmd_kwargs) + when :analyze_document + cmd_kwargs[:index_name] = cmd.index_name + cmd_kwargs[:document] = JSON.parse(cmd.document) + AnalyzeDocumentCommand.create_command(**cmd_kwargs) + else + raise PerformerError, "Search index management command `#{cmd_type}` not implemented" + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/allow_querying_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/allow_querying_command.rb new file mode 100644 index 00000000..8f973746 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/allow_querying_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class AllowQueryingCommand + def initialize(manager:, index_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [index_name] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::AllowQueryingOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.allow_querying(*@cmd_args) + end + end + + def self.create_command(...) + cmd = AllowQueryingCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/analyze_document_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/analyze_document_command.rb new file mode 100644 index 00000000..e86414bf --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/analyze_document_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class AnalyzeDocumentCommand + def initialize(manager:, index_name:, document:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [index_name, document] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::DisallowQueryingOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_analyze_document_result(initiated: @initiated) do + @manager.analyze_document(*@cmd_args) + end + end + + def self.create_command(...) + cmd = AnalyzeDocumentCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/disallow_querying_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/disallow_querying_command.rb new file mode 100644 index 00000000..ad5ebad8 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/disallow_querying_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class DisallowQueryingCommand + def initialize(manager:, index_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [index_name] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::DisallowQueryingOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.disallow_querying(*@cmd_args) + end + end + + def self.create_command(...) + cmd = DisallowQueryingCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/drop_index_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/drop_index_command.rb new file mode 100644 index 00000000..3c29cd0a --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/drop_index_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class DropIndexCommand + def initialize(manager:, index_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [index_name] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::DropIndexOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.drop_index(*@cmd_args) + end + end + + def self.create_command(...) + cmd = DropIndexCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/freeze_plan_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/freeze_plan_command.rb new file mode 100644 index 00000000..aa36e751 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/freeze_plan_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class FreezePlanCommand + def initialize(manager:, index_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [index_name] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::FreezePlanOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.freeze_plan(*@cmd_args) + end + end + + def self.create_command(...) + cmd = FreezePlanCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/get_all_indexes_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/get_all_indexes_command.rb new file mode 100644 index 00000000..985722d3 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/get_all_indexes_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class GetAllIndexesCommand + def initialize(manager:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::GetAllIndexesOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_search_indexes(initiated: @initiated) do + @manager.get_all_indexes(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetAllIndexesCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/get_index_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/get_index_command.rb new file mode 100644 index 00000000..5ba5b6a8 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/get_index_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class GetIndexCommand + def initialize(manager:, index_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [index_name] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::GetIndexOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_search_index(initiated: @initiated) do + @manager.get_index(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetIndexCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/get_indexed_documents_count_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/get_indexed_documents_count_command.rb new file mode 100644 index 00000000..6d36d31f --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/get_indexed_documents_count_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class GetIndexedDocumentsCountCommand + def initialize(manager:, index_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [index_name] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::GetIndexedDocumentsCountOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_indexed_documents_count(initiated: @initiated) do + @manager.get_indexed_documents_count(*@cmd_args) + end + end + + def self.create_command(...) + cmd = GetIndexedDocumentsCountCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/options_builder.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/options_builder.rb new file mode 100644 index 00000000..4d8914fd --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/options_builder.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module FIT + module Performer + module Commands + module SearchIndexManager + class OptionsBuilder < OptionsBuilderBase + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/pause_ingest_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/pause_ingest_command.rb new file mode 100644 index 00000000..4b7f09c0 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/pause_ingest_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class PauseIngestCommand + def initialize(manager:, index_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [index_name] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::PauseIngestOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.pause_ingest(*@cmd_args) + end + end + + def self.create_command(...) + cmd = PauseIngestCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/results.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/results.rb new file mode 100644 index 00000000..45a27d0f --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/results.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fit/protocol/sdk.search.index_manager_pb' + +module FIT + module Performer + module Commands + module SearchIndexManager + class Results < SharedResults + def self.as_search_index(initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + res = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new( + search_index_manager_result: FIT::Protocol::SDK::Search::IndexManager::Result.new( + index: to_search_index(res), + ), + ), + initiated: initiated, + elapsed_nanos: nanos, + ) + end + + def self.as_search_indexes(initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + res = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new( + search_index_manager_result: FIT::Protocol::SDK::Search::IndexManager::Result.new( + indexes: to_search_indexes(res), + ), + ), + initiated: initiated, + elapsed_nanos: nanos, + ) + end + + def self.as_indexed_documents_count(initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + res = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new( + search_index_manager_result: FIT::Protocol::SDK::Search::IndexManager::Result.new( + indexed_document_counts: res, + ), + ), + initiated: initiated, + elapsed_nanos: nanos, + ) + end + + def self.as_analyze_document_result(initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + res = yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + + proto_res = FIT::Protocol::SDK::Search::IndexManager::AnalyzeDocumentResult.new + res.each do |r| + proto_res.results << r.to_json + end + + create_run_result( + sdk_result: FIT::Protocol::SDK::Result.new( + search_index_manager_result: FIT::Protocol::SDK::Search::IndexManager::Result.new( + analyze_document: proto_res, + ), + ), + initiated: initiated, + elapsed_nanos: nanos, + ) + end + + def self.to_search_index(res) + FIT::Protocol::SDK::Search::IndexManager::SearchIndex.new( + uuid: res.uuid, + name: res.name, + type: res.type, + source_uuid: res.source_uuid, + source_type: res.source_type, + params: (res.params.empty? ? nil : res.params).to_json, + source_params: (res.source_params.empty? ? nil : res.source_params).to_json, + plan_params: (res.plan_params.empty? ? nil : res.plan_params).to_json, + ) + end + + def self.to_search_indexes(res) + proto_res = FIT::Protocol::SDK::Search::IndexManager::SearchIndexes.new + res.each do |r| + proto_res.indexes << to_search_index(r) + end + proto_res + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/resume_ingest_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/resume_ingest_command.rb new file mode 100644 index 00000000..537e8602 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/resume_ingest_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class ResumeIngestCommand + def initialize(manager:, index_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [index_name] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::ResumeIngestOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.resume_ingest(*@cmd_args) + end + end + + def self.create_command(...) + cmd = ResumeIngestCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/unfreeze_plan_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/unfreeze_plan_command.rb new file mode 100644 index 00000000..6f424aec --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/unfreeze_plan_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class UnfreezePlanCommand + def initialize(manager:, index_name:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [index_name] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::UnfreezePlanOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.unfreeze_plan(*@cmd_args) + end + end + + def self.create_command(...) + cmd = UnfreezePlanCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/search_index_manager/upsert_index_command.rb b/fit-performer/lib/fit/performer/commands/search_index_manager/upsert_index_command.rb new file mode 100644 index 00000000..b4faf90e --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/search_index_manager/upsert_index_command.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'results' +require_relative 'options_builder' + +module FIT + module Performer + module Commands + module SearchIndexManager + class UpsertIndexCommand + def initialize(manager:, index_definition:, initiated:, get_span_fn:, return_result: true, raw_options: nil) + @manager = manager + @raw_options = raw_options + @cmd_args = [index_definition] + @initiated = initiated + @return_result = return_result + @get_span_fn = get_span_fn + end + + def set_options + return if @raw_options.nil? + + builder = OptionsBuilder.new( + options: Couchbase::Management::SearchIndexManager::UpsertIndexOptions.new, + raw_options: @raw_options, + ) + builder.set_timeout + .set_parent_span(@get_span_fn) + + @cmd_args.append(builder.options) + end + + def execute_command + Results.as_success(initiated: @initiated) do + @manager.upsert_index(*@cmd_args) + end + end + + def self.create_command(...) + cmd = UpsertIndexCommand.new(...) + cmd.set_options + cmd + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/commands/shared_results.rb b/fit-performer/lib/fit/performer/commands/shared_results.rb new file mode 100644 index 00000000..1c9487e3 --- /dev/null +++ b/fit-performer/lib/fit/performer/commands/shared_results.rb @@ -0,0 +1,185 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'couchbase/errors' + +require 'fit/protocol/sdk.workload_pb' +require 'fit/protocol/run.top_level_pb' +require 'fit/protocol/shared.exceptions_pb' +require 'fit/protocol/shared.content_pb' + +module FIT + module Performer + module Commands + class SharedResults + def self.as_success(initiated:) + begin + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + yield + end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + rescue StandardError => e + return to_exception(error: e, initiated: initiated) + end + nanos = ((end_time - start_time) * (10**9)).round + to_success(elapsed_nanos: nanos, initiated: initiated) + end + + def self.to_exception(error:, initiated: nil, unwrapped: false) + name = error.class.to_s + serialized = error.to_s + pb_exception = + if error.is_a?(Couchbase::Error::CouchbaseError) || error.is_a?(Couchbase::Error::InvalidArgument) + FIT::Protocol::Shared::Exception.new( + couchbase: FIT::Protocol::Shared::CouchbaseExceptionEx.new( + name: name, + serialized: serialized, + type: ERROR_MAP[error.class], + ), + ) + else + FIT::Protocol::Shared::Exception.new( + other: FIT::Protocol::Shared::ExceptionOther.new(name: name, serialized: serialized), + ) + end + + return pb_exception if unwrapped # Returns the protobuf exception object only - used for errors in streams + + sdk_result = FIT::Protocol::SDK::Result.new(exception: pb_exception) + create_run_result(sdk_result: sdk_result, initiated: initiated) + end + + def self.to_success(elapsed_nanos:, initiated:) + sdk_result = FIT::Protocol::SDK::Result.new(success: true) + create_run_result(sdk_result: sdk_result, initiated: initiated, elapsed_nanos: elapsed_nanos) + end + + def self.create_run_result(sdk_result:, initiated:, elapsed_nanos: nil) + kwargs = { + initiated: initiated, + sdk: sdk_result, + } + kwargs[:elapsedNanos] = elapsed_nanos unless elapsed_nanos.nil? + FIT::Protocol::Run::Result.new(**kwargs) + end + + def self.get_content(content:, content_as: nil) + res_content = FIT::Protocol::Shared::ContentTypes.new + as = content_as&.as + case as + when :as_string + res_content.content_as_string = + if content.is_a?(String) + content + else + content.to_json + end + when :as_byte_array + res_content.content_as_bytes = + if content.is_a?(String) + content.b + else + content.to_json.b + end + when :as_json_object, :as_json_array, nil + res_content.content_as_bytes = content.to_json.b + when :as_boolean + res_content.content_as_bool = !content.nil? && content != false + when :as_integer + res_content.content_as_int64 = content.to_i + when :as_floating_point + res_content.content_as_double = content.to_f + else + raise PerformerError, "ContentAs type #{content_as.as} not supported" + end + res_content + end + + ERROR_MAP = { + Couchbase::Error::CouchbaseError => 0, + Couchbase::Error::Timeout => 1, + Couchbase::Error::RequestCanceled => 2, + Couchbase::Error::InvalidArgument => 3, + Couchbase::Error::ServiceNotAvailable => 4, + Couchbase::Error::InternalServerFailure => 5, + Couchbase::Error::AuthenticationFailure => 6, + Couchbase::Error::TemporaryFailure => 7, + Couchbase::Error::ParsingFailure => 8, + Couchbase::Error::CasMismatch => 9, + Couchbase::Error::BucketNotFound => 10, + Couchbase::Error::CollectionNotFound => 11, + Couchbase::Error::UnsupportedOperation => 12, + Couchbase::Error::AmbiguousTimeout => 13, + Couchbase::Error::UnambiguousTimeout => 14, + Couchbase::Error::FeatureNotAvailable => 15, + Couchbase::Error::ScopeNotFound => 16, + Couchbase::Error::IndexNotFound => 17, + Couchbase::Error::IndexExists => 18, + Couchbase::Error::EncodingFailure => 19, + Couchbase::Error::DecodingFailure => 20, + Couchbase::Error::RateLimited => 21, + Couchbase::Error::QuotaLimited => 22, + Couchbase::Error::DocumentNotFound => 101, + Couchbase::Error::DocumentIrretrievable => 102, + Couchbase::Error::DocumentLocked => 103, + Couchbase::Error::ValueTooLarge => 104, + Couchbase::Error::DocumentExists => 105, + Couchbase::Error::DurabilityLevelNotAvailable => 107, + Couchbase::Error::DurabilityImpossible => 108, + Couchbase::Error::DurabilityAmbiguous => 109, + Couchbase::Error::DurableWriteInProgress => 110, + Couchbase::Error::DurableWriteReCommitInProgress => 111, + Couchbase::Error::PathNotFound => 113, + Couchbase::Error::PathMismatch => 114, + Couchbase::Error::PathInvalid => 115, + Couchbase::Error::PathTooBig => 116, + Couchbase::Error::PathTooDeep => 117, + Couchbase::Error::ValueTooDeep => 118, + Couchbase::Error::ValueInvalid => 119, + Couchbase::Error::DocumentNotJson => 120, + Couchbase::Error::NumberTooBig => 121, + Couchbase::Error::DeltaInvalid => 122, + Couchbase::Error::PathExists => 123, + Couchbase::Error::XattrUnknownMacro => 124, + Couchbase::Error::XattrInvalidKeyCombo => 126, + Couchbase::Error::XattrUnknownVirtualAttribute => 127, + Couchbase::Error::XattrCannotModifyVirtualAttribute => 128, + Couchbase::Error::XattrNoAccess => 130, + Couchbase::Error::DocumentNotLocked => 131, + Couchbase::Error::PlanningFailure => 201, + Couchbase::Error::IndexFailure => 202, + Couchbase::Error::PreparedStatementFailure => 203, + Couchbase::Error::DmlFailure => 204, + Couchbase::Error::CompilationFailure => 301, + Couchbase::Error::JobQueueFull => 302, + Couchbase::Error::DatasetNotFound => 303, + Couchbase::Error::DataverseNotFound => 304, + Couchbase::Error::DatasetExists => 305, + Couchbase::Error::DataverseExists => 306, + Couchbase::Error::LinkNotFound => 307, + Couchbase::Error::ViewNotFound => 501, + Couchbase::Error::DesignDocumentNotFound => 502, + Couchbase::Error::CollectionExists => 601, + Couchbase::Error::ScopeExists => 602, + Couchbase::Error::UserNotFound => 603, + Couchbase::Error::GroupNotFound => 604, + Couchbase::Error::BucketExists => 605, + Couchbase::Error::UserExists => 606, + Couchbase::Error::BucketNotFlushable => 607, + }.freeze + end + end + end +end diff --git a/fit-performer/lib/fit/performer/connection.rb b/fit-performer/lib/fit/performer/connection.rb new file mode 100644 index 00000000..b147605c --- /dev/null +++ b/fit-performer/lib/fit/performer/connection.rb @@ -0,0 +1,241 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "fit/performer/observability/otel" + +require "couchbase/opentelemetry" + +require "securerandom" +require "tmpdir" + +module FIT + module Performer + class Connection + attr_accessor :cluster + attr_accessor :tracer + + def initialize(request) + @logger = Logger.new($stdout) + opts = create_cluster_options(request) + conn_str = get_connection_string(request) + @logger.info("Creating connection using connection string `#{conn_str}` (ID = #{request.cluster_connection_id})") + @cluster = Couchbase::Cluster.connect(conn_str, opts) + end + + def close + @tracer_provider&.shutdown + @meter_provider&.shutdown + @cluster.disconnect + + return if @temp_dir.nil? + + FileUtils.remove_entry(@temp_dir) + end + + def create_authenticator(proto_auth) + case proto_auth.authenticator + when :password_auth + Couchbase::PasswordAuthenticator.new( + proto_auth.password_auth.username, + proto_auth.password_auth.password, + ) + when :certificate_auth + certificate_path = File.join(temp_dir, "#{SecureRandom.uuid}-cert.pem") + File.write(certificate_path, proto_auth.certificate_auth.cert) + key_path = File.join(temp_dir, "#{SecureRandom.uuid}-key.pem") + File.write(key_path, proto_auth.certificate_auth.key) + + Couchbase::CertificateAuthenticator.new( + certificate_path, + key_path, + ) + when :jwt_auth + Couchbase::JWTAuthenticator.new(proto_auth.jwt_auth.jwt) + else + raise PerformerError, "Authenticator type `#{proto_auth.authenticator}` is not supported" + end + end + + private + + def temp_dir + @temp_dir ||= Dir.mktmpdir + end + + def get_duration(config, field, unit = :millis) + has_method_name = :"has_#{field}?" + return unless config.public_send(has_method_name) + + factor = + case unit + when :millis + 1 + when :secs + 1000 + else + raise "Unknown duration unit `#{unit}`" + end + config.public_send(field) * factor + end + + def create_application_telemetry_options(config) + Couchbase::Options::Cluster::ApplicationTelemetry.new do |opts| + opts.enable = config.enable_app_telemetry if config.has_enable_app_telemetry? + opts.backoff = get_duration(config, :app_telemetry_backoff_secs, :secs) + opts.ping_interval = get_duration(config, :app_telemetry_ping_interval_secs, :secs) + opts.ping_timeout = get_duration(config, :app_telemetry_ping_timeout_secs, :secs) + opts.override_endpoint = config.app_telemetry_endpoint if config.has_enable_app_telemetry? + end + end + + def create_tracer(tracing_config) + @tracer_provider = Observability::Otel.create_tracer_provider(tracing_config) + @tracer = Couchbase::OpenTelemetry::RequestTracer.new(@tracer_provider) + end + + def create_meter(metrics_config) + @meter_provider = Observability::Otel.create_meter_provider(metrics_config) + @meter = Couchbase::OpenTelemetry::Meter.new(@meter_provider) + end + + def create_cluster_options(request) + cluster_options = {} + cluster_options[:authenticator] = if request.has_authenticator? + create_authenticator(request.authenticator) + else + Couchbase::PasswordAuthenticator.new( + request.cluster_username, + request.cluster_password, + ) + end + + if request.has_cluster_config? + config = request.cluster_config + cluster_options.update( + { + key_value_timeout: get_duration(config, :kv_timeout_millis), + view_timeout: get_duration(config, :view_timeout_secs, :secs), + query_timeout: get_duration(config, :query_timeout_secs, :secs), + analytics_timeout: get_duration(config, :analytics_timeout_secs, :secs), + search_timeout: get_duration(config, :search_timeout_secs, :secs), + management_timeout: get_duration(config, :management_timeout_secs, :secs), + tcp_keep_alive_interval: get_duration(config, :tcp_keep_alive_time_millis), + config_poll_interval: get_duration(config, :config_poll_interval_secs, :secs), + config_poll_floor: get_duration(config, :config_poll_floor_interval_secs, :secs), + config_idle_redial_timeout: get_duration(config, :config_idle_redial_timeout_secs, :secs), + idle_http_connection_timeout: get_duration(config, :idle_http_connection_timeout_secs, :secs), + preferred_server_group: config.has_preferred_server_group? ? config.preferred_server_group : nil, + # [if:3.8.0] + application_telemetry: create_application_telemetry_options(config), + # [end] + }, + ) + + if config.has_observability_config? + observability_config = config.observability_config + + if observability_config.use_noop_tracer + cluster_options[:enable_tracing] = false + elsif observability_config.has_tracing? + cluster_options[:tracer] = create_tracer(observability_config.tracing) + end + + cluster_options[:meter] = create_meter(observability_config.metrics) if observability_config.has_metrics? + + if observability_config.has_threshold_logging_tracer? + threshold_config = observability_config.threshold_logging_tracer + cluster_options.update( + { + threshold_emit_interval: get_duration(threshold_config, :emit_interval_millis), + key_value_threshold: get_duration(threshold_config, :kv_threshold_millis), + query_threshold: get_duration(threshold_config, :query_threshold_millis), + analytics_threshold: get_duration(threshold_config, :analytics_threshold_millis), + search_threshold: get_duration(threshold_config, :search_threshold_millis), + view_threshold: get_duration(threshold_config, :views_threshold_millis), + threshold_sample_size: threshold_config.has_sample_size? ? threshold_config.sample_size : nil, + enable_tracing: threshold_config.has_enabled? ? threshold_config.enabled : nil, + }, + ) + end + + if observability_config.has_logging_meter? + logging_meter_config = observability_config.logging_meter + cluster_options.update( + { + metrics_emit_interval: get_duration(logging_meter_config, :emit_interval_millis), + enable_metrics: logging_meter_config.has_enabled? ? logging_meter_config.enabled : nil, + }, + ) + end + + if observability_config.has_orphan_response? + orphan_config = observability_config.orphan_response + cluster_options.update( + { + orphaned_emit_interval: get_duration(orphan_config, :emit_interval_millis), + oprhaned_sample_size: orphan_config.has_sample_size? ? orphan_config.sample_size : nil, + # TODO(RCBC-539): We don't currently have an option for disabling orphan reporting. Once this ticket + # is done, add the option here. + }, + ) + end + end + end + + Couchbase::Options::Cluster.new(**cluster_options) + end + + def get_connection_string(request) + conn_str = request.cluster_hostname + conn_str = "couchbase://#{conn_str}" unless conn_str.include?("://") + + return conn_str unless request.has_cluster_config? + + config = request.cluster_config + + if config.has_insecure? && config.insecure + conn_str += if conn_str.include?("?") + "&tls_verify=none" + else + "?tls_verify=none" + end + end + + # Either 'cert' or 'cert_path' are set, not both + cert_path = + if config.has_cert_path? + config.cert_path + elsif config.has_cert? + # Create a temporary file with the certificate + path = File.join(temp_dir, "#{SecureRandom.uuid}-cluster-cert.pem") + File.write(path, config.cert) + path + end + + return conn_str if cert_path.nil? + + # The certificate path in the connection string takes precedence over the one in ClusterConfig + return conn_str if conn_str.include?("trust_certificate=") + + if conn_str.include?("?") + conn_str + "&trust_certificate=#{cert_path}" + else + conn_str + "?trust_certificate=#{cert_path}" + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/connections.rb b/fit-performer/lib/fit/performer/connections.rb new file mode 100644 index 00000000..63ba34c6 --- /dev/null +++ b/fit-performer/lib/fit/performer/connections.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "fit/protocol/shared.cluster_pb" +require "fit/performer/connection" + +module FIT + module Performer + class Connections + def initialize + @logger = Logger.new($stdout) + @connections = {} + end + + def create_connection(request) + @connections[request.cluster_connection_id] = Connection.new(request) + FIT::Protocol::Shared::ClusterConnectionCreateResponse.new(cluster_connection_count: count) + end + + def close_connection(request) + @connections[request.cluster_connection_id].close + @connections.delete(request.cluster_connection_id) + FIT::Protocol::Shared::ClusterConnectionCloseResponse.new(cluster_connection_count: count) + end + + def disconnect_connections(_request) + @connections.values.map(&:close) + @connections.clear + FIT::Protocol::Shared::DisconnectConnectionsResponse.new + end + + def count + @connections.size + end + + def [](cluster_id) + raise PerformerError, "Connection with ID #{cluster_id} not found" unless @connections.key?(cluster_id) + + @connections[cluster_id] + end + end + end +end diff --git a/fit-performer/lib/fit/performer/multi_thread_executor.rb b/fit-performer/lib/fit/performer/multi_thread_executor.rb new file mode 100644 index 00000000..9964f0c7 --- /dev/null +++ b/fit-performer/lib/fit/performer/multi_thread_executor.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'logger' +require 'fit/performer/workloads' + +module FIT + module Performer + class MultiThreadExecutor + attr_reader :results + + def initialize(connection, run_id, stream_owner, span_owner) + @logger = Logger.new($stdout) + @results = Queue.new + @connection = connection + @run_id = run_id + @stream_owner = stream_owner + @span_owner = span_owner + @global_counters = {} + @global_semaphore = Mutex.new + end + + def build_workloads(request) + @workloads = request.workloads.horizontal_scaling.map do |h| + h.workloads.map do |w| + Workloads.build_workload(w, @connection, @run_id, @stream_owner, @span_owner) + end + end + end + + def start_workload_thread(workloads) + Thread.new do # rubocop:disable ThreadSafety/NewThread + workloads.each do |w| + w.set_bounds(@global_counters, @global_semaphore) + w.execute(@results, @global_counters, @global_semaphore) + end + end + end + + def execute_workloads + Thread.new do # rubocop:disable ThreadSafety/NewThread + threads = [] + @workloads.each do |w| + thread = start_workload_thread(w) + threads.append(thread) + end + + # Wait for all threads to finish their workloads + threads.each(&:join) + + # Wait for all streams to finish + @stream_owner.wait_for_all_streams_from_run(@run_id) + + # Close the result queue to indicate that no more workloads will be executed + @results.close + end + end + + def self.build_executor(request, connection, run_id, stream_owner, span_owner) + executor = MultiThreadExecutor.new(connection, run_id, stream_owner, span_owner) + executor.build_workloads(request) + executor + end + end + end +end diff --git a/fit-performer/lib/fit/performer/observability/otel.rb b/fit-performer/lib/fit/performer/observability/otel.rb new file mode 100644 index 00000000..a535bc13 --- /dev/null +++ b/fit-performer/lib/fit/performer/observability/otel.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "opentelemetry-exporter-otlp" +require "opentelemetry-exporter-otlp-metrics" +require "opentelemetry-metrics-sdk" +require "opentelemetry-sdk" + +module FIT + module Performer + module Observability + module Otel + SAMPLING_PERCENTAGE_EPSILON = 0.00001 + TRACING_PATH = "/v1/traces" + METRICS_PATH = "/v1/metrics" + + def self.create_resource(proto_resources) + OpenTelemetry::SDK::Resources::Resource.create( + proto_resources.map do |k, v| # rubocop:disable Style/MapToHash -- Google::Protobuf::Map does not implement the full Hash API + [k, v.public_send(v.value)] + end.to_h, + ) + end + + def self.create_tracer_provider(config) + exporter = OpenTelemetry::Exporter::OTLP::Exporter.new( + endpoint: URI.join(config.endpoint_hostname, TRACING_PATH).to_s, + ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, + ) + + span_processor = + if config.batching + OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + exporter, + schedule_delay: config.export_every_millis, + ) + else + OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(exporter) + end + + sampler = + if config.sampling_percentage < SAMPLING_PERCENTAGE_EPSILON + OpenTelemetry::SDK::Trace::Samplers::ALWAYS_OFF + elsif config.sampling_percentage > 1.0 - SAMPLING_PERCENTAGE_EPSILON + OpenTelemetry::SDK::Trace::Samplers::ALWAYS_ON + else + OpenTelemetry::SDK::Trace::Samplers::TraceIdRatioBased.new(config.sampling_percentage) + end + + provider = OpenTelemetry::SDK::Trace::TracerProvider.new( + sampler: sampler, + resource: create_resource(config.resources), + ) + provider.add_span_processor(span_processor) + provider + end + + def self.create_meter_provider(config) + exporter = OpenTelemetry::Exporter::OTLP::Metrics::MetricsExporter.new( + endpoint: URI.join(config.endpoint_hostname, METRICS_PATH).to_s, + ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, + ) + + reader = OpenTelemetry::SDK::Metrics::Export::PeriodicMetricReader.new( + export_interval_millis: config.export_every_millis, + exporter: exporter, + ) + + provider = OpenTelemetry::SDK::Metrics::MeterProvider.new(resource: create_resource(config.resources)) + provider.add_metric_reader(reader) + provider + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/observability/span_owner.rb b/fit-performer/lib/fit/performer/observability/span_owner.rb new file mode 100644 index 00000000..8f87ed29 --- /dev/null +++ b/fit-performer/lib/fit/performer/observability/span_owner.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "fit/performer/performer_error" + +module FIT + module Performer + module Observability + class SpanOwner + def initialize + @spans = {} + @mutex = Mutex.new + end + + def get_span(span_id) + @mutex.synchronize do + raise PerformerError, "Span with ID #{span_id} not found" unless @spans.key?(span_id) + + @spans[span_id] + end + end + + def create_span(tracer, request) + parent_span = + (get_span(request.parent_span_id) if request.has_parent_span_id?) + + span = tracer.request_span(request.name, parent: parent_span) + request.attributes.each do |key, value| + span.set_attribute(key, value.public_send(value.value)) + end + + @mutex.synchronize do + @spans[request.id] = span + end + end + + def finish_span(request) + @mutex.synchronize do + raise PerformerError, "Span with ID #{request.id} not found" unless @spans.key?(request.id) + + @spans.delete(request.id) + end&.finish + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/performer_error.rb b/fit-performer/lib/fit/performer/performer_error.rb new file mode 100644 index 00000000..4b5634ce --- /dev/null +++ b/fit-performer/lib/fit/performer/performer_error.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module FIT + module Performer + class PerformerError < StandardError + end + end +end diff --git a/fit-performer/lib/fit/performer/request_executor.rb b/fit-performer/lib/fit/performer/request_executor.rb new file mode 100644 index 00000000..99b991ca --- /dev/null +++ b/fit-performer/lib/fit/performer/request_executor.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'logger' + +module FIT + module Performer + class RequestExecutor + def initialize(workload_executor, request) + @logger = Logger.new($stdout) + @workload_executor = workload_executor + @request = request + @conn_id = @request.workloads.cluster_connection_id + end + + def set_batch_size + @batch_size = 1 + return unless @request.has_config? && @request.config.has_streaming_config? && @request.config.streaming_config.has_batch_size? + + @batch_size = @request.config.streaming_config.batch_size + end + + def execute_request + @workload_executor.execute_workloads + end + + def results + Enumerator.new do |enum| + loop do + result = next_result + break if result.nil? + + enum << result + end + end + end + + def self.build_request(workload_executor, request) + executor = RequestExecutor.new(workload_executor, request) + executor.set_batch_size + executor + end + + private + + def next_result + batch = [] + deadline = Time.now + 0.1 + loop do + break if batch.size == @batch_size || Time.now > deadline + + result = @workload_executor.results.pop + break if result.nil? # No more results + + batch.append(result) + end + + return nil if batch.empty? + return batch[0] if batch.size == 1 + + batched_result = FIT::Protocol::Run::BatchedResult.new(result: batch) + FIT::Protocol::Run::Result.new(batched: batched_result) + end + end + end +end diff --git a/fit-performer/lib/fit/performer/service.rb b/fit-performer/lib/fit/performer/service.rb new file mode 100644 index 00000000..7394b34c --- /dev/null +++ b/fit-performer/lib/fit/performer/service.rb @@ -0,0 +1,193 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'logger' +require 'securerandom' + +require 'grpc' + +require 'couchbase' + +require 'fit/performer/connections' +require 'fit/performer/performer_error' +require 'fit/performer/multi_thread_executor' +require 'fit/performer/request_executor' +require 'fit/performer/streaming' +require 'fit/performer/observability/span_owner' + +require 'fit/protocol/performer_services_pb' +require 'fit/protocol/performer.caps_pb' +require 'fit/protocol/shared.echo_pb' + +module FIT + module Performer + class Service < FIT::Protocol::PerformerService::Service + # rubocop:disable Naming/VariableNumber + PERFORMER_CAPS = [ + :KV_SUPPORT_1, + :CLUSTER_CONFIG_CERT, + :CLUSTER_CONFIG_INSECURE, + :CUSTOM_SERIALIZATION_SUPPORT_FOR_SEARCH, + :OBSERVABILITY_1, + ].freeze + + SDK_CAPS = [ + :SDK_QUERY_INDEX_MANAGEMENT, + :SDK_COLLECTION_QUERY_INDEX_MANAGEMENT, + :SDK_SEARCH_INDEX_MANAGEMENT, + :SDK_QUERY, + :SDK_LOOKUP_IN, + :SDK_BUCKET_MANAGEMENT, + :SDK_KV, + :SDK_SEARCH, + :SUPPORTS_AUTHENTICATOR, + :SDK_QUERY_READ_FROM_REPLICA, + :SDK_KV_RANGE_SCAN, + :SDK_LOOKUP_IN_REPLICAS, + :SDK_COLLECTION_MANAGEMENT, + :SDK_MANAGEMENT_HISTORY_RETENTION, + :SDK_DOCUMENT_NOT_LOCKED, + :SDK_VECTOR_SEARCH, + :SDK_SCOPE_SEARCH_INDEX_MANAGEMENT, + :SDK_SCOPE_SEARCH, + :SDK_INDEX_MANAGEMENT_RFC_REVISION_25, + :SDK_SEARCH_RFC_REVISION_11, + :SDK_VECTOR_SEARCH_BASE64, + :SDK_ZONE_AWARE_READ_FROM_REPLICA, + :SDK_BUCKET_SETTINGS_NUM_VBUCKETS, + :SDK_PREFILTER_VECTOR_SEARCH, + :SDK_APP_TELEMETRY, + :SDK_SET_AUTHENTICATOR, + :SDK_JWT, + :SDK_OBSERVABILITY_CLUSTER_LABELS, + :SDK_OBSERVABILITY_RFC_REV_24, + :SDK_STABLE_OTEL_SEMANTIC_CONVENTIONS, + :SDK_STABLE_OTEL_SEMANTIC_CONVENTIONS_EMITTED_BY_DEFAULT, + ].freeze + + # We don't currently support transactions. However, the driver calls transactions_factory_create during the + # non-transactional observability tests if we don't declare EXT_SDK_INTEGRATION. So, let's declare it here until + # that issue is fixed in the driver. + TRANSACTION_CAPS = [ + :EXT_SDK_INTEGRATION, + ].freeze + + # rubocop:enable Naming/VariableNumber + + SDK_VERSION = Couchbase::VERSION[:sdk] + + def initialize + super + @logger = Logger.new($stdout) + @connections = Connections.new + @stream_owner = Streaming::StreamOwner.new + @span_owner = Observability::SpanOwner.new + @logger.info("Using Ruby SDK version #{SDK_VERSION}") + end + + def performer_caps_fetch(_request, _call) + @logger.info("performer_caps_fetch called") + resp = FIT::Protocol::Performer::PerformerCapsFetchResponse.new( + transaction_implementations_caps: TRANSACTION_CAPS, + sdk_implementation_caps: SDK_CAPS, + performer_caps: PERFORMER_CAPS, + performer_user_agent: "ruby", + library_version: SDK_VERSION, + supported_apis: [:DEFAULT], + ) + @logger.info("Reporting caps #{resp}") + resp + end + + def cluster_connection_create(request, _call) + @logger.info("cluster_connection_create called") + @connections.create_connection(request) + end + + def cluster_connection_close(request, _call) + @logger.info("cluster_connection_close called") + @connections.close_connection(request) + end + + def disconnect_connections(request, _call) + @logger.info("disconnect_connections called") + @connections.disconnect_connections(request) + end + + def echo(request, _call) + @logger.info("====== #{request.testName}: #{request.message} ======") + FIT::Protocol::Shared::EchoResponse.new + end + + def run(request, _call) + @logger.info("run called") + + # Generate a random run ID - used for identifying streams started from this run + run_id = SecureRandom.uuid + + request_type = request.request + case request_type + when :workloads + begin + connection = @connections[request.workloads.cluster_connection_id] + workload_executor = MultiThreadExecutor.build_executor( + request, connection, run_id, @stream_owner, @span_owner + ) + @logger.info("Created the workload executor") + req_executor = RequestExecutor.build_request(workload_executor, request) + @logger.info("Created the request executor") + req_executor.execute_request + @logger.info("Executing the request") + req_executor.results + rescue PerformerError => e + raise GRPC::Unimplemented, e.message + rescue StandardError => e + @logger.error(e.message) + @logger.error(e.backtrace.join("\n")) + raise e + end + else + raise GRPC::Unimplemented, "Run request type #{request_type} not supported" + end + end + + def stream_cancel(request, _call) + @stream_owner.cancel(request) + FIT::Protocol::Streams::CancelResponse.new + end + + def stream_request_items(request, _call) + @stream_owner.request_items(request) + FIT::Protocol::Streams::RequestItemsResponse.new + end + + def span_create(request, _call) + tracer = @connections[request.cluster_connection_id].tracer + @span_owner.create_span(tracer, request) + FIT::Protocol::Observability::SpanCreateResponse.new + rescue PerformerError => e + raise GRPC::FailedPrecondition, e.message + end + + def span_finish(request, _call) + @span_owner.finish_span(request) + FIT::Protocol::Observability::SpanFinishResponse.new + rescue PerformerError => e + raise GRPC::FailedPrecondition, e.message + end + end + end +end diff --git a/fit-performer/lib/fit/performer/streaming.rb b/fit-performer/lib/fit/performer/streaming.rb new file mode 100644 index 00000000..0c069635 --- /dev/null +++ b/fit-performer/lib/fit/performer/streaming.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative 'streaming/stream' +require_relative 'streaming/stream_owner' + +module FIT + module Performer + module Streaming + end + end +end diff --git a/fit-performer/lib/fit/performer/streaming/stream.rb b/fit-performer/lib/fit/performer/streaming/stream.rb new file mode 100644 index 00000000..177e1471 --- /dev/null +++ b/fit-performer/lib/fit/performer/streaming/stream.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "fit/protocol/run.top_level_pb" +require "fit/protocol/streams.top_level_pb" +require "fit/protocol/shared.exceptions_pb" + +module FIT + module Performer + module Streaming + class Stream + attr_reader :run_id + attr_reader :stream_id + + def initialize(run_id:, results:, stream_type:, stream_id:, on_demand:, enumerator:) + @logger = Logger.new($stdout) + @run_id = run_id + @results = results + @stream_type = stream_type + @stream_id = stream_id + @on_demand = on_demand + @enumerator = enumerator + @stop = false + @thread = nil + @requested_item_count = 0 + @requested_item_count_semaphore = Mutex.new + end + + def start + @logger.info("Starting stream #{@stream_id}") + @thread = Thread.new do # rubocop:disable ThreadSafety/NewThread + send_created_signal + run + end + end + + def cancel + @stop = true + end + + def wait + @thread&.join + end + + def request_items(num_items) + @requested_item_count_semaphore.synchronize do + @requested_item_count += num_items + end + end + + def self.build_stream(run_id:, results:, sdk_command:, enumerator:) + stream_strategy = sdk_command.stream_config.stream_when + on_demand = + if stream_strategy == :automatically + false + elsif stream_strategy == :on_demand + true + else + raise PerformerError, "Stream strategy #{stream_strategy} not supported" + end + + Stream.new( + run_id: run_id, + results: results, + stream_type: sdk_command.stream_type, + stream_id: sdk_command.stream_config.stream_id, + on_demand: on_demand, + enumerator: enumerator, + ) + end + + private + + def run + @logger.info("Running stream #{@stream_id}") + loop do + if @stop + send_cancelled_signal + return + end + + if @on_demand && @requested_item_count <= 0 + sleep(0.1) + next + end + + begin + result = @enumerator.next + rescue StopIteration + send_complete_signal + return + end + + send_error_signal(result) if result.is_a?(FIT::Protocol::Shared::Exception) + @results << result + + next unless @on_demand + + @requested_item_count_semaphore.synchronize do + @requested_item_count -= 1 + end + end + end + + def send_created_signal + @results.push( + FIT::Protocol::Run::Result.new( + stream: FIT::Protocol::Streams::Signal.new( + created: FIT::Protocol::Streams::Created.new(stream_id: @stream_id, type: @stream_type), + ), + ), + ) + end + + def send_cancelled_signal + @results.push( + FIT::Protocol::Run::Result.new( + stream: FIT::Protocol::Streams::Signal.new( + cancelled: FIT::Protocol::Streams::Cancelled.new(stream_id: @stream_id), + ), + ), + ) + end + + def send_complete_signal + @results.push( + FIT::Protocol::Run::Result.new( + stream: FIT::Protocol::Streams::Signal.new( + complete: FIT::Protocol::Streams::Complete.new(stream_id: @stream_id), + ), + ), + ) + end + + def send_error_signal(proto_exception) + @logger.error("Sending error signal #{proto_exception.to_h}") + @results.push( + FIT::Protocol::Run::Result.new( + stream: FIT::Protocol::Streams::Signal.new( + error: FIT::Protocol::Streams::Error.new(stream_id: @stream_id, exception: proto_exception), + ), + ), + ) + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/streaming/stream_owner.rb b/fit-performer/lib/fit/performer/streaming/stream_owner.rb new file mode 100644 index 00000000..89feb248 --- /dev/null +++ b/fit-performer/lib/fit/performer/streaming/stream_owner.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module FIT + module Performer + module Streaming + class StreamOwner + def initialize + @logger = Logger.new($stdout) + @streams = {} + end + + def cancel(request) + @streams[request.stream_id].cancel + @streams.delete(request.stream_id) + end + + def request_items(request) + @streams[request.stream_id].request_items(request.num_items) + end + + def wait_for_all_streams_from_run(run_id) + @streams.each_value do |stream| + stream.wait if stream.run_id == run_id + end + @logger.info("All streams from run #{run_id} have finished") + @streams.delete_if { |_key, stream| stream.run_id == run_id } + end + + def register_and_start_stream(stream) + stream_id = stream.stream_id + + raise PerformerError, "Stream #{stream_id} already exists" if @streams.include?(stream_id) + + @logger.info("Registering stream #{stream_id}") + @streams[stream_id] = stream + stream.start + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/version.rb b/fit-performer/lib/fit/performer/version.rb new file mode 100644 index 00000000..74a4bae2 --- /dev/null +++ b/fit-performer/lib/fit/performer/version.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module FIT + module Performer + VERSION = "0.1.0" + end +end diff --git a/fit-performer/lib/fit/performer/workloads.rb b/fit-performer/lib/fit/performer/workloads.rb new file mode 100644 index 00000000..78106d2f --- /dev/null +++ b/fit-performer/lib/fit/performer/workloads.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fit/performer/workloads/sdk_workload' +require 'fit/performer/performer_error' + +module FIT + module Performer + module Workloads + def self.build_workload(raw_workload, connection, run_id, stream_owner, span_owner) + workload_type = raw_workload.workload + case workload_type + when :sdk + SdkWorkload.new(raw_workload.sdk, connection, run_id, stream_owner, span_owner) + else + raise PerformerError, "Workload type `#{workload_type}` not supported" + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/workloads/base_workload.rb b/fit-performer/lib/fit/performer/workloads/base_workload.rb new file mode 100644 index 00000000..334ba754 --- /dev/null +++ b/fit-performer/lib/fit/performer/workloads/base_workload.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'fit/performer/performer_error' + +module FIT + module Performer + module Workloads + class BaseWorkload + def initialize(raw_workload, connection, run_id, stream_owner, span_owner) + @logger = Logger.new($stdout) + @raw_workload = raw_workload + @connection = connection + @run_id = run_id + @stream_owner = stream_owner + @span_owner = span_owner + @command_count = @raw_workload.command.size + + # Bounds-related attributes + @counter_id = nil + @deadline = nil + @remaining_command_count = nil + end + + def execute(_results, _global_counters, _global_semaphore) + raise "`execute` method must be implemented" + end + + def set_bounds(global_counters, global_semaphore) + unless @raw_workload.has_bounds? + @remaining_command_count = @command_count + return + end + + raw_bounds = @raw_workload.bounds + case raw_bounds.bounds + when :counter + raw_counter = raw_bounds.counter + @counter_id = raw_counter.counter_id + + case raw_counter.counter + when :global + global_semaphore.synchronize do + unless global_counters.include?(@counter_id) + initial_count = raw_counter.global.count + global_counters[@counter_id] = initial_count + end + end + else + raise PerformerError, "Counter type `#{counter.counter}` not recognised" + end + when :for_time + @deadline = Time.now + raw_bounds.for_time.seconds + else + raise PerformerError, "Bounds type `#{@raw_workload.bounds}` not recognised" + end + end + + def within_bounds(global_counters, global_semaphore) + if !@counter_id.nil? + global_semaphore.synchronize do + global_counters[@counter_id] -= 1 + global_counters[@counter_id] >= 0 + end + elsif !@deadline.nil? + Time.now < @deadline + elsif !@remaining_command_count.nil? + @remaining_command_count -= 1 + @remaining_command_count >= 0 + else + raise PerformerError, "Bounds have not been set" + end + end + + private + + def execute_command(_raw_command, _executed_cmd_count, _results) + raise '`execute_command` method must be implemented' + end + end + end + end +end diff --git a/fit-performer/lib/fit/performer/workloads/sdk_workload.rb b/fit-performer/lib/fit/performer/workloads/sdk_workload.rb new file mode 100644 index 00000000..d8ae199f --- /dev/null +++ b/fit-performer/lib/fit/performer/workloads/sdk_workload.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative '../commands' +require_relative '../streaming' +require_relative 'base_workload' + +module FIT + module Performer + module Workloads + class SdkWorkload < BaseWorkload + def initialize(raw_workload, connection, run_id, stream_owner, span_owner) + super + @logger = Logger.new($stdout) + end + + def execute(results, global_counters, global_semaphore) + executed_cmd_count = 0 + while within_bounds(global_counters, global_semaphore) + raw_cmd = @raw_workload.command[executed_cmd_count % @command_count] + execute_command(raw_cmd, executed_cmd_count, results) + executed_cmd_count += 1 + end + end + + def execute_command(raw_command, executed_cmd_count, results) + command = Commands.build_command(raw_command, @connection, @span_owner, executed_cmd_count) + result = command.execute_command + + if result.is_a?(Enumerator) + @stream_owner.register_and_start_stream( + Streaming::Stream.build_stream( + run_id: @run_id, + results: results, + sdk_command: command, + enumerator: result, + ), + ) + else + results.push(result) + end + end + end + end + end +end diff --git a/fit-performer/lib/fit/protocol.rb b/fit-performer/lib/fit/protocol.rb new file mode 100644 index 00000000..00f34993 --- /dev/null +++ b/fit-performer/lib/fit/protocol.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module FIT + module Protocol + end +end diff --git a/fit-performer/proto/README.md b/fit-performer/proto/README.md new file mode 100644 index 00000000..5bcd75d1 --- /dev/null +++ b/fit-performer/proto/README.md @@ -0,0 +1,97 @@ +# GRPC format +References: +* https://developers.google.com/protocol-buffers/docs/proto3 + +## Rules for Performers +The 'golden rule' is that the performer is meant to be a dumb-as-rocks passthrough agent. If there is something in the +FIT GRPC that it cannot handle, please raise it with SDKQE to explore options, rather than trying to interpret or +manipulate it into something it or the SDK can use. With that in mind: + +1. The performer should pass all fields to the SDK directly. + For instance, some performers were prepending "couchbase://" to the connection string, which a) masked an SDK bug and + b) caused problems when the driver did start sending the connection string. + If passing fields to the SDK causes an issue then raise with SDKQE to explore options. + +2. If the performer gets an RPC or parameter that either it or the SDK can't do anything with, or doesn't recognise, then + raise with SDKQE to explore options. If they find there a reasonable case for the SDK not being able to handle + something (remember that we want all SDKs to be consistent), then the performer should raise UNSUPPORTED if it receives it. + This may require appropriate driver-side logic to be added. + The performer should also raise UNSUPPORTED as the default on any `oneof` handling. + +3. Many fields are `optional`, especially configuration ones. These help test default options. If a configuration block + is optional and not provided, then the performer should call the appropriate no-option overload - e.g. + `cluster.query(statement)` rather than `cluster.query(statement, options)`. + +## Packages +The following rules are used for packages: + +1. A flat directory structure is used so performers can pull in all protobuf files easily. +2. As a replacement for a directory structure, the filename includes the package name. E.g. `sdk.kv.options.proto` which is in package `protocol.sdk.kv`. +3. The "protocol.shared" package is used for all shared/base code: anything that could be shared between SDK and transactions, for instance. +4. Otherwise aim to put new GRPC into a package, or create a package. Most will go under `sdk`, e.g. `sdk.query`. +5. Avoid import package cycles (see below) to allow Go to compile. +6. We use 'sdk.kv.Get' naming, rather than say 'sdk.kv.SdkKvGet'. + 6.1. One exception to this are transactions, e.g. `transactions.TransactionResult`. These messages existed prior to this rule, and to reduce breakage, they were not renamed. +7. Don't have filenames that match message/enum names. E.g. "PerformerCaps" enum exists, so can't have "performer_caps.proto". It produces non-compilable Java. +8. For simplicity we keep a one-to-one mapping between the GRPC package and the generated packages. E.g.: + ``` + package protocol.sdk.kv; + option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Kv"; + option java_package = "com.couchbase.client.performer.protocol.sdk.kv"; + ``` +9. With Serverless it's becoming increasingly common to need similar interfaces at the Cluster, Bucket Scope and/or Collection level. + To help, the new package naming scheme is `sdk.[cluster/scope/collection]...` + More broadly, the idea now is that the right-most part of the package identifies broadly what it is, with each preceding + part of the package revealing more and more specificity. + Taking example `sdk.cluster.query.index_manager`: the right-most part identifies the broad strokes, e.g. that this is + an IndexManager. What type of IndexManager is it? A QueryIndexManager. What level is it at? Cluster. + We're following the precedent set in the SDK here, e.g. CollectionQueryIndexManager. + Where there is a Cluster and Scope/Collection version of the same interface, they will often share some messages. + The convention here is EITHER create 3 files with packages "sdk.cluster.search.index_manager", "sdk.scope.search.index_manager" + and "sdk.search.index_manager" (for the shared messages). + OR just one package "sdk.search". + The approach selected should depend on how complex the interface is (e.g. does it have multiple methods), how much + it differs from Cluster/Bucket vs Scope/Collection forms, and whether that could change in future. + +## Columnar +// todo tidyup +1. Continues to apply. +2. Probably tweaking. +3. Continues to apply. "protocol.shared" package now used for everything that applies to both SDKs. +4. Probably tweaking. +5. Continues to apply. +6. Continues to apply, but we are using much shorter package names following feedback. +7. Continues to apply. +8. Probably Continues to apply. +10. Follow standard industry conventions: + - Protobuf best practices + - https://protobuf.dev/programming-guides/api + - https://protobuf.dev/programming-guides/dos-donts/ + - (Note Google's GRPC errors (https://cloud.google.com/apis/design/errors) AREN'T followed, as we need more + flexibility to express exactly what the SDK returned) + + +## Import package cycles +A Go-specific issue exists that impacted the design: + +``` +a.proto: +package protocol; +import "c.proto"; + +b.proto: +package protocol; + +c.proto: +package protocol.sdk; +import "b.proto"; +``` + +Go cannot compile this setup as c.proto is importing from the top-level package again, and Go sees this as an import cycle. + +## Java Specifics +This will produce one Java class for each enum & message, in the com.couchbase.grpc.protocol package. Default +behaviour is to wrap those classes in a class named after this .proto file, which makes refactoring hard. +``` +option java_multiple_files = true; +``` diff --git a/fit-performer/proto/hooks.transactions.proto b/fit-performer/proto/hooks.transactions.proto new file mode 100644 index 00000000..e7c75534 --- /dev/null +++ b/fit-performer/proto/hooks.transactions.proto @@ -0,0 +1,204 @@ +syntax = "proto3"; + +// hooks.transactions rather than transactions.hooks, to avoid Go circular imports on SingleQueryTransactionOptions +package protocol.hooks.transactions; +option csharp_namespace = "Couchbase.Grpc.Protocol.Hooks.Transactions"; +option java_package = "com.couchbase.client.protocol.hooks.transactions"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/hooks/transactions"; +option ruby_package = "FIT::Protocol::Hooks::Transactions"; +option java_multiple_files = true; + +// Generally the BEFORE_ hook is used to inject an error, e.g. as if the server had returned a failure +// The AFTER_ hook is used for ambiguity testing. E.g. the op succeeded but FAIL_AMBIGUOUS was returned. +enum HookPoint { + BEFORE_ATR_COMMIT = 0; + AFTER_ATR_COMMIT = 1; + BEFORE_DOC_COMMITTED = 2; + BEFORE_DOC_ROLLED_BACK = 3; + AFTER_DOC_COMMITTED_BEFORE_SAVING_CAS = 4; + AFTER_DOC_COMMITTED = 5; + BEFORE_DOC_REMOVED = 6; + AFTER_DOC_REMOVED_PRE_RETRY = 7; + AFTER_DOC_REMOVED_POST_RETRY = 8; + AFTER_DOCS_REMOVED = 9; + BEFORE_ATR_PENDING = 10; + AFTER_ATR_COMPLETE = 11; + BEFORE_ATR_ROLLED_BACK = 12; + AFTER_GET_COMPLETE = 13; + BEFORE_ROLLBACK_DELETE_INSERTED = 15; + AFTER_STAGED_REPLACE_COMPLETE = 16; + AFTER_STAGED_REMOVE_COMPLETE = 17; + BEFORE_STAGED_INSERT = 18; + BEFORE_STAGED_REMOVE = 19; + BEFORE_STAGED_REPLACE = 20; + AFTER_STAGED_INSERT_COMPLETE = 21; + BEFORE_GET_ATR_FOR_ABORT = 22; + BEFORE_ATR_ABORTED = 23; + AFTER_ATR_ABORTED = 24; + AFTER_ATR_ROLLED_BACK = 25; + AFTER_ROLLBACK_REPLACE_OR_REMOVE = 26; + AFTER_ROLLBACK_DELETE_INSERTED = 27; + BEFORE_REMOVING_DOC_DURING_STAGING_INSERT = 28; + BEFORE_CHECK_ATR_ENTRY_FOR_BLOCKING_DOC = 29; + BEFORE_DOC_GET = 30; + BEFORE_GET_DOC_IN_EXISTS_DURING_STAGED_INSERT = 31; + AFTER_ATR_PENDING = 32; + BEFORE_ATR_COMPLETE = 33; + BEFORE_ATR_COMMIT_AMBIGUITY_RESOLUTION = 36; + BEFORE_QUERY = 37; + BEFORE_REMOVE_STAGED_INSERT = 38; + AFTER_REMOVE_STAGED_INSERT = 39; + AFTER_QUERY = 40; + BEFORE_DOC_CHANGED_DURING_COMMIT = 41; + // The BEFORE_UNLOCK_ hooks are deprecated, removed from ExtThreadSafety, and FIT will not send them. Performers + // do not need to implement them, and can remove them. + BEFORE_UNLOCK_GET = 42; + BEFORE_UNLOCK_INSERT = 43; + BEFORE_UNLOCK_REPLACE = 44; + BEFORE_UNLOCK_REMOVE = 45; + BEFORE_UNLOCK_QUERY = 46; + BEFORE_DOC_CHANGED_DURING_ROLLBACK = 47; + BEFORE_DOC_CHANGED_DURING_STAGING = 48; + + // Injects that the transaction has expired at a certain point + // Does not take a HookAction + // hookConditionParam2 determines the hook point + HAS_EXPIRED = 34; + + // Overrides the ATR id chosen for a given vbucket + ATR_ID_FOR_VBUCKET = 35; + + // Cleanup hooks + CLEANUP_BEFORE_COMMIT_DOC = 101; + CLEANUP_BEFORE_REMOVE_DOC_STAGED_FOR_REMOVAL = 102; + CLEANUP_BEFORE_DOC_GET = 103; + CLEANUP_BEFORE_REMOVE_DOC = 104; + CLEANUP_BEFORE_REMOVE_DOC_LINKS = 105; + CLEANUP_BEFORE_ATR_REMOVE = 106; + + CLEANUP_MARKER_LAST = 199; + + // Client record hooks + CLIENT_RECORD_BEFORE_UPDATE_CAS = 201; // No longer exists as of TXNJ-274 + CLIENT_RECORD_BEFORE_CREATE = 202; + CLIENT_RECORD_BEFORE_GET = 203; + CLIENT_RECORD_BEFORE_UPDATE = 204; + CLIENT_RECORD_BEFORE_REMOVE_CLIENT = 205; + + CLIENT_RECORD_MARKER_LAST = 299; +} + +enum HookCondition { + ALWAYS = 0; + + // Tracks each call to the hook, and triggers it on a particular run. + // Set which run with hookConditionParam1. + // Call count is 1-based indexing. + ON_CALL = 1; + + // Use is HookPoint specific, but generally it's whether some variable + // equals hookConditionParam1 or hookConditionParam2 + EQUALS = 2; + + // Use is HookPoint specific, and only used for HookPoint HAS_EXPIRED, which needs to check whether both + // docId==hookConditionParam2 and hookStage==hookConditionParam3 + EQUALS_BOTH = 3; + + // Combines ON_CALL and EQUALS. + // Note that this tracks how many calls have been made to this hook _using a given param_. + ON_CALL_AND_EQUALS = 4; + + // Similar to ON_CALL but triggers for all attempts less than or equal to hookConditionParam1 (1-indexed) + ON_CALL_LE = 5; + + // Similar to ON_CALL but triggers for all attempts greater than or equal to hookConditionParam1 (1-indexed) + ON_CALL_GE = 6; + + // Executes the hook as long as the transaction has not expired. + WHILE_NOT_EXPIRED = 7; + + // Executes the hook while the transaction is expired. + WHILE_EXPIRED = 8; +} + +enum HookAction { + // Fail the transaction immediately, make no attempt to roll it back, app gets TransactionFailed exception + // Txn will be left in whatever state it reached. This is not a real-world exception that would ever be + // voluntarily raised in the transaction library. It is designed to simulate an application hard crashing, or + // similar. + // In the real-world these should be fairly rare cases, so this shouldn't be the first error class reached for + // by a test, but it is important to check them. + FAIL_HARD = 0; + + // This simulates any exception that doesn't fall into one of the other FAIL_ categories. + FAIL_OTHER = 1; + + // Simulates a failure that is likely to be transient, such as a SDK returning an unambiguous TimeoutException as + // server repeatedly returning a TEMP_FAIL + FAIL_TRANSIENT = 2; + + // Inject an error indicating that operation was ambiguously successful. + // Some real-world examples that would cause this: + // Server returning ERR_AMBIG durable write error + // SDK returning an ambiguous TimeoutException, e.g. the server did not respond on a mutation + FAIL_AMBIGUOUS = 3; + + // Inject an error that the doc already existed. + FAIL_DOC_ALREADY_EXISTS = 4; + + // Inject an error that the doc was not found. + FAIL_DOC_NOT_FOUND = 5; + + // Inject an error that the path already exists. + FAIL_PATH_ALREADY_EXISTS = 10; + + // Inject an error that the path was not found. + FAIL_PATH_NOT_FOUND = 6; + + // Inject an error that there was a CAS mismatch.. + FAIL_CAS_MISMATCH = 7; + + FAIL_WRITE_WRITE_CONFLICT = 8; + + FAIL_ATR_FULL = 9; + + // Mutate the doc using a regular KV SET. + // hookActionParam1 - the doc's id is sent in hookActionParam1, in format + // "bucket-name/collection-name/doc-id". e.g. + // "my-bucket/_default/some-doc-id" + // hookActionParam2 - the JSON to write + MUTATE_DOC = 20; + + // Remove the doc using a regular KV op. + // hookActionParam1 - the doc's id is sent in hookActionParam1, in format + // "bucket-name/collection-name/doc-id". e.g. + // "my-bucket/_default/some-doc-id" + REMOVE_DOC = 21; + + // Just return value in hookActionParam1 + RETURN_STRING = 22; + + // Add a blocking wait. The wait is hookActionParam1, converted to an int, in millis. + BLOCK = 23; +} + +// Hooks allow injecting actions at specific points of the code. Useful for error injection. +message Hook { + // At which point in the code should the hook execute + HookPoint hook_point = 1; + + // What conditions activate the hook + HookCondition hook_condition = 2; + int32 hook_condition_param1 = 3; + string hook_condition_param2 = 4; + // Only used for EQUALS_BOTH + string hook_condition_param3 = 5; + + // What should the hook do + HookAction hook_action = 6; + + // Parameterise the hook's action - see HookAction for how these apply + string hook_action_param1 = 7; + string hook_action_param2 = 8; +} + diff --git a/fit-performer/proto/meta.workload.proto b/fit-performer/proto/meta.workload.proto new file mode 100644 index 00000000..139ab6c6 --- /dev/null +++ b/fit-performer/proto/meta.workload.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +// Can't use protocol.grpc as it causes ambiguity issues for C++ (a number of GRPC library headers use a grpc namespace), +// so meta instead. +package protocol.meta; +option csharp_namespace = "Couchbase.Grpc.Protocol.Meta"; +option java_package = "com.couchbase.client.protocol.meta"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/meta"; +option ruby_package = "FIT::Protocol::Meta"; +option java_multiple_files = true; + +import "shared.bounds.proto"; + +message Ping { +} + +message Command { + oneof command { + Ping ping = 1; + } +} + +// Used for meta purposes such as testing GRPC workflow. Only sent if performer declares support for PerformerCaps.GRPC_TESTING. +message Workload { + // The command to run. The performer should execute this in a loop until `bounds` is completed. + Command command = 1; + + // Controls how the commands should be run. + shared.Bounds bounds = 2; +} + +message PingResult { +} + +message Result { + oneof result { + PingResult ping_result = 1; + } +} diff --git a/fit-performer/proto/metrics.top_level.proto b/fit-performer/proto/metrics.top_level.proto new file mode 100644 index 00000000..ab7ba1e4 --- /dev/null +++ b/fit-performer/proto/metrics.top_level.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package protocol.metrics; +option csharp_namespace = "Couchbase.Grpc.Protocol.Metrics"; +option java_package = "com.couchbase.client.protocol.metrics"; +option java_multiple_files = true; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/metrics"; +option ruby_package = "FIT::Protocol::Metrics"; + +import "google/protobuf/timestamp.proto"; + +message Result { + // A JSON blob that can contain any information the performer likes. '{"cpu":83}' for example. It will be written + // directly into the database. + // The performer can return whatever fields it wants, named whatever it wants (with some consideration given to database + // size). But if it is returning any of the following it should use these standardised field names: + // memHeapUsedMB: how much heap memory has been used by the performer+SDK, in MB + // processCpu: how much CPU (total, across all cores) the performer+SDK is using + // systemCpu: how much CPU (total, across all cores) the whole system is using + // threadCount: how many threads the performer+SDK are using + string metrics = 1; + google.protobuf.Timestamp initiated = 2; +} diff --git a/fit-performer/proto/observability.config.proto b/fit-performer/proto/observability.config.proto new file mode 100644 index 00000000..d634e415 --- /dev/null +++ b/fit-performer/proto/observability.config.proto @@ -0,0 +1,101 @@ +syntax = "proto3"; + +package protocol.observability; +option csharp_namespace = "Couchbase.Grpc.Protocol.Observability"; +option java_package = "com.couchbase.client.protocol.observability"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/observability"; +option ruby_package = "FIT::Protocol::Observability"; +option java_multiple_files = true; + +import "observability.top.proto"; + +// Configures the performer to output OpenTelemetry tracing data. +message TracingConfig { + // Where to send the OTLP traffic. + string endpoint_hostname = 1; + + // [0-1]: what percentage of the traces emitted should be sent to the endpoint. + // The performer should use an epsilon based check, and if this value ~= 0, set it to never output, and if ~= 1, always output. + float sampling_percentage = 2; + + // Whether the performer should enable batching of the traces. + bool batching = 3; + + // How frequently to export data, as a best effort. This lets us tune for quick observability during integration + // tests, and reducing traffic and overhead in performance tests. + int32 export_every_millis = 4; + + // Attributes to provide to the TracerProvider. While this is under-documented in OpenTelemetry, the intent here is + // to provide fields such as "service.name". + map resources = 5; +} + +// Configures the performer to output OpenTelemetry metrics data. +message MetricsConfig { + // Where to send the OTLP metrics traffic. + string endpoint_hostname = 1; + + // How frequently to export data, as a best effort. This lets us tune for quick observability during integration + // tests, and reducing traffic and overhead in performance tests. + int32 export_every_millis = 2; + + // N.b. there is no sampling and batching in the MetricsConfig, unlike the TracingConfig, as at least the Java OTel + // SDK does not support them. + + // Attributes to provide to the MeterProvider. While this is under-documented in OpenTelemetry, the intent here is + // to provide fields such as "service.name". + map resources = 3; +} + +message ThresholdLoggingTracerConfig { + optional int32 emit_interval_millis = 1; + optional int32 kv_threshold_millis = 2; + optional int32 query_threshold_millis = 3; + optional int32 views_threshold_millis = 4; + optional int32 search_threshold_millis = 5; + optional int32 analytics_threshold_millis = 6; + optional int32 transactions_threshold_millis = 7; + optional int32 sample_size = 8; + optional bool enabled = 9; +} + +message LoggingMeterConfig { + optional int32 emit_interval_millis = 1; + optional bool enabled = 2; +} + +message OrphanResponseConfig { + optional int32 emit_interval_millis = 1; + optional int32 sample_size = 2; + optional bool enabled = 3; +} + +enum SemanticConvention { + DATABASE = 0; + DATABASE_DUP = 1; +} + +message Config { + // If not sent, the performer should not enable tracing output. + optional TracingConfig tracing = 1; + + // If set, the performer should set a NoopTracer as the requestTracer. + // The driver will not send `tracing` config if this is set. + bool use_noop_tracer = 2; + + // If not sent, the performer should not enable metrics output. + optional MetricsConfig metrics = 3; + + optional ThresholdLoggingTracerConfig threshold_logging_tracer = 4; + + optional LoggingMeterConfig logging_meter = 5; + + optional OrphanResponseConfig orphan_response = 6; + + // If sent, the performer should set the corresponding ClusterOptions field. + // Will only be sent iff the performer declares support for SDK_STABLE_OTEL_SEMANTIC_CONVENTIONS, + // but not SDK_STABLE_OTEL_SEMANTIC_CONVENTIONS_EMITTED_BY_DEFAULT + repeated SemanticConvention observability_semantic_convention_opt_in = 7; +} + + diff --git a/fit-performer/proto/observability.top.proto b/fit-performer/proto/observability.top.proto new file mode 100644 index 00000000..574c7d2e --- /dev/null +++ b/fit-performer/proto/observability.top.proto @@ -0,0 +1,52 @@ +syntax = "proto3"; + +package protocol.observability; +option csharp_namespace = "Couchbase.Grpc.Protocol.Observability"; +option java_package = "com.couchbase.client.protocol.observability"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/observability"; +option ruby_package = "FIT::Protocol::Observability"; +option java_multiple_files = true; + +// The performer creates and starts a new RequestSpan, via the RequestTracer associated with a particular +// previously-created Cluster. +// The performer needs to hold onto this in a global Map, with the key being the provided `id`. +message SpanCreateRequest { + // The performer can assume it's a driver bug if this references a non-existent Cluster, and throw an error. + string cluster_connection_id = 1; + + // The span is created with this name. + string name = 2; + + // This is used to reference the RequestSpan later. + string id = 3; + + // References a previously created RequestSpan to supply as a parent. + // If it's not present then don't provide a parent. + // The performer can assume it's a driver bug if this references a non-existent span, and throw an error. + optional string parent_span_id = 4; + + // Attributes to set on the span. + map attributes = 5; +} + +// Empty but included anyway per GRPC best practices. +message SpanCreateResponse { +} + +// The performer finishes a span and removes it from its map. +// The performer can assume it's a driver bug if this references a non-existent span, and throw an error. +message SpanFinishRequest { + string id = 1; +} + +// Empty but included anyway per GRPC best practices. +message SpanFinishResponse { +} + +message Attribute { + oneof value { + string value_string = 2; + int64 value_long = 3; + bool value_boolean = 4; + } +} \ No newline at end of file diff --git a/fit-performer/proto/performer.caps.proto b/fit-performer/proto/performer.caps.proto new file mode 100644 index 00000000..4eebc5ab --- /dev/null +++ b/fit-performer/proto/performer.caps.proto @@ -0,0 +1,121 @@ +syntax = "proto3"; + +package protocol.performer; +option csharp_namespace = "Couchbase.Grpc.Protocol.Performer"; +option java_package = "com.couchbase.client.protocol.performer"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/performer"; +option ruby_package = "FIT::Protocol::Performer"; +option java_multiple_files = true; + +import "transactions.extensions.proto"; +import "shared.basic.proto"; +import "sdk.caps.proto"; + +// Capabilities of the performer itself (not what it's testing). +enum Caps { + // The performer supports GRPC workloads. It's optional, but allows various GRPC streaming approaches to be tested + // with the performer, which can help find one that reliably lets results be streamed back. + GRPC_TESTING = 0; + + // Level 1 KV support: the initial KV commands (get, insert, replace, remove, upsert) together with options. + KV_SUPPORT_1 = 1; + + // Whether this performer has level 1 transactions support, e.g. it has implemented protocol 1.0+. + // It includes the top-level transactions rpcs: transactionCreate, closeTransactions, transactionStream, transactionCleanup + // transactionCleanupATR, cleanupSetFetch, clientRecordProcess. + // It does not include transactions.Workload - that is covered by TRANSACTIONS_WORKLOAD_1. + TRANSACTIONS_SUPPORT_1 = 2; + + // The performer supports all parameters in ClusterConfig up to idleHttpConnectionTimeoutSecs, 23. + CLUSTER_CONFIG_1 = 3; + + // Support `ClusterConfig.cert`. + CLUSTER_CONFIG_CERT = 7; + + // Support `ClusterConfig.insecure`. + CLUSTER_CONFIG_INSECURE = 8; + + // Whether this performer supports transactions.Workload, allowing it to do performance testing of transactions. + // All GRPC messages transitively required by this workload must also be supported. + // The exception here is TransactionOptions.hooks: these can be significant work for an implementation, and are + // not (initially at least) required for transactions performance testing. + // The performer also needs to support the new commands: transactions.Insert,Replace,Remove,Get. + // CLUSTER_CONFIG_1 must be supported. + TRANSACTIONS_WORKLOAD_1 = 4; + + // Support for requesting the performer output observability (OpenTelemetry) data. + // The performer must support: + // * ClusterConfig.observabilityConfig and everything under it. + // * SpanCreateRequest and SpanFinishRequest, and everywhere they are used. + // * parentSpanId on all messages the performer also supports. This is a UUID corresponding to a previous SpanCreateRequest. + // The performer can assume it is a bug if it does not know about this span or it has been previously finished, and + // throw an error. + // * spanCreate and spanFinish RPCs. + // The SDK must support everything in the observability SDK-RFC revision #24, including: + // * RequestSpan.recordException + // * Counter support + OBSERVABILITY_1 = 6; + + // The performer will set `elapsedNanos` for failed operations as well as successful ones. + TIMING_ON_FAILED_OPS = 9; + + // The performer supports sending and receiving values of up to 25MiB. + LARGE_VALUES = 10; + + // If the performer supports ContentAsPerformerValidation. + CONTENT_AS_PERFORMER_VALIDATION = 11; + + /** + * The performer supports custom serialization logic for search options ie .serialize(CustomSerializer). + * This capability includes the ability to parse results custom serializer implemented in respective performers. + * The custom serializer should adhere to the following logic: + 1. During serialization: + - Convert the input object to a JSON string. + - Add a "Serialized": true field to the JSON object. + - Return the JSON object as a byte array. + 2. During deserialization: + - Parse the byte array into a JSON object. + - Upsert a "Serialized": false field to the JSON object. + - Return the updated JSON object as the deserialized result. + */ + CUSTOM_SERIALIZATION_SUPPORT_FOR_SEARCH = 12; + + // The performer supports content types as null value. + AS_NULL_CONTENT_TYPE_SUPPORT = 13; + + // The performer supports client_context_id as Transaction Query Option. + TXN_CLIENT_CONTEXT_ID_SUPPORT = 14; +} + +message PerformerCapsFetchRequest { +} + +message PerformerCapsFetchResponse { + // The transactions capabilities of the implementation-under-test. + repeated transactions.Caps transaction_implementations_caps = 1; + + // The SDK (non-transaction) capabilities of the implementation-under-test. + repeated sdk.Caps sdk_implementation_caps = 8; + + // The capabilities of this performer. + repeated Caps performer_caps = 7; + + // Human-readable string identifying the performer. For example, "java". + // ExtSDKIntegration: performers should use a new user agent to disambiguate, such as "java" and "java-sdk". + string performer_user_agent = 2; + + // Identifies the version of the library (SDK or transactions library, as appropriate) under test. For example, "1.1.2". + string library_version = 4; + + // Defined https://hackmd.io/foGjnSSIQmqfks2lXwNp8w#Protocol-Versions + // For transactions workloads, must be "1.0", "2.0" or "2.1" any other values will cause the driver to abort. + // For non-transactions workloads, return nothing. + optional string transactions_protocol_version = 3; + + // The APIs this implementation supports. This is primarily used to run tests on multiple APIs, so if an implementation + // only supports one, it should just return one - see comments for `API` for details. + // If the performer does not return anything, it will be assumed to support just API.DEFAULT. + repeated shared.API supported_apis = 6; +} + + diff --git a/fit-performer/proto/performer.proto b/fit-performer/proto/performer.proto new file mode 100644 index 00000000..612c5e69 --- /dev/null +++ b/fit-performer/proto/performer.proto @@ -0,0 +1,164 @@ +syntax = "proto3"; + +package protocol; +option csharp_namespace = "Couchbase.Grpc.Protocol"; +option java_package = "com.couchbase.client.protocol"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol"; +option ruby_package = "FIT::Protocol"; +option java_multiple_files = true; + +import "transactions.client_record.proto"; +import "transactions.cleanup.proto"; +import "performer.caps.proto"; +import "transactions.legacy.proto"; +import "transactions.performer.proto"; +import "run.top_level.proto"; +import "shared.cluster.proto"; +import "streams.top_level.proto"; +import "shared.echo.proto"; +import "observability.top.proto"; +import "shared.bounds.proto"; + +// All rpcs related to transactions and SDK performance and integration testing. +service PerformerService { + // Requests the performer's capabilities. + // Added in ExtSDKIntegration. + // If the performer has not implemented this, the driver will gracefully degrade to using the now-legacy + // CreateConnectionRequest instead. + // Required for performers that implement RealPerformerCaps.KV_SUPPORT_1. + // If any of performerCapsFetch, clusterConnectionCreate and clusterConnectionClose is implemented, they must all + // be implemented. + rpc performerCapsFetch (performer.PerformerCapsFetchRequest) returns (performer.PerformerCapsFetchResponse); + + // Creates a Cluster connection, and optionally configures it. + // Added in ExtSDKIntegration, and will only be sent if the performer declares support for that extension or RealPerformerCaps.KV_SUPPORT_1. + // Otherwise CreateConnectionRequest will be used. + // Required for performers that implement RealPerformerCaps.KV_SUPPORT_1. + // If any of performerCapsFetch, clusterConnectionCreate and clusterConnectionClose is implemented, they must all + // be implemented. + // The performer should not call waitUntilReady as part of the cluster connection. Tests that require a reliable + // cluster connection should use the WaitUntilReady ClusterLevelCommand. + rpc clusterConnectionCreate (shared.ClusterConnectionCreateRequest) returns (shared.ClusterConnectionCreateResponse); + + // Close a particular cluster connection. + // Added in ExtSDKIntegration, and will only be sent if the performer declares support for that extension or RealPerformerCaps.KV_SUPPORT_1. + // There is no real analogue pre-ExtSDKIntegration, so all cluster connections will remain open. + // Required for performers that implement RealPerformerCaps.KV_SUPPORT_1. + // If any of performerCapsFetch, clusterConnectionCreate and clusterConnectionClose is implemented, they must all + // be implemented. + rpc clusterConnectionClose (shared.ClusterConnectionCloseRequest) returns (shared.ClusterConnectionCloseResponse); + + // Closes all cluster connections. + // Since ExtSDKIntegrated, the driver should be cleaning up all resources it creates. But this method remains useful + // to send upfront before all tests, in case the previous run did not finish cleanly. + // It can only be sent before or after all tests, to avoid destroying the default cluster connection. + // Required for performers that implement RealPerformerCaps.KV_SUPPORT_1. + rpc disconnectConnections (shared.DisconnectConnectionsRequest) returns (shared.DisconnectConnectionsResponse); + + // Creates and runs a transaction and returns the result. + // Abstracts over ExtSDKIntegration by passing both a clusterConnectionId and a transactionsFactoryRef so the + // performer can use a Cluster or Transactions object as appropriate. In ExtSDKIntegration, the transactionsFactoryRef + // should be ignored. + rpc transactionCreate (transactions.TransactionCreateRequest) returns (transactions.TransactionResult); + + // Closes all unclosed transaction factories. + // ExtSDKIntegration performers should not implement this, and the driver will gracefully handle that. + // The same comments for `disconnectConnections` apply. + rpc closeTransactions (transactions.CloseTransactionsRequest) returns (transactions.CloseTransactionsResponse); + + // Transactions that can communicate bi-directionally with the tests + // This is for more complex tests that require e.g. concurrent transactions, gating each other's progress. + rpc transactionStream (stream transactions.TransactionStreamDriverToPerformer) returns (stream transactions.TransactionStreamPerformerToDriver); + + // Performs cleanup of a single transaction. It will fetch the ATR entry first. + // This tests the internals of the cleanup algo, the part shared by the two cleanup systems. + rpc transactionCleanup (transactions.TransactionCleanupRequest) returns (transactions.TransactionCleanupAttempt); + + // Performs cleanup of a full ATR. Useful for testing multiple performers concurrently cleaning up same ATR. + // Note it will only cleanup expired entries. + rpc transactionCleanupATR (transactions.TransactionCleanupATRRequest) returns (transactions.TransactionCleanupATRResult); + + // Fetches the set of collections currently being cleaned up. + // Added in ExtSDKIntegration, and will only be sent if the performer declares support for that extension. + rpc cleanupSetFetch (transactions.CleanupSetFetchRequest) returns (transactions.CleanupSetFetchResponse); + + // Request that the implementation do its normal client record processing logic (creating CR if needed). + rpc clientRecordProcess (transactions.ClientRecordProcessRequest) returns (transactions.ClientRecordProcessResponse); + + // Perform a single query transaction. + // Added in ExtSingleQuery, and will only be sent if the performer declares support for that extension. + // Abstracts over ExtSDKIntegration by passing both a clusterConnectionId and a transactionsFactoryRef so the + // performer can use a Cluster or Transactions object as appropriate. In ExtSDKIntegration, the transactionsFactoryRef + // should be ignored. + rpc transactionSingleQuery (transactions.TransactionSingleQueryRequest) returns (transactions.TransactionSingleQueryResponse); + + // Request that the performer Echo a string to the performer logs. + // Required for performers that implement RealPerformerCaps.KV_SUPPORT_1. + rpc echo (shared.EchoRequest) returns (shared.EchoResponse); + + + // The performer will execute all run.Request items, and stream back all expected responses. + // + // It's intentionally a very generic interface and concept, as it's designed to be used for many purposes. + // Initially it's for performance and integration testing, for both transactions and SDK workloads (plus some other + // bits, such as meta-testing performance of various GRPC streaming approaches). + // + // Error handling: + // If the performer encounters anything in the requests that it does not understand, then it should close the stream with + // an error with a standard GRPC Status.UNIMPLEMENTED code (https://grpc.github.io/grpc/core/md_doc_statuscodes.html) + // so the driver can cleanly handle this. + // If the performer hits any kind of error that does not already have an appropriate error response in the GRPC (e.g. + // something truly unexpected has happened), then it should raise an error with a standard GRPC Status.UNKNOWN code. + rpc run (run.Request) returns (stream run.Result); + + // Stream-related requests. The stream itself will have been established by another rpc, e.g. `run`. + // These rpcs are at this top-level (rather than say run.Request) because, while a stream's lifespan is bound to + // a grpc return `stream`, streams are a generic concept and we don't want to duplicate these rpcs inside future other + // rpcs. + rpc streamCancel (streams.CancelRequest) returns (streams.CancelResponse); + rpc streamRequestItems (streams.RequestItemsRequest) returns (streams.RequestItemsResponse); + + // Set the counter with the given id to a new value + rpc setCounter (shared.Counter) returns (shared.SetCounterResponse); + + // Clear all counters for a given performer + rpc clearAllCounters (shared.ClearAllCountersRequest) returns (shared.ClearAllCountersResponse); + + // Observability-related requests. + rpc spanCreate(observability.SpanCreateRequest) returns (observability.SpanCreateResponse); + rpc spanFinish(observability.SpanFinishRequest) returns (observability.SpanFinishResponse); + + // ExtSDKIntegration: the following methods will not be sent to ExtSDKIntegration-enabled performers. + // Once we eventually end support for non-ExtSDKIntegration implementations, they will be removed. + + // Create or close a Transactions (e.g. a transactions factory), returning a transactionsFactoryRef. + // Will be removed eventually as Transactions objects no longer really apply in ExtSDKIntegration - the Cluster is the 'lifecycle object'. + // Performers that need to support non-ExtSDKIntegration will need to keep this, otherwise they can remove it. + rpc transactionsFactoryCreate (transactions.TransactionsFactoryCreateRequest) returns (transactions.TransactionsFactoryCreateResponse); + rpc transactionsFactoryClose (transactions.TransactionsFactoryCloseRequest) returns (transactions.TransactionGenericResponse); + + // The following can all be removed earlier, as soon as performers no longer use them. + + // Creates a clusterConnection between performer and Couchbase Server, and returns the performer caps. + // Deprecated as ClusterConnectionCreateRequest does the job better, and has a proper close method. + // Performers can (and are recommended to) implement clusterConnectionCreate and remove support for this, including + // if they don't support ExtSDKIntegration yet. + rpc createConnection (transactions.CreateConnectionRequest) returns (transactions.CreateConnectionResponse) { + option deprecated = true; + }; + + // Request a Transactions object forces cleanup of all requests on its queue. + // This was removed as it's quite niche, used by very few tests, almost redundant since ExtRemoveComplete, and requires + // backdoor access into cleanup. It will no longer be sent and performers should remove it. + rpc transactionProcessCleanupQueue (transactions.TransactionProcessCleanupQueueRequest) returns (transactions.TransactionCleanupQueueResult) { + option deprecated = true; + }; + + // Request that the implementation remove this client from the client record, from all buckets. + // This was removed as it's quite niche, used by very few tests, and requires backdoor access into cleanup. And its + // behaviour can be easily replicated with clusterConnectionClose/transactionsFactoryClose, which will remove the + // client record everywhere too. Performers can remove this, the driver will no longer send it. + rpc clientRecordRemove (transactions.ClientRecordRemoveRequest) returns (transactions.ClientRecordRemoveResponse) { + option deprecated = true; + }; +} diff --git a/fit-performer/proto/run.config.proto b/fit-performer/proto/run.config.proto new file mode 100644 index 00000000..fd9a11c2 --- /dev/null +++ b/fit-performer/proto/run.config.proto @@ -0,0 +1,49 @@ +syntax = "proto3"; + +package protocol.run; +option csharp_namespace = "Couchbase.Grpc.Protocol.Run"; +option java_package = "com.couchbase.client.protocol.run"; +option java_multiple_files = true; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/run"; +option ruby_package = "FIT::Protocol::Run"; + + +// Controls how the performer streams back results. +// +// The driver currently maintains a 10 second buffer, e.g. it is only writing data that's at least 10 seconds old. This +// is to handle out-of-order responses from the performer and to give it some flexibility w.r.t. batching. The performer +// should never send back data that is older than this buffer - the results are undefined if it does. +// +// Note that there is no provision for dropping packets currently. It is expected that the performer (and driver, and +// network) can keep up streaming back results from any workload. Workloads could potentially last multiple hours, even +// days, and most workloads will generate continuous traffic. + +// See CBD-4975 for discussion of streaming. The two main points are: +// 1. Naively sending back results one at a time did not scale. +// 2. Batching results in BatchedResults increased flow-rate by nearly 2 orders of magnitude. +message ConfigStreaming { + // The next two parameters are only mandatory on grpc.Workloads, which are explicitly testing various streaming + // approaches. Otherwise they should be regarded as optional + // hints. If the performer follows them then it will automatically get what has been tested to most reliably + // return results. But it can override them if it has done its own testing and found its own best path for that + // language. + + // If present, the performer should stream back BatchedResults, aiming to contain this number of results. + // The performer can return less elements than this in a batch (so it does not have to wait for a write queue to + // fill first). + optional int32 batch_size = 1; + + // Whether the performer should enable flow control. This will mean different things to different + // GRPC implementations, but the concept is to only send responses when the GRPC response stream reports itself ready. + // If the performer is unable to keep up with the flow rate then this effectively just moves the filling queue from + // GRPC's to one owned by the performer, but this can still be beneficial as the performer can keep better metrics + // on its own queue. + bool flow_control = 2; + + // Whether the performer should stream back metrics. + bool enable_metrics = 3; +} + +message Config { + optional ConfigStreaming streaming_config = 1; +} diff --git a/fit-performer/proto/run.top_level.proto b/fit-performer/proto/run.top_level.proto new file mode 100644 index 00000000..16dac196 --- /dev/null +++ b/fit-performer/proto/run.top_level.proto @@ -0,0 +1,94 @@ +syntax = "proto3"; + +package protocol.run; +option csharp_namespace = "Couchbase.Grpc.Protocol.Run"; +option java_package = "com.couchbase.client.protocol.run"; +option java_multiple_files = true; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/run"; +option ruby_package = "FIT::Protocol::Run"; + +import "sdk.workload.proto"; +import "meta.workload.proto"; +import "google/protobuf/timestamp.proto"; +import "transactions.performer.proto"; +import "streams.top_level.proto"; +import "metrics.top_level.proto"; +import "run.config.proto"; +import "run.workloads.proto"; + +// For optimal streaming performance, batches of results can be returned. +message BatchedResult { + repeated Result result = 1; +} + +// The performer returns a stream of run.Result to the driver. +// When and what results it should return, are detailed elsewhere in the GRPC. +message Result { + oneof result { + // Each sdk.Command should return back one of these results. + sdk.Result sdk = 3; + + // Each meta.Command should return back one of these results. + meta.Result grpc = 4; + + // The performer can send back metrics whenever it wishes. However, due to the way database results are currently + // joined with the bucket results (which are in one second buckets), it's only useful to send back a maximum of + // one per second. + metrics.Result metrics = 5; + + // For optimal streaming performance, batches of results can be returned. + BatchedResult batched = 6; + + // Something significant has happened on a stream (such as creation or completion). + // The actual streamed results will be returned elsewhere (usually as Results). + streams.Signal stream = 7; + + // Result of a single transaction. + transactions.TransactionResult transaction = 8; + } + + // Performance times are always returned so it's easier for the performers to code-share between performance and + // integration testing. + // + // Clocks are hard. Many OS/platform combinations cannot guarantee nanosecond level precision of a wallclock time, + // but can provide such precision for elapsed time. + // So: + // + // `elapsedNanos` is intended to be, as precisely as the platform can measure it, the exact time taken by the operation in nanoseconds. + // Measured from just before sending the operation into the SDK (e.g. after handling any GRPC work and JSON conversion), and just after + // the SDK returns. To stress, this should not include any JSON or GRPC processing. + // + // `initiated` is a wallclock time, used to place this operation into a one second bucket. This should be as + // accurate as the platform can provide (often realistically this is only accurate to 10 millis or so). Hopefully + // a few operations ending up in the wrong bucket each second will not dramatically impact the results. It is ok + // to set `initiated` before the GRPC work, due to the reduced precision. + // For failed operations: + // `initiated` needs to be the wallclock time the operation was initiated, not when it failed which can + // be potentially X seconds later. + // `elapsedNanos` historically was not required by the performer, but some tests do now use it. The SDK can opt into + // these tests by always returning this field and declaring support for performer cap TIMING_ON_FAILED_OPS. + int64 elapsedNanos = 1; + google.protobuf.Timestamp initiated = 2; +} + +message Request { + // Configures how the performer executes this run. + optional Config config = 1; + + // What the performer should do in this run. A `oneof` for future extension. + oneof request { + Workloads workloads = 2; + } + + // Can apply SDK tunables that can't be represented generically in GRPC. + // + // One example is for performance testing, where we may want to experiment with various approaches in say the Java SDK. + // So one run may send "com.couchbase.executorMaxThreadCount"="10", and the next set it to "20". + // + // Interpretation of the tunables, and how to apply them inside the SDK, is completely SDK-dependent. + // + // Generally the performer should unset all tunables at the end of the run RPC. + // + // Generally performers can ignore this field unless they are explicitly planning on using it. + map tunables = 3; +} \ No newline at end of file diff --git a/fit-performer/proto/run.workloads.proto b/fit-performer/proto/run.workloads.proto new file mode 100644 index 00000000..27b2f0b0 --- /dev/null +++ b/fit-performer/proto/run.workloads.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package protocol.run; +option csharp_namespace = "Couchbase.Grpc.Protocol.Run"; +option java_package = "com.couchbase.client.protocol.run"; +option java_multiple_files = true; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/run"; +option ruby_package = "FIT::Protocol::Run"; + +import "sdk.workload.proto"; +import "meta.workload.proto"; +import "transactions.workload.proto"; + +// A workload is generally a bunch of commands to run. +message Workload { + oneof workload { + sdk.Workload sdk = 1; + meta.Workload grpc = 2; + transactions.Workload transaction = 3; + } +} + +// "HorizontalScaling" is an abstraction over there being many forms of concurrency. The core idea is that the driver +// is trying to increase the parallelism, and it's up to the performer to choose a suitable platform-dependent way to +// do this. +// For some languages that will be threads. For some, a larger pool of concurrent Future/Promises. For some, +// forking a new process. +// Whatever is produced (thread, new process), it should run the provided workload(s) in a tight loop. So essentially +// HorizontalScaling is the number of concurrent operations taking place. +message HorizontalScaling { + // Performer will run these workloads in this 'thread-like', in the specified order. + repeated Workload workloads = 1; +} + +// Allows running multiple Workloads concurrently. +message Workloads { + // The previously established cluster connection to use, for all operations instigated under this task. + // Operations below this that have their own clusterConnectionId (like TransactionCreateRequest) do _not_ override + // this value - any future exceptions to that rule will be explicitly documented here. + string cluster_connection_id = 1; + + // See HorizontalScaling for a discussion of this. Broadly, it's the number of concurrent operations + // required. + repeated HorizontalScaling horizontal_scaling = 2; +} + + diff --git a/fit-performer/proto/sdk.bucket.collection_manager.proto b/fit-performer/proto/sdk.bucket.collection_manager.proto new file mode 100644 index 00000000..48dd2568 --- /dev/null +++ b/fit-performer/proto/sdk.bucket.collection_manager.proto @@ -0,0 +1,131 @@ +syntax = "proto3"; + +package protocol.sdk.bucket.collection_manager; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Bucket.CollectionManager"; +option java_package = "com.couchbase.client.protocol.sdk.bucket.collectionmanager"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/bucket/collectionmanager"; +option ruby_package = "FIT::Protocol::SDK::Bucket::CollectionManager"; +option java_multiple_files = true; + +message Command { + + oneof command { + + GetAllScopesRequest get_all_scopes = 1; + + CreateScopeRequest create_scope = 2; + + DropScopeRequest drop_scope = 3; + + CreateCollectionRequest create_collection = 4; + + UpdateCollectionRequest update_collection = 5; + + DropCollectionRequest drop_collection = 6; + + } + +} + +message Result { + oneof result { + GetAllScopesResult get_all_scopes_result = 1; + } +} + +message CollectionSpec { + string name = 1; + string scope_name = 2; + optional int32 expiry_secs = 3; + optional bool history = 4; +} + +message CreateCollectionSettings { + optional int32 expiry_secs = 1; + optional bool history = 2; +} + +message UpdateCollectionSettings { + optional int32 expiry_secs = 1; + optional bool history = 2; +} + +message ScopeSpec { + string name = 1; + repeated CollectionSpec collections = 2; +} + +//Returns Result.get_all_scopes_result +message GetAllScopesRequest { + optional GetAllScopesOptions options = 2; +} + +message GetAllScopesOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +message GetAllScopesResult { + repeated ScopeSpec result = 1; +} + +//Returns void (Result.success) +message CreateScopeRequest { + string name = 1; + optional CreateScopeOptions options = 2; +} + +message CreateScopeOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message DropScopeRequest { + string name = 1; + optional DropScopeOptions options = 2; +} + +message DropScopeOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + + +//Returns void (Result.success) +message CreateCollectionRequest { + string name = 1; + string scope_name = 2; + optional CreateCollectionSettings settings = 3; + optional CreateCollectionOptions options = 4; +} + +message CreateCollectionOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message UpdateCollectionRequest { + string name = 1; + string scope_name = 2; + UpdateCollectionSettings settings = 3; + optional UpdateCollectionOptions options = 4; +} + +message UpdateCollectionOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message DropCollectionRequest { + string name = 1; + string scope_name = 2; + optional DropCollectionOptions options = 3; +} + +message DropCollectionOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} diff --git a/fit-performer/proto/sdk.caps.proto b/fit-performer/proto/sdk.caps.proto new file mode 100644 index 00000000..bd6df94a --- /dev/null +++ b/fit-performer/proto/sdk.caps.proto @@ -0,0 +1,155 @@ +syntax = "proto3"; + +package protocol.sdk; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk"; +option java_package = "com.couchbase.client.protocol.sdk"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk"; +option ruby_package = "FIT::Protocol::SDK"; +option java_multiple_files = true; + +enum Caps { + SDK_PRESERVE_EXPIRY = 0; + + // KV range scan support. See CBD-5161 for requirements. + SDK_KV_RANGE_SCAN = 1; + + // Query index management support - All APIs available to the Cluster level QueryIndexManager: cluster.queryIndexes() + SDK_QUERY_INDEX_MANAGEMENT = 2; + + // All APIs available to the Collection level CollectionQueryIndexManager: collection.queryIndexes() + SDK_COLLECTION_QUERY_INDEX_MANAGEMENT = 3; + + // Can execute cluster.search() with all SearchOptions. + // This is everything under sdk.search.proto, except VectorQuery. + SDK_SEARCH = 4; + + // Supports sdk.search.VectorQuery. + // Support for SDK_SEARCH is a prerequisite. + SDK_VECTOR_SEARCH = 22; + + // Support for SDK_VECTOR_SEARCH is a prerequisite. + SDK_VECTOR_SEARCH_BASE64 = 23; + + // Can execute scope.search() with all SearchOptions. + // Support for SDK_SEARCH is a prerequisite. + SDK_SCOPE_SEARCH = 6; + + // The SDK implements support for search SDK-RFC 52 revision 11. + // (Improved FeatureNotAvailable handling against older clusters for scoped and vector indexes) + SDK_SEARCH_RFC_REVISION_11 = 24; + + // Can execute all APIs under cluster.searchIndexes(), with all options. + // This is everything under sdk.management.search.proto. + SDK_SEARCH_INDEX_MANAGEMENT = 5; + + // Can execute all APIs under scope.searchIndexes(), with all options. + // This is everything under sdk.management.scope.search.proto. + SDK_SCOPE_SEARCH_INDEX_MANAGEMENT = 7; + + //The performer supports WaitUntilReady with options, at cluster and bucket level + WAIT_UNTIL_READY = 8; + + //The SDK supports couchbase2:// connection strings + PROTOSTELLAR = 9; + + //The performer supports the BucketManager API. + //This is everything under sdk.cluster.bucket_manager + SDK_BUCKET_MANAGEMENT = 10; + + //The performer supports the EventingFunctionManager API. + //This is everything under sdk.cluster.eventing_function_manager + SDK_EVENTING_FUNCTION_MANAGER = 11; + + //The performer supports ClusterLevelCommand.query, ScopeLevelCommand.query, + //and everything needed by sdk.query.Command + //it supports sending results back with sdk.query.QueryResult + //Does not include the use_replica Query option + SDK_QUERY = 12; + + // Implements base LookupIn functionality from sdk.collection.lookup_in.proto + SDK_LOOKUP_IN = 13; + + // Implements LookupInAllReplicas and LookupInAnyReplica messages from sdk.collection.lookup_in.proto + SDK_LOOKUP_IN_REPLICAS = 14; + + // Implements use_replica Query option + + SDK_QUERY_READ_FROM_REPLICA = 15; + + // The performer implements everything under sdk.kv.commands and sdk.kv.mutate_in, + // and is able to support all forms of shared.Content for Get and MutateIn commands + // The performer cap KV_SUPPORT_1 is a prerequisite + SDK_KV = 17; + + // Supports Extended SDK Observability SDK-RFC 67 revision 24. E.g. the SDK is sending additional metric tags. + // Support for performer cap OBSERVABILITY_1 is a pre-requisite. + SDK_OBSERVABILITY_RFC_REV_24 = 16; + + // Implements history retention settings for bucket and collection management. + SDK_MANAGEMENT_HISTORY_RETENTION = 18; + + //The performer supports the CollectionManager API. + //This is everything under sdk.bucket.collection_manager + SDK_COLLECTION_MANAGEMENT = 19; + + // SDK can report the SDK_DOCUMENT_NOT_LOCKED_EXCEPTION error. + // The SDK_KV cap is a pre-requisite. + SDK_DOCUMENT_NOT_LOCKED = 20; + + SDK_CIRCUIT_BREAKERS = 21; + + // The SDK implements support for index management SDK-RFC 54 revision 25. + // (Improved FeatureNotAvailable handling against older clusters for scoped and vector indexes) + SDK_INDEX_MANAGEMENT_RFC_REVISION_25 = 25; + + // The SDK implements support for observability with couchbase2, as specified in SDK-RFC 77. + SDK_COUCHBASE2_OBSERVABILITY = 26; + + // The SDK implements support for Zone aware read from replica in Lookupin and Get operations + // The options requiring support are SELECTED_SERVER_GROUP + // Add SDK_ZONE_AWARE_READ_FROM_REPLICA_SELECTED_SERVER_GROUP_OR_ALL_AVAILABLE if your performer also supports the optional + // SELECTED_SERVER_GROUP_OR_ALL_AVAILABLE mode. + SDK_ZONE_AWARE_READ_FROM_REPLICA = 27; + SDK_ZONE_AWARE_READ_FROM_REPLICA_SELECTED_SERVER_GROUP_OR_ALL_AVAILABLE = 29; + + // The SDK will send cluster name and uuid labels in spans and metrics. + // OBSERVABILITY_1 is a prerequisite. (But not SDK_OBSERVABILITY_RFC_REV_24) + SDK_OBSERVABILITY_CLUSTER_LABELS = 28; + + // The SDK implements support for the Application Telemetry feature, as detailed in SDK-RFC 84. + SDK_APP_TELEMETRY = 30; + + // The SDK implements the num_vbuckets attribute of BucketSettings + SDK_BUCKET_SETTINGS_NUM_VBUCKETS = 31; + + // The SDK implements support for Vector Search prefiltering. + SDK_PREFILTER_VECTOR_SEARCH = 32; + + // The SDK supports setting both positional and named parameters for query and analytics_query. + // * SDK3 Query RFC Revision #9 + // * SDK3 Analytics RFC Revision #3 + SDK_QUERY_BOTH_POSITIONAL_AND_NAMED_PARAMETERS = 33; + + // The SDK supports the Authenticator type. + SUPPORTS_AUTHENTICATOR = 34; + + // The SDK supports setting a new Authenticator. + SDK_SET_AUTHENTICATOR = 35; + + // The SDK supports JwtAuthenticator. + SDK_JWT = 36; + + // The SDK supports the stable OpenTelemetry semantic conventions (ExtendedObservability RFC Rev 26) + SDK_STABLE_OTEL_SEMANTIC_CONVENTIONS = 37; + + // The SDK emits the stable OpenTelemetry semantic conventions by default. + // SDK_STABLE_OTEL_SEMANTIC_CONVENTIONS is a pre-requisite. This should only be declared by SDKs that do not + // support the legacy semantic conventions (C++ & wrappers, Rust). + SDK_STABLE_OTEL_SEMANTIC_CONVENTIONS_EMITTED_BY_DEFAULT = 38; + + // The SDK supports the collection.getOrNull() API. + SDK_GET_OR_NULL = 39; + + // The SDK supports the RESUMING status in the result of functions_status() of the eventing management API. + SDK_EVENTING_RESUMING_FUNCTION_STATUS = 40; +} diff --git a/fit-performer/proto/sdk.circuit_breaker.config.proto b/fit-performer/proto/sdk.circuit_breaker.config.proto new file mode 100644 index 00000000..dd7317cd --- /dev/null +++ b/fit-performer/proto/sdk.circuit_breaker.config.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package protocol.sdk.circuit_breaker; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.CircuitBreaker"; +option java_package = "com.couchbase.client.protocol.sdk.circuit_breaker"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/circuit_breaker"; +option ruby_package = "FIT::Protocol::Sdk::CircuitBreaker"; +option java_multiple_files = true; + +// All settings here copied from the circuit breaker SDK-RFC 33. +message ServiceConfig { + optional bool enabled = 1; + optional int32 volume_threshold = 2; + optional int32 error_threshold_percentage = 3; + optional int32 sleep_window_ms = 4; + optional int32 rolling_window_ms = 5; + optional int32 canary_timeout_ms = 6; +} + +// SDK-RFC 33 is sparsely specced and the SDK may not support all of these. As usual the performer should return UNSUPPORTED +// if faced with any requests it cannot handle. +message Config { + optional ServiceConfig kv = 1; + optional ServiceConfig query = 2; + optional ServiceConfig view = 3; + optional ServiceConfig search = 4; + optional ServiceConfig analytics = 5; + optional ServiceConfig manager = 6; + optional ServiceConfig eventing = 7; + optional ServiceConfig backup = 8; +} \ No newline at end of file diff --git a/fit-performer/proto/sdk.cluster.bucket_manager.proto b/fit-performer/proto/sdk.cluster.bucket_manager.proto new file mode 100644 index 00000000..7260f72f --- /dev/null +++ b/fit-performer/proto/sdk.cluster.bucket_manager.proto @@ -0,0 +1,163 @@ +syntax = "proto3"; +import "shared.basic.proto"; + +package protocol.sdk.cluster.bucket_manager; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Cluster.BucketManager"; +option java_package = "com.couchbase.client.protocol.sdk.cluster.bucketmanager"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/cluster/bucketmanager"; +option ruby_package = "FIT::Protocol::SDK::Cluster::BucketManager"; +option java_multiple_files = true; + +message Command { + + oneof command { + + GetBucketRequest get_bucket = 1; + + GetAllBucketsRequest get_all_buckets = 2; + + CreateBucketRequest create_bucket = 3; + + DropBucketRequest drop_bucket = 4; + + FlushBucketRequest flush_bucket = 5; + + UpdateBucketRequest update_bucket = 6; + + } + +} + +//Returns Result.bucket_settings +message GetBucketRequest { + string bucket_name = 1; + optional GetBucketOptions options = 2; +} + +message GetBucketOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +//Returns Result.get_all_buckets_result +message GetAllBucketsRequest { + optional GetAllBucketsOptions options = 1; +} + +message GetAllBucketsOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message CreateBucketRequest { + CreateBucketSettings settings = 1; + optional CreateBucketOptions options = 2; +} + +message CreateBucketOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message DropBucketRequest { + string bucket_name = 1; + optional DropBucketOptions options = 2; +} + +message DropBucketOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message FlushBucketRequest { + string bucket_name = 1; + optional FlushBucketOptions options = 2; +} + +message FlushBucketOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message UpdateBucketRequest { + BucketSettings settings = 1; + optional UpdateBucketOptions options = 2; +} + +message UpdateBucketOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +message Result { + oneof result { + BucketSettings bucket_settings = 1; + + GetAllBucketsResult get_all_buckets_result = 2; + } +} + +message CreateBucketSettings { + BucketSettings settings = 1; + optional ConflictResolutionType conflict_resolution_type = 2; +} + +message BucketSettings { + string name = 1; + optional bool flush_enabled = 2; + int64 ram_quota_MB = 3; + optional int32 num_replicas = 4; + optional bool replica_indexes = 5; + optional BucketType bucket_type = 6; + optional EvictionPolicyType eviction_policy = 7; + optional int32 max_expiry_seconds = 8; + optional CompressionMode compression_mode = 9; + optional shared.Durability minimum_durability_level = 10; + optional StorageBackend storage_backend = 11; + optional bool history_retention_collection_default = 12; + optional int64 history_retention_seconds = 13; + optional uint64 history_retention_bytes = 14; + optional uint32 num_vbuckets = 15; +} + +message GetAllBucketsResult { + map result = 1; +} + +enum BucketType { + COUCHBASE = 0; + EPHEMERAL = 1; + MEMCACHED = 2; +} + +message EjectionPolicy { + EvictionPolicyType ejection_policy = 1; +} + +enum EvictionPolicyType { + FULL = 0; + NO_EVICTION = 1; + NOT_RECENTLY_USED = 2; + VALUE_ONLY = 3; +} + +enum CompressionMode { + ACTIVE = 0; + OFF = 1; + PASSIVE = 2; +} + +enum StorageBackend { + COUCHSTORE = 0; + MAGMA = 1; +} + +enum ConflictResolutionType { + TIMESTAMP = 0; + SEQUENCE_NUMBER = 1; + CUSTOM = 2; +} diff --git a/fit-performer/proto/sdk.cluster.eventing_function_manager.proto b/fit-performer/proto/sdk.cluster.eventing_function_manager.proto new file mode 100644 index 00000000..088cb271 --- /dev/null +++ b/fit-performer/proto/sdk.cluster.eventing_function_manager.proto @@ -0,0 +1,318 @@ +syntax = "proto3"; +import "shared.basic.proto"; +import "google/protobuf/duration.proto"; + +package protocol.sdk.cluster.eventing_function_manager; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Cluster.EventingFunctionManager"; +option java_package = "com.couchbase.client.protocol.sdk.cluster.eventingfunctionmanager"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/cluster/eventingfunctionmanager"; +option ruby_package = "FIT::Protocol::SDK::Cluster::EventingFunctionManager"; +option java_multiple_files = true; + +message Command { + + oneof command { + + GetFunctionRequest get_function = 1; + + GetAllFunctionsRequest get_all_functions = 2; + + DeployFunctionRequest deploy_function = 3; + + DropFunctionRequest drop_function = 4; + + FunctionsStatusRequest functions_status = 5; + + PauseFunctionRequest pause_function = 6; + + ResumeFunctionRequest resume_function = 7; + + UndeployFunctionRequest undeploy_function = 8; + + UpsertFunctionRequest upsert_function = 9; + + } + +} + +//Returns Result.eventing_function +message GetFunctionRequest { + string name = 1; + optional GetFunctionOptions options = 2; +} + +message GetFunctionOptions { + optional google.protobuf.Duration timeout = 1; + optional string parent_span_id = 2; +} + +//Returns Result.eventing_function_list +message GetAllFunctionsRequest { + optional GetAllFunctionsOptions options = 1; +} + +message GetAllFunctionsOptions { + optional google.protobuf.Duration timeout = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message DeployFunctionRequest { + string name = 1; + optional DeployFunctionOptions options = 2; +} + +message DeployFunctionOptions { + optional google.protobuf.Duration timeout = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message DropFunctionRequest { + string name = 1; + optional DropFunctionOptions options = 2; +} + +message DropFunctionOptions { + optional google.protobuf.Duration timeout = 1; + optional string parent_span_id = 2; +} + +//Returns Result.eventing_status +message FunctionsStatusRequest { + optional FunctionsStatusOptions options = 1; +} + +message FunctionsStatusOptions { + optional google.protobuf.Duration timeout = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message PauseFunctionRequest { + string name = 1; + optional PauseFunctionOptions options = 2; +} + +message PauseFunctionOptions { + optional google.protobuf.Duration timeout = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message ResumeFunctionRequest { + string name = 1; + optional ResumeFunctionOptions options = 2; +} + +message ResumeFunctionOptions { + optional google.protobuf.Duration timeout = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message UndeployFunctionRequest { + string name = 1; + optional UndeployFunctionOptions options = 2; +} + +message UndeployFunctionOptions { + optional google.protobuf.Duration timeout = 1; + optional string parent_span_id = 2; +} + +//Returns void (Result.success) +message UpsertFunctionRequest { + EventingFunction function = 1; + optional UpsertFunctionOptions options = 2; +} + +message UpsertFunctionOptions { + optional google.protobuf.Duration timeout = 1; + optional string parent_span_id = 2; +} + + + +message Result { + oneof result { + EventingFunction eventing_function = 1; + + EventingFunctionList eventing_function_list = 2; + + EventingStatus eventing_status = 3; + } + +} + +message EventingFunctionList { + repeated EventingFunction functions = 1; +} + + +message EventingFunction { + repeated EventingFunctionBucketBinding bucket_bindings = 1; + string code = 2; + repeated EventingFunctionConstantBinding constant_bindings = 3; + optional bool enforce_schema = 4; + optional string function_instance_id = 5; + optional int64 handler_uuid = 6; + EventingFunctionKeyspace metadata_keyspace = 7; + string name = 8; + EventingFunctionSettings settings = 9; + EventingFunctionKeyspace source_keyspace = 10; + repeated EventingFunctionUrlBinding url_bindings = 11; + optional string version = 12; + +} + +message EventingStatus { + repeated EventingFunctionState functions = 1; + int32 num_eventing_nodes = 2; +} + +message EventingFunctionState { + EventingFunctionDeploymentStatus deployment_status = 1; + string name = 2; + int32 num_bootstrapping_nodes = 3; + int32 num_deployed_nodes = 4; + EventingFunctionProcessingStatus processing_status = 5; + EventingFunctionStatus status = 6; +} + + +enum EventingFunctionDeploymentStatus { + // Using DEPLOYMENT_STATUS_ prefix to avoid clashing with EventingFunctionStatus + DEPLOYMENT_STATUS_DEPLOYED = 0; + DEPLOYMENT_STATUS_UNDEPLOYED = 1; +} + +enum EventingFunctionProcessingStatus { + // Using PROCESSING_STATUS_ prefix to avoid clashing with EventingFunctionStatus + PROCESSING_STATUS_PAUSED = 0; + PROCESSING_STATUS_RUNNING = 1; +} + +enum EventingFunctionStatus { + DEPLOYED = 0; + DEPLOYING = 1; + PAUSED = 2; + PAUSING = 3; + UNDEPLOYED = 4; + UNDEPLOYING = 5; + RESUMING = 6; + UNKNOWN = 10; +} + +message EventingFunctionBucketBinding { + EventingFunctionBucketAccess access = 1; + string alias = 2; + EventingFunctionKeyspace name = 3; +} + +enum EventingFunctionBucketAccess { + READ_ONLY = 0; + READ_WRITE = 1; +} + +message EventingFunctionConstantBinding { + string alias = 1; + string literal = 2; +} + +message EventingFunctionKeyspace { + string bucket = 1; + optional string scope = 2; + optional string collection = 3; +} + + +message EventingFunctionUrlBinding { + string alias = 1; + bool allow_cookies = 2; + EventingFunctionUrlAuth auth = 3; + string hostname = 4; + bool validate_ssl_certificate = 5; +} + +message EventingFunctionUrlAuth { + oneof auth { + EventingFunctionUrlAuthBasic basic_auth = 1; + + EventingFunctionUrlAuthBearer bearer_auth = 2; + + EventingFunctionUrlAuthDigest digest_auth = 3; + + EventingFunctionUrlNoAuth no_auth = 4; + } +} + +message EventingFunctionUrlAuthBasic { + string username = 1; + string password = 2; +} + +message EventingFunctionUrlAuthBearer { + string key = 1; +} + +message EventingFunctionUrlAuthDigest { + string username = 1; + string password = 2; +} + +message EventingFunctionUrlNoAuth { +} + + +message EventingFunctionSettings { + optional string app_log_dir = 1; + optional int64 app_log_max_files = 2; + optional int64 app_log_max_size = 3; + optional int64 bucket_cache_age = 4; + optional int64 bucket_cache_size = 5; + optional google.protobuf.Duration checkpoint_interval = 6; + optional int64 cpp_worker_thread_count = 7; + optional int64 curl_max_allowed_resp_size = 8; + optional EventingFunctionDcpBoundary dcp_stream_boundary = 9; + optional EventingFunctionDeploymentStatus deployment_status = 10; + optional string description = 11; + optional bool enable_app_log_rotation = 12; + optional google.protobuf.Duration execution_timeout = 13; + repeated string handler_footers = 14; + repeated string handler_headers = 15; + optional EventingFunctionLanguageCompatibility language_compatibility = 16; + optional int64 lcb_inst_capacity = 17; + optional int64 lcb_retry_count = 18; + optional google.protobuf.Duration lcb_timeout = 19; + optional EventingFunctionLogLevel log_level = 20; + optional int64 num_timer_partitions = 21; + optional EventingFunctionProcessingStatus processing_status = 22; + optional shared.ScanConsistency query_consistency = 23; + optional bool query_prepare_all = 24; + optional int64 sock_batch_size = 25; + optional google.protobuf.Duration tick_duration = 26; + optional int64 timer_context_size = 27; + optional string user_prefix = 28; + optional int64 worker_count = 29; +} + +enum EventingFunctionDcpBoundary { + EVERYTHING = 0; + FROM_NOW = 1; +} + +enum EventingFunctionLanguageCompatibility { + VERSION_6_0_0 = 0; + VERSION_6_5_0 = 1; + VERSION_6_6_2 = 2; + VERSION_7_2_0 = 3; +} + +enum EventingFunctionLogLevel { + DEBUG = 0; + ERROR = 1; + INFO = 2; + TRACE = 3; + WARNING = 4; +} diff --git a/fit-performer/proto/sdk.cluster.query.index_manager.proto b/fit-performer/proto/sdk.cluster.query.index_manager.proto new file mode 100644 index 00000000..4aa8fe17 --- /dev/null +++ b/fit-performer/proto/sdk.cluster.query.index_manager.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package protocol.sdk.cluster.query.index_manager; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Cluster.Query.IndexManager"; +option java_package = "com.couchbase.client.protocol.sdk.cluster.query.indexmanager"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/cluster/query/indexmanager"; +option ruby_package = "FIT::Protocol::SDK::Cluster::Query::IndexManager"; +option java_multiple_files = true; + +import "sdk.query.index_manager.proto"; +import "sdk.query.index_manager.options.proto"; + + +// This file is for QueryIndexManager: cluster.queryIndexes() +// For Collection-level query index manager (collection.queryIndexes()), see sdk.collection.query.index_manager.proto. + +message Command { + // Rather than have two copies of CreatePrimaryIndex etc., we factor out the bucket_name parameter that's taken by + // all QueryIndexManager calls. + // (QueryIndexManager should really be bucket.queryIndexes()) which would allow it to be more cleanly DRYed into + // BucketLevelCommand). + string bucket_name = 1; + + oneof command { + // Currently the QueryIndexManager and CollectionQueryIndexManager APIs are largely identical, and the intersection is + // represented by this shared message. + sdk.query.index_manager.Command shared = 2; + + // Any APIs added only to QueryIndexManager should go here. + } +} + diff --git a/fit-performer/proto/sdk.cluster.search.index_manager.proto b/fit-performer/proto/sdk.cluster.search.index_manager.proto new file mode 100644 index 00000000..8a8618fa --- /dev/null +++ b/fit-performer/proto/sdk.cluster.search.index_manager.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package protocol.sdk.cluster.search.index_manager; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Cluster.Search.IndexManager"; +option java_package = "com.couchbase.client.protocol.sdk.cluster.search.indexmanager"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/cluster/search/indexmanager"; +option java_multiple_files = true; + +import "sdk.search.index_manager.proto"; + +// This file is for global SearchIndexManager: cluster.searchIndexes() +// For scope search index manager (scope.searchIndexes()), see sdk.search.index_manager.scope.proto. +// Currently the APIs are identical, but since they could diverge in future, separating them now. + +message Command { + oneof command { + // Currently the SearchIndexManager and ScopeSearchIndexManager APIs are largely identical, and the intersection is + // represented by this shared message. + sdk.search.index_manager.Command shared = 1; + + // Any APIs added only to SearchIndexManager should go here. + } +} diff --git a/fit-performer/proto/sdk.cluster.wait_until_ready.proto b/fit-performer/proto/sdk.cluster.wait_until_ready.proto new file mode 100644 index 00000000..f344b3a4 --- /dev/null +++ b/fit-performer/proto/sdk.cluster.wait_until_ready.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package protocol.sdk.cluster.wait_until_ready; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Cluster.WaitUntilReady"; +option java_package = "com.couchbase.client.protocol.sdk.cluster.waituntilready"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/cluster/waituntilready"; +option ruby_package = "FIT::Protocol::SDK::Cluster::WaitUntilReady"; +option java_multiple_files = true; + +message WaitUntilReadyRequest { + int32 timeoutMillis = 2; + optional WaitUntilReadyOptions options = 3; +} + +message WaitUntilReadyOptions { + optional ClusterState desiredState = 1; + repeated ServiceType serviceTypes = 2; +} + +enum ClusterState { + DEGRADED = 0; + OFFLINE = 1; + ONLINE = 2; +} + +enum ServiceType { + ANALYTICS = 0; + BACKUP = 1; + EVENTING = 2; + KV = 3; + MANAGER = 4; + QUERY = 5; + SEARCH = 6; + VIEWS = 7; +} \ No newline at end of file diff --git a/fit-performer/proto/sdk.collection.query.index_manager.proto b/fit-performer/proto/sdk.collection.query.index_manager.proto new file mode 100644 index 00000000..4876327d --- /dev/null +++ b/fit-performer/proto/sdk.collection.query.index_manager.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package protocol.sdk.collection.query.index_manager; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Collection.Query.IndexManager"; +option java_package = "com.couchbase.client.protocol.sdk.collection.query.indexmanager"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/collection/query/indexmanager"; +option ruby_package = "FIT::Protocol::SDK::Collection::Query::IndexManager"; +option java_multiple_files = true; + +import "sdk.query.index_manager.proto"; + + +// This file is for CollectionQueryIndexManager: collection.queryIndexes() +// For Cluster-level query index manager (cluster.queryIndexes()), see sdk.cluster.query.index_manager.proto. + +message Command { + oneof command { + // Currently the QueryIndexManager and CollectionQueryIndexManager APIs are largely identical, and the intersection is + // represented by this shared message. + sdk.query.index_manager.Command shared = 1; + + // Any APIs added only to CollectionQueryIndexManager should go here. + } +} + diff --git a/fit-performer/proto/sdk.kv.binary.commands.proto b/fit-performer/proto/sdk.kv.binary.commands.proto new file mode 100644 index 00000000..17218fad --- /dev/null +++ b/fit-performer/proto/sdk.kv.binary.commands.proto @@ -0,0 +1,45 @@ +syntax = "proto3"; + +package protocol.sdk.kv; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Kv"; +option java_package = "com.couchbase.client.protocol.sdk.kv"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/kv"; +option ruby_package = "FIT::Protocol::SDK::KV"; +option java_multiple_files = true; + +import "sdk.kv.binary.options.proto"; +import "shared.doc_location.proto"; +import "shared.basic.proto"; + + +// Performer will return a MutationResult +message Append { + shared.DocLocation location = 1; + bytes content = 2; + optional AppendOptions options = 3; +} + +// Performer will return a MutationResult +message Prepend { + shared.DocLocation location = 1; + bytes content = 2; + optional PrependOptions options = 3; +} + +// Performer will return a CounterResult +message Decrement { + shared.DocLocation location = 1; + optional DecrementOptions options = 3; +} + +// Performer will return a CounterResult +message Increment { + shared.DocLocation location = 1; + optional IncrementOptions options = 3; +} + +message CounterResult { + int64 cas = 1; + optional shared.MutationToken mutation_token = 2; + int64 content = 3; +} diff --git a/fit-performer/proto/sdk.kv.binary.options.proto b/fit-performer/proto/sdk.kv.binary.options.proto new file mode 100644 index 00000000..cb41fd8d --- /dev/null +++ b/fit-performer/proto/sdk.kv.binary.options.proto @@ -0,0 +1,42 @@ +syntax = "proto3"; + +package protocol.sdk.kv; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Kv"; +option java_package = "com.couchbase.client.protocol.sdk.kv"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/kv"; +option ruby_package = "FIT::Protocol::SDK::KV"; +option java_multiple_files = true; + +import "shared.basic.proto"; + +message IncrementOptions { + optional int32 timeout_msecs = 1; + optional shared.Expiry expiry = 2; + optional int64 delta = 3; + optional int64 initial = 4; + optional shared.DurabilityType durability = 5; + optional string parent_span_id = 6; +} + +message DecrementOptions { + optional int32 timeout_msecs = 1; + optional shared.Expiry expiry = 2; + optional int64 delta = 3; + optional int64 initial = 4; + optional shared.DurabilityType durability = 5; + optional string parent_span_id = 6; +} + +message AppendOptions { + optional int32 timeout_msecs = 1; + optional int64 cas = 2; + optional shared.DurabilityType durability = 5; + optional string parent_span_id = 6; +} + +message PrependOptions { + optional int32 timeout_msecs = 1; + optional int64 cas = 2; + optional shared.DurabilityType durability = 5; + optional string parent_span_id = 6; +} \ No newline at end of file diff --git a/fit-performer/proto/sdk.kv.commands.proto b/fit-performer/proto/sdk.kv.commands.proto new file mode 100644 index 00000000..fa8a4c67 --- /dev/null +++ b/fit-performer/proto/sdk.kv.commands.proto @@ -0,0 +1,148 @@ +syntax = "proto3"; + +package protocol.sdk.kv; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Kv"; +option java_package = "com.couchbase.client.protocol.sdk.kv"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/kv"; +option ruby_package = "FIT::Protocol::SDK::KV"; +option java_multiple_files = true; + +import "sdk.kv.options.proto"; +import "shared.doc_location.proto"; +import "shared.content.proto"; +import "shared.basic.proto"; +import "google/protobuf/duration.proto"; +import "streams.top_level.proto"; + + +// Performer will return a MutationResult +message Insert { + shared.DocLocation location = 1; + shared.Content content = 2; + optional InsertOptions options = 3; +} + +// Performer will return a GetResult +message Get { + shared.DocLocation location = 1; + optional GetOptions options = 2; + + shared.ContentAs content_as = 3; +} + +// Performer will return a GetResult or a Result.null_result=true +message GetOrNull { + shared.DocLocation location = 1; + // This must be a GetOrNullOptions block in the SDK in case they ever diverge, but in FIT where future breakage is + // more easily patched, we can be lazy and DRY. + optional GetOptions options = 2; + + shared.ContentAs content_as = 3; +} + +// Performer will return a GetResult +message GetAndLock { + shared.DocLocation location = 1; + google.protobuf.Duration duration = 2; + optional GetAndLockOptions options = 3; + + shared.ContentAs content_as = 4; +} + +// Performer will return a GetReplicaResult +message GetAllReplicas { + shared.DocLocation location = 1; + optional GetAllReplicasOptions options = 2; + + // Not sent on to the SDK - instead specifies how the performer should handle the streaming. + streams.Config stream_config = 3; + + shared.ContentAs content_as = 4; +} + +// Performer will return a GetReplicaResult +message GetAnyReplica { + shared.DocLocation location = 1; + optional GetAnyReplicaOptions options = 2; + + shared.ContentAs content_as = 3; +} + +// Performer will return an empty result with success set +message Unlock { + shared.DocLocation location = 1; + int64 cas = 2; + optional UnlockOptions options = 3; +} + +// Performer will return a GetResult +message GetAndTouch { + shared.DocLocation location = 1; + shared.Expiry expiry = 2; + optional GetAndTouchOptions options = 3; + + shared.ContentAs content_as = 4; +} + +// Performer will return a MutationResult +message Touch { + shared.DocLocation location = 1; + shared.Expiry expiry = 2; + optional TouchOptions options = 3; +} + +// Performer will return a MutationResult +message Remove { + shared.DocLocation location = 1; + optional RemoveOptions options = 2; +} + +// Performer will return a MutationResult +message Replace { + shared.DocLocation location = 1; + shared.Content content = 2; + optional ReplaceOptions options = 3; +} + +// Performer will return a MutationResult +message Upsert { + shared.DocLocation location = 1; + shared.Content content = 2; + optional UpsertOptions options = 3; +} + +// Performer will return an ExistsResult +message Exists { + shared.DocLocation location = 1; + ExistsOptions options = 2; +} + +message MutationResult { + int64 cas = 1; + optional shared.MutationToken mutation_token = 2; +} + +message GetResult { + int64 cas = 1; + shared.ContentTypes content = 2; + // Represented as the number of seconds since January 1, 1970 (e.g. what the server returns) + optional int64 expiry_time = 3; +} + +message ExistsResult { + int64 cas = 1; + bool exists = 2; +} + +message GetReplicaResult { + int64 cas = 1; + shared.ContentTypes content = 2; + bool is_replica = 3; + + // Represented as the number of seconds since January 1, 1970 (e.g. what the server returns) + optional int64 expiry_time = 4; + + // Not part of the SDK response - identifies what stream this is from. + // This is only needed when returning the result of collection.getAllReplicas() + optional string stream_id = 5; +} \ No newline at end of file diff --git a/fit-performer/proto/sdk.kv.lookup_in.proto b/fit-performer/proto/sdk.kv.lookup_in.proto new file mode 100644 index 00000000..f0115c8c --- /dev/null +++ b/fit-performer/proto/sdk.kv.lookup_in.proto @@ -0,0 +1,153 @@ +syntax = "proto3"; + +package protocol.sdk.kv.lookup_in; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Kv.LookupIn"; +option java_package = "com.couchbase.client.protocol.sdk.kv.lookupin"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/kv/lookupin"; +option ruby_package = "FIT::Protocol::SDK::KV::LookupIn"; +option java_multiple_files = true; + +import "shared.basic.proto"; +import "shared.content.proto"; +import "shared.doc_location.proto"; +import "streams.top_level.proto"; +import "shared.exceptions.proto"; + +message LookupIn { + // Use for collection and ID - Should be used over the collection level command collection + shared.DocLocation location = 1; + + repeated LookupInSpec spec = 2; + + optional LookupInOptions options = 3; +} + +// A Scan will result in the performer sending back a stream of (in the golden path): +// (1) A stream.Created result with type STREAM_LOOKUP_IN_ALL_REPLICAS, followed by +// (2) Zero or more LookupInAllReplicasResult results (and/or streams.Error), followed by +// (3) A stream.Complete result. +message LookupInAllReplicas { + shared.DocLocation location = 1; + + repeated LookupInSpec spec = 2; + + optional LookupInAllReplicasOptions options = 3; + + // Not sent on to the SDK - instead specifies how the performer should handle the streaming. + streams.Config stream_config = 4; +} + +message LookupInAnyReplica { + shared.DocLocation location = 1; + + repeated LookupInSpec spec = 2; + + optional LookupInAnyReplicaOptions options = 3; +} + +message LookupInResult { + // For each LookupInSpec we should return a LookupInSpecResult within the total LookupInResult + repeated LookupInSpecResult results = 1; + + int64 cas = 2; +} + +message LookupInSpecResult { + // If from an exists lookup: + // Either an error propagated from exists_result or the value of exists_result converted to the requested content type + // Otherwise: + // Returns an exception or the value of the result in the requested content type + shared.ContentOrError content_as_result = 1; + + // True if a result was found, false if the path was not found. Otherwise will be an exception + BooleanOrError exists_result = 2; +} + +message BooleanOrError { + oneof result { + bool value = 1; + protocol.shared.Exception exception = 2; + } +} + +message LookupInAllReplicasResult { + LookupInReplicaResult lookup_in_replica_result = 1; + + // Not part of the SDK response - identifies what stream this is from. + string stream_id = 2; +} + +message LookupInReplicaResult { + // For each LookupInSpec we should return a LookupInSpecResult within the total LookupInAnyReplicasResult + repeated LookupInSpecResult results = 1; + // Returns true if the read was from a replica node + bool is_replica = 2; + + int64 cas = 3; +} + +message LookupInOptions { + optional uint32 timeout_millis = 1; + + // Serializer - Needs to think on implementation and follow up + + optional bool access_deleted = 3; + + optional string parent_span_id = 4; +} + +message LookupInAllReplicasOptions { + optional uint32 timeout_millis = 1; + + // Serializer - Needs to think on implementation and follow up + + optional string parent_span_id = 3; + + // optional shared.RetryStrategy retry_strategy = 4; - Need to think on FIT wide retry strategy implementation + + // Will only be sent if support is declared for SDK_ZONE_AWARE_READ_FROM_REPLICA + optional shared.ReadPreference read_preference = 5; +} + +message LookupInAnyReplicaOptions { + optional uint32 timeout_millis = 1; + + // Serializer - Needs to think on implementation and follow up + + optional string parent_span_id = 3; + + // optional shared.RetryStrategy retry_strategy = 4; - Need to think on FIT wide retry strategy implementation + + // Will only be sent if support is declared for SDK_ZONE_AWARE_READ_FROM_REPLICA + optional shared.ReadPreference read_preference = 5; +} + +message LookupInSpec { + oneof operation { + ExistsOperation exists = 1; + + GetOperation get = 2; + + CountOperation count = 3; + } + + shared.ContentAs content_as = 4; +} + +message ExistsOperation { + string path = 1; + + optional bool xattr = 2; +} + +message GetOperation { + string path = 1; + + optional bool xattr = 2; +} + +message CountOperation { + string path = 1; + + optional bool xattr = 2; +} \ No newline at end of file diff --git a/fit-performer/proto/sdk.kv.mutate_in.proto b/fit-performer/proto/sdk.kv.mutate_in.proto new file mode 100644 index 00000000..a1b1c782 --- /dev/null +++ b/fit-performer/proto/sdk.kv.mutate_in.proto @@ -0,0 +1,197 @@ +syntax = "proto3"; + +package protocol.sdk.kv.mutate_in; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Collection.MutateIn"; +option java_package = "com.couchbase.client.protocol.sdk.collection.mutatein"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/collection/mutatein"; +option ruby_package = "FIT::Protocol::SDK::KV::MutateIn"; +option java_multiple_files = true; + +import "shared.basic.proto"; +import "shared.content.proto"; +import "shared.doc_location.proto"; + +// Performer will return a MutateInResult +message MutateIn { + // Use for collection and ID + shared.DocLocation location = 1; + + repeated MutateInSpec spec = 2; + + optional MutateInOptions options = 3; +} + +message MutateInResult { + int64 cas = 1; + optional shared.MutationToken mutation_token = 2; + repeated MutateInSpecResult results = 3; +} + +message MutateInSpecResult { + shared.ContentOrError content_as_result = 1; +} + +message MutateInOptions { + optional int32 timeout_millis = 1; + + optional shared.Expiry expiry = 2; + + optional int64 cas = 3; + + optional shared.DurabilityType durability = 4; + + optional StoreSemantics store_semantics = 7; + + optional bool access_deleted = 8; + + optional bool preserve_expiry = 9; + + optional bool create_as_deleted = 10; + + optional string parent_span_id = 11; +} + +enum StoreSemantics { + INSERT = 0; + REPLACE = 1; + UPSERT = 2; +} + +message MutateInSpec { + oneof operation { + + UpsertOperation upsert = 1; + + InsertOperation insert = 2; + + ReplaceOperation replace = 3; + + RemoveOperation remove = 4; + + ArrayAppendOperation array_append = 5; + + ArrayPrependOperation array_prepend = 6; + + ArrayInsertOperation array_insert = 7; + + ArrayAddUniqueOperation array_add_unique = 8; + + IncrementOperation increment = 9; + + DecrementOperation decrement = 10; + + } + + // The performer will be expected to return the content in the form specified by the content_as field + // This is because mutateInSpec operations can return counter results or mutation results. + // If not provided then the performer should return a MutateInResult with an empty value for the index at this point + optional shared.ContentAs content_as = 11; +} + +message ContentOrMacro { + oneof content_or_macro { + shared.Content content = 2; + MutateInMacro macro = 5; + } +} + +message UpsertOperation { + string path = 1; + + ContentOrMacro content = 2; + + optional bool xattr = 3; + + optional bool create_path = 4; +} + +message InsertOperation { + string path = 1; + + ContentOrMacro content = 2; + + optional bool xattr = 3; + + optional bool create_path = 4; + +} + +message ReplaceOperation { + string path = 1; + + ContentOrMacro content = 2; + + optional bool xattr = 3; +} + +message RemoveOperation { + string path = 1; + + optional bool xattr = 2; +} + +message ArrayAppendOperation { + string path = 1; + + repeated ContentOrMacro content = 2; + + optional bool xattr = 3; + + optional bool create_path = 4; +} + +message ArrayPrependOperation { + string path = 1; + + repeated ContentOrMacro content = 2; + + optional bool xattr = 3; + + optional bool create_path = 4; +} + +message ArrayInsertOperation { + string path = 1; + + repeated ContentOrMacro content = 2; + + optional bool xattr = 3; + + optional bool create_path = 4; +} + +message ArrayAddUniqueOperation { + string path = 1; + + ContentOrMacro content = 2; + + optional bool xattr = 3; + + optional bool create_path = 4; +} + +message IncrementOperation { + string path = 1; + + int64 delta = 2; + + optional bool xattr = 3; + + optional bool create_path = 4; +} + +message DecrementOperation { + string path = 1; + + int64 delta = 2; + + optional bool xattr = 3; + + optional bool create_path = 4; +} + +enum MutateInMacro { + CAS = 0; + SEQ_NO = 1; + VALUE_CRC_32C = 2; +} \ No newline at end of file diff --git a/fit-performer/proto/sdk.kv.options.proto b/fit-performer/proto/sdk.kv.options.proto new file mode 100644 index 00000000..ad956fa6 --- /dev/null +++ b/fit-performer/proto/sdk.kv.options.proto @@ -0,0 +1,101 @@ +syntax = "proto3"; + +package protocol.sdk.kv; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Kv"; +option java_package = "com.couchbase.client.protocol.sdk.kv"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/kv"; +option ruby_package = "FIT::Protocol::SDK::KV"; +option java_multiple_files = true; + +import "shared.transcoders.proto"; +import "shared.basic.proto"; + +message InsertOptions { + optional int32 timeout_msecs = 1; + optional shared.DurabilityType durability = 2; + optional shared.Expiry expiry = 3; + optional shared.Transcoder transcoder = 4; + optional string parent_span_id = 5; +} + +message ReplaceOptions { + optional int32 timeout_msecs = 1; + optional shared.DurabilityType durability = 2; + optional shared.Expiry expiry = 3; + optional bool preserve_expiry = 4; + optional int64 cas = 5; + optional shared.Transcoder transcoder = 6; + optional string parent_span_id = 7; +} + +message UpsertOptions { + optional int32 timeout_msecs = 1; + optional shared.DurabilityType durability = 2; + optional shared.Expiry expiry = 3; + optional bool preserve_expiry = 4; + optional shared.Transcoder transcoder = 5; + optional string parent_span_id = 6; +} + +message RemoveOptions { + optional int32 timeout_msecs = 1; + optional shared.DurabilityType durability = 2; + optional int64 cas = 3; + optional string parent_span_id = 4; +} + +message GetOptions { + optional int32 timeout_msecs = 1; + optional bool with_expiry = 2; + // It's not possible to have 'optional repeated' in GRPC. If this is empty, don't set the projected option. + repeated string projection = 3; + optional shared.Transcoder transcoder = 4; + optional string parent_span_id = 5; +} + +message GetAndTouchOptions { + optional int32 timeout_msecs = 1; + optional shared.Transcoder transcoder = 2; + optional string parent_span_id = 3; +} + +message GetAndLockOptions { + optional int32 timeout_msecs = 1; + optional shared.Transcoder transcoder = 2; + optional string parent_span_id = 3; +} + +message UnlockOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 3; +} + +message ExistsOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +message TouchOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +message GetAllReplicasOptions { + optional int32 timeout_msecs = 1; + optional shared.Transcoder transcoder = 2; + + // Will only be sent if support is declared for SDK_ZONE_AWARE_READ_FROM_REPLICA + optional shared.ReadPreference read_preference = 3; + + optional string parent_span_id = 4; +} + +message GetAnyReplicaOptions { + optional int32 timeout_msecs = 1; + optional shared.Transcoder transcoder = 2; + + // Will only be sent if support is declared for SDK_ZONE_AWARE_READ_FROM_REPLICA + optional shared.ReadPreference read_preference = 3; + + optional string parent_span_id = 4; +} \ No newline at end of file diff --git a/fit-performer/proto/sdk.kv.rangescan.top_level.proto b/fit-performer/proto/sdk.kv.rangescan.top_level.proto new file mode 100644 index 00000000..a6061335 --- /dev/null +++ b/fit-performer/proto/sdk.kv.rangescan.top_level.proto @@ -0,0 +1,132 @@ +syntax = "proto3"; + +package protocol.sdk.kv.rangescan; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Kv.RangeScan"; +option java_package = "com.couchbase.client.protocol.sdk.kv.rangescan"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/kv/rangescan"; +option ruby_package = "FIT::Protocol::SDK::KV::RangeScan"; +option java_multiple_files = true; + +import "shared.basic.proto"; +import "shared.collection.proto"; +import "shared.content.proto"; +import "shared.transcoders.proto"; +import "streams.top_level.proto"; + +message ScanTerm { + // This is now stored as a string and the user can only pass a string. + // Deprecating the bytes method but not removing for the same reasons as below + oneof term { + string as_string = 1; + bytes as_bytes = 2 [deprecated=true]; + } + + // If exclusive is set to false then the SDK should explicitly set it to false not leave it as default + optional bool exclusive = 3; +} + +message ScanTermChoice { + oneof choice { + // These options are deprecated as they were removed from the RFC. + // I have not removed them as it breaks compilation on existing uses which will slow down this work. Please remove when you can + bool minimum = 1 [deprecated=true]; + bool maximum = 2 [deprecated=true]; + + ScanTerm term = 3; + + // Since the behaviour if a term is not supplied is to use min for from and max for to then this option is to represent the user not supplying any term + bool default = 4; + } +} + +// Moved to new message containing from and to +message Range { + ScanTermChoice from = 1; + ScanTermChoice to = 2; +} + +message RangeScan { + // Only one of these will be passed. If your SDK isn't planning to support doc_id_prefix we will disable those tests through caps or some other method + oneof range { + Range from_to = 1; + + // This was created and named before it became "a first class part of the API". + // Rather than break existing implementations this will remain the same, but now represents the PrefixScan ScanType + string doc_id_prefix = 2; + } +} + +message SamplingScan { + uint64 limit = 1; + optional uint64 seed = 2; +} + +message ScanType { + oneof type { + RangeScan range = 1; + SamplingScan sampling = 2; + } +} + +//Sorting is not supported for RangeScans anymore. +enum ScanSort { + KV_RANGE_SCAN_SORT_NONE = 0 [deprecated = true]; + KV_RANGE_SCAN_SORT_ASCENDING = 1 [deprecated = true]; +} + +message ScanOptions { + // Renamed in the RFC. Used to be "without_content" and will need changing in performers + optional bool ids_only = 1; + optional shared.MutationState consistent_with = 2; + optional ScanSort sort = 3 [deprecated = true]; //Sorting is not supported for RangeScans anymore. + optional shared.Transcoder transcoder = 4; + optional int32 timeout_msecs = 5; + // As usual, parent_span_id will only be sent if the performer declares support for Caps.OBSERVABILITY_1 + optional string parent_span_id = 6; + optional uint32 batch_byte_limit = 7; + optional uint32 batch_item_limit = 8; + + // Since this isn't going to be present in all sdks then the logic to not use it will be on the test-driver side with @ignoreWhen + optional uint32 batch_time_limit = 9; + + // Since all SDKs do not support this option the logic to not use it will be on the driver side with @ignoreWhen + optional uint32 concurrency = 10; +} + +message ScanResult { + string id = 1; + + // Following three fields shouldn't be present if withoutContent was set. + optional shared.ContentTypes content = 2; // This field should also be empty if the test did not set ContentAs in the Scan message + + // Represented as the number of seconds since January 1, 1970 (e.g. what the server returns) + optional int64 expiry_time = 3; + optional int64 cas = 4; + + // Not part of the SDK response - identifies what stream this is from. + string stream_id = 5; + + // Returns the value of idsOnly from the scan associated with the result + bool id_only = 6; +} + +// A Scan will result in the performer sending back a stream of (in the golden path): +// (1) A stream.Created result with type STREAM_KV_RANGE_SCAN, followed by +// (2) Zero or more ScanResult results (and/or streams.Error), followed by +// (1) A stream.Complete result. +// +// A streams.CancelRequest from the driver should be handled per the RFC - with a range scan cancellation sent to KV +// on each vbucket stream. +message Scan { + shared.Collection collection = 1; + ScanType scan_type = 2; + optional ScanOptions options = 3; + + // Not sent on to the SDK - instead specifies how the performer should handle the streaming. + streams.Config stream_config = 4; + + // Controls how all ScanResult.contentAs() calls are done by the SDK for this scan. + // If not specified, the performer should not do contentAs. + // If there is any error from result.contentAs(), each error needs to raise a streams.Error, instead of the normal ScanResult. + optional shared.ContentAs content_as = 5; +} diff --git a/fit-performer/proto/sdk.query.index_manager.options.proto b/fit-performer/proto/sdk.query.index_manager.options.proto new file mode 100644 index 00000000..c88fbb4d --- /dev/null +++ b/fit-performer/proto/sdk.query.index_manager.options.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; + +package protocol.sdk.query.index_manager; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Query.IndexManager"; +option java_package = "com.couchbase.client.protocol.sdk.query.indexmanager"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/query/index/manager"; +option ruby_package = "FIT::Protocol::SDK::Query::IndexManager"; +option java_multiple_files = true; + +message GetAllQueryIndexOptions { + optional string scope_name = 1; + optional string collection_name = 2; + optional int32 timeout_msecs = 3; + optional string parent_span_id = 4; +} + +message CreatePrimaryQueryIndexOptions { + optional bool ignore_if_exists = 1; + optional int32 num_replicas = 2; + optional bool deferred = 3; + optional string index_name = 4; + optional string scope_name = 5; + optional string collection_name = 6; + optional int32 timeout_msecs = 7; + optional string parent_span_id = 8; +} + +message CreateQueryIndexOptions { + optional bool ignore_if_exists = 1; + optional int32 num_replicas = 2; + optional bool deferred = 3; + optional string scope_name = 4; + optional string collection_name = 5; + optional int32 timeout_msecs = 6; + optional string parent_span_id = 7; +} + +message DropPrimaryIndexOptions { + optional bool ignore_if_not_exists = 1; + optional string scope_name = 2; + optional string collection_name = 3; + optional int32 timeout_msecs = 4; + optional string parent_span_id = 5; +} + +message DropIndexOptions { + optional bool ignore_if_not_exists = 1; + optional string scope_name = 2; + optional string collection_name = 3; + optional int32 timeout_msecs = 4; + optional string parent_span_id = 5; +} + +message WatchIndexesOptions { + optional bool watch_primary = 1; + optional string scope_name = 2; + optional string collection_name = 3; + optional string parent_span_id = 4; +} + +message BuildDeferredIndexesOptions { + optional string scope_name = 1; + optional string collection_name = 2; + optional int32 timeout_msecs = 3; + optional string parent_span_id = 4; +} diff --git a/fit-performer/proto/sdk.query.index_manager.proto b/fit-performer/proto/sdk.query.index_manager.proto new file mode 100644 index 00000000..bde3a2ed --- /dev/null +++ b/fit-performer/proto/sdk.query.index_manager.proto @@ -0,0 +1,82 @@ +syntax = "proto3"; + +package protocol.sdk.query.index_manager; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Query.IndexManager"; +option java_package = "com.couchbase.client.protocol.sdk.query.indexmanager"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/query/index/manager"; +option ruby_package = "FIT::Protocol::SDK::Query::IndexManager"; +option java_multiple_files = true; + +import "sdk.query.index_manager.options.proto"; + + +message GetAllIndexes { + optional GetAllQueryIndexOptions options = 2; +} + +message CreatePrimaryIndex { + optional CreatePrimaryQueryIndexOptions options = 2; +} + +message CreateIndex { + string index_name = 2; + repeated string fields = 3; + optional CreateQueryIndexOptions options = 4; +} + +message DropPrimaryIndex { + optional DropPrimaryIndexOptions options = 2; +} + +message DropIndex { + string index_name = 2; + optional DropIndexOptions options = 3; +} + +message WatchIndexes { + repeated string index_names = 2; + int32 timeout_msecs = 3; + optional WatchIndexesOptions options = 4; +} + +message BuildDeferredIndexes { + optional BuildDeferredIndexesOptions options = 2; +} + +message QueryIndexes { + repeated QueryIndex indexes = 1; +} + +message QueryIndex { + string name = 1; + bool is_primary = 2; + QueryIndexType type = 3; + string state = 4; + string keyspace = 5; + repeated string index_key = 6; + optional string condition = 7; + optional string partition = 8; + string bucket_name = 9; + optional string scope_name = 10; + optional string collection_name = 11; +} + +enum QueryIndexType { + VIEW = 0; + GSI = 1; +} + +// These commands are shared between CollectionQueryIndexManager and QueryIndexManager. +// If API calls are added to just one of those, they should not go here. Add them in +// collection.query.index_manager.Command or cluster.query.index_manager.Command instead. +message Command { + oneof command { + CreatePrimaryIndex create_primary_index = 1; + CreateIndex create_index = 2; + GetAllIndexes get_all_indexes = 3; + DropPrimaryIndex drop_primary_index = 4; + DropIndex drop_index = 5; + WatchIndexes watch_indexes = 6; + BuildDeferredIndexes build_deferred_indexes = 7; + } +} diff --git a/fit-performer/proto/sdk.query.proto b/fit-performer/proto/sdk.query.proto new file mode 100644 index 00000000..c8339516 --- /dev/null +++ b/fit-performer/proto/sdk.query.proto @@ -0,0 +1,97 @@ +syntax = "proto3"; + +package protocol.sdk.query; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Query"; +option java_package = "com.couchbase.client.protocol.sdk.query"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/query"; +option ruby_package = "FIT::Protocol::SDK::Query"; +option java_multiple_files = true; + +import "shared.basic.proto"; +import "hooks.transactions.proto"; +import "google/protobuf/duration.proto"; +import "shared.content.proto"; + +message Command { + string statement = 1; + optional sdk.query.QueryOptions options = 2; + shared.ContentAs content_as = 3; +} + +message QueryResult { + repeated shared.ContentTypes content = 1; + QueryMetaData meta_data = 2; +} + +message QueryMetaData { + string request_id = 1; + string client_context_id = 2; + QueryStatus status = 3; + optional bytes signature = 4; + repeated QueryWarning warnings = 5; + optional QueryMetrics metrics = 6; + optional bytes profile = 7; +} + +enum QueryStatus { + RUNNING = 0; + SUCCESS = 1; + ERRORS = 2; + COMPLETED = 3; + STOPPED = 4; + TIMEOUT = 5; + CLOSED = 6; + FATAL = 7; + ABORTED = 8; + UNKNOWN = 9; +} + +message QueryWarning { + int32 code = 1; + string message = 2; +} + +message QueryMetrics { + google.protobuf.Duration elapsed_time = 1; + google.protobuf.Duration execution_time = 2; + uint64 sort_count = 3; + uint64 result_count = 4; + uint64 result_size = 5; + uint64 mutation_count = 6; + uint64 error_count = 7; + uint64 warning_count = 8; +} + +message QueryOptions { + optional shared.ScanConsistency scan_consistency = 1; + map< string, string > raw = 2; + optional bool adhoc = 3; + // "off", "phases", or "timings" + optional string profile = 4; + optional bool readonly = 5; + repeated string parameters_positional = 6; + map< string, string > parameters_named = 7; + optional bool flex_index = 8; + optional int32 pipeline_cap = 9; + optional int32 pipeline_batch = 10; + optional int32 scan_cap = 11; + optional int32 scan_wait_millis = 12; + optional int64 timeout_millis = 13; + optional int32 max_parallelism = 14; + optional bool metrics = 15; + optional SingleQueryTransactionOptions single_query_transaction_options = 16; + optional string parent_span_id = 17; + + optional bool use_replica = 18; + optional string client_context_id = 19; + optional shared.MutationState consistent_with = 20; + optional bool preserve_expiry = 21; +} + +// Ideally this would be in the transactions package, but that would introduce a Go package import cycle +message SingleQueryTransactionOptions { + // Override the durability level. + optional shared.Durability durability = 1; + + repeated hooks.transactions.Hook hook = 2; +} diff --git a/fit-performer/proto/sdk.scope.search.index_manager.proto b/fit-performer/proto/sdk.scope.search.index_manager.proto new file mode 100644 index 00000000..7d3004db --- /dev/null +++ b/fit-performer/proto/sdk.scope.search.index_manager.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package protocol.sdk.scope.search.index_manager; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Scope.Search.IndexManager"; +option java_package = "com.couchbase.client.protocol.sdk.scope.search.indexmanager"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/scope/search/indexmanager"; +option java_multiple_files = true; + +import "sdk.search.index_manager.proto"; + +// This file is for ScopeSearchIndexManager: scope.searchIndexes() +// For global search index manager (cluster.searchIndexes()), see sdk.search.index_manager.proto. + +message Command { + oneof command { + // Currently the SearchIndexManager and ScopeSearchIndexManager APIs are largely identical, and the intersection is + // represented by this shared message. + sdk.search.index_manager.Command shared = 1; + + // Any APIs added only to SearchIndexManager should go here. + } +} diff --git a/fit-performer/proto/sdk.search.index_manager.proto b/fit-performer/proto/sdk.search.index_manager.proto new file mode 100644 index 00000000..393c2151 --- /dev/null +++ b/fit-performer/proto/sdk.search.index_manager.proto @@ -0,0 +1,197 @@ +syntax = "proto3"; + +package protocol.sdk.search.index_manager; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Search.IndexManager"; +option java_package = "com.couchbase.client.protocol.sdk.search.indexmanager"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/search/index-manager"; +option ruby_package = "FIT::Protocol::SDK::Search::IndexManager"; +option java_multiple_files = true; + +// Messages shared by sdk.management.search.proto and sdk.management.scope.search.proto. +// This is how they are shared in the SDKs as well. + +message GetSearchIndexOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +message SearchIndex { + string uuid = 1; + string name = 2; + string type = 3; + string source_uuid = 5; + string source_type = 7; + + // It's left intentionally ambiguous in the SDK-RFC how to represent these in the SDK. + // For FIT, they should be converted into JSON. + bytes params = 4; + bytes source_params = 6; + bytes plan_params = 8; +} + +// Returns Result.index. +message GetIndex { + string index_name = 1; + optional GetSearchIndexOptions options = 2; +} + +// Returns Result.indexes. +message GetAllIndexes { + optional GetAllSearchIndexesOptions options = 1; +} + +message GetAllSearchIndexesOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +// Returns void (e.g. Result.success). +message UpsertIndex { + // A JSON blob that the performer should use to create a SearchIndex. + bytes index_definition = 1; + optional UpsertSearchIndexOptions options = 2; +} + +message UpsertSearchIndexOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +// Returns void (e.g. Result.success). +message DropIndex { + string index_name = 1; + optional DropSearchIndexOptions options = 2; +} + +message DropSearchIndexOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +// Returns Result.indexed_document_counts. +message GetIndexedDocumentsCount { + string index_name = 1; + optional GetIndexedSearchIndexOptions options = 2; +} + +message GetIndexedSearchIndexOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +// Returns void (e.g. Result.success). +message PauseIngest { + string index_name = 1; + optional PauseIngestSearchIndexOptions options = 2; +} + +message PauseIngestSearchIndexOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +// Returns void (e.g. Result.success). +message ResumeIngest { + string index_name = 1; + optional ResumeIngestSearchIndexOptions options = 2; +} + +message ResumeIngestSearchIndexOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +// Returns void (e.g. Result.success). +message AllowQuerying { + string index_name = 1; + optional AllowQueryingSearchIndexOptions options = 2; +} + +message AllowQueryingSearchIndexOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +// Returns void (e.g. Result.success). +message DisallowQuerying { + string index_name = 1; + optional DisallowQueryingSearchIndexOptions options = 2; +} + +message DisallowQueryingSearchIndexOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +// Returns void (e.g. Result.success). +message FreezePlan { + string index_name = 1; + optional FreezePlanSearchIndexOptions options = 2; +} + +message FreezePlanSearchIndexOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +// Returns void (e.g. Result.success). +message UnfreezePlan { + string index_name = 1; + optional UnfreezePlanSearchIndexOptions options = 2; +} + +message UnfreezePlanSearchIndexOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +// Returns Result.analyze_document; +message AnalyzeDocument { + string index_name = 1; + bytes document = 2; + optional AnalyzeDocumentOptions options = 3; +} + +message AnalyzeDocumentOptions { + optional int32 timeout_msecs = 1; + optional string parent_span_id = 2; +} + +message AnalyzeDocumentResult { + repeated bytes results = 1; +} + +message SearchIndexes { + repeated SearchIndex indexes = 1; +} + +message Result { + oneof result { + // Used for void results. + bool success = 1; + SearchIndex index = 2; + SearchIndexes indexes = 3; + int32 indexed_document_counts = 4; + AnalyzeDocumentResult analyze_document = 5; + } +} + +// These commands are shared between SearchIndexManager and ScopeSearchIndexManager. +// If API calls are added to just one of those, they should not go here. Add them in +// scope.search.index_manager.Command or cluster.search.index_manager.Command instead. +message Command { + oneof command { + GetIndex get_index = 1; + GetAllIndexes get_all_indexes = 2; + UpsertIndex upsert_index = 3; + DropIndex drop_index = 4; + GetIndexedDocumentsCount get_indexed_documents_count = 5; + PauseIngest pause_ingest = 6; + ResumeIngest resume_ingest = 7; + AllowQuerying allow_querying = 8; + DisallowQuerying disallow_querying = 9; + FreezePlan freeze_plan = 10; + UnfreezePlan unfreeze_plan = 11; + AnalyzeDocument analyze_document = 12; + } +} diff --git a/fit-performer/proto/sdk.search.proto b/fit-performer/proto/sdk.search.proto new file mode 100644 index 00000000..e65a6e12 --- /dev/null +++ b/fit-performer/proto/sdk.search.proto @@ -0,0 +1,489 @@ +syntax = "proto3"; + +package protocol.sdk.search; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk.Search"; +option java_package = "com.couchbase.client.protocol.sdk.search"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk/search"; +option java_multiple_files = true; +option ruby_package = "FIT::Protocol::SDK::Search"; + +import "shared.content.proto"; +import "shared.basic.proto"; +import "google/protobuf/timestamp.proto"; +import "streams.top_level.proto"; +import "shared.transcoders.proto"; + +// Executing either a `cluster.search()` or `scope.search()` FTS query. +// This is the new API, replacing `cluster.searchQuery()`, that also supports vector search. +// +// The performer can return one of these 3: +// +// 1. An protocol.sdk.Result.exception, if the `cluster/scope.search()` call immediately fails. +// +// 2. A BlockingSearchResult, but only if that's genuinely the most natural representation. For example, the Java SDK's +// blocking API would return this. The performer should not select this just because it's easier to +// implement than the stream! Supporting streams allows more sophisticated FIT tests to be performed. +// Most SDK APIs have streaming responses and should instead use: +// +// 3. A stream. This consists of streaming back: +// 3.1 A streams.Created, with type STREAM_FULL_TEXT_SEARCH. +// 3.2 Zero+ StreamingSearchResults, which contain the FTS rows, facets and metadata. +// 3.3 A streams.Complete. +// And as usual the performer should handle requests for more items, stream cancellation requests, and +// look at stream_config for details of how to stream. +// See streams.top_level.proto for details. +// If any error occurs on the stream (as opposed to from the initial cluster/scope.search() call) +// as usual return a streams.Error. +// Note the same types are used as for `cluster/scope.searchQuery()`, reflecting the API. +message SearchWrapper { + // Represents the SDK search. + SearchV2 search = 1; + + // Specifies how the performer should convert SearchRow.fields. If not present, the performer should not do result.fieldsAs(). + optional protocol.shared.ContentAs fields_as = 2; + + // Specifies how the performer should handle any streaming. + // Can be ignored if the performer is returning a BlockingSearchResult rather than a stream. + streams.Config stream_config = 3; +} + +// Note that "V2" naming should not be used anywhere in the SDK itself for this feature. It's purely to workaround +// having an existing "Search" wrapper message in the GRPC. +message SearchV2 { + string indexName = 1; + SearchRequest request = 2; + optional SearchOptions options = 3; +} + +// At least one of these will usually be specified. If both are not specified, the SDK should simulate the user +// not providing either - e.g. providing null or similar for both. +message SearchRequest { + optional SearchQuery search_query = 1; + optional VectorSearch vector_search = 2; +} + +message VectorQuery { + repeated float vector_query = 1; + string vector_field_name = 2; + optional VectorQueryOptions options = 3; + optional string base64_vector_query = 4; +} + +message VectorQueryOptions { + optional int32 num_candidates = 1; + optional float boost = 2; + optional SearchQuery prefilter = 3; +} + +message VectorSearch { + repeated VectorQuery vector_query = 1; + optional VectorSearchOptions options = 2; +} + +message VectorSearchOptions { + optional VectorQueryCombination vector_query_combination = 1; +} + +enum VectorQueryCombination { + AND = 0; + OR = 1; +} + + +// Executing a `cluster.searchQuery()` FTS query. +// +// The performer can return either: +// +// 1. An protocol.sdk.Result.exception, if the `cluster/scope.searchQuery()` call immediately fails. +// +// 2. A BlockingSearchResult, iff that's the most natural representation. For example, the Java SDK's +// blocking API would return this. The performer should not select this just because it's easier to +// implement than the stream! +// Most SDK APIs have streaming responses and should instead use: +// +// 3. A stream. This consists of streaming back: +// 3.1 A streams.Created, with type STREAM_FULL_TEXT_SEARCH. +// 3.2 Zero+ StreamingSearchResults, which contain the FTS rows, facets and metadata. +// 3.3 A streams.Complete. +// And as usual the performer should handle requests for more items, stream cancellation requests, and +// look at stream_config for details of how to stream. +// See streams.top_level.proto for details. +// If any error occurs on the stream (as opposed to from the initial cluster.search() call) +// as usual return a streams.Error. +message Search { + string indexName = 1; + SearchQuery query = 2; + optional SearchOptions options = 3; + + // Controls how SearchRow.fields should be converted. Only applied if a BlockingSearchResult is being returned. + // If not present, the performer should not do result.fieldsAs(). + optional protocol.shared.ContentAs fields_as = 4; + + // Not sent on to the SDK - instead specifies how the performer should handle any streaming. + // Can be ignored if the performer is choosing to return a BlockingSearchResult rather than a stream. + streams.Config stream_config = 5; +} + +message SearchFragments { + repeated string fragments = 1; +} + +message SearchRowLocation { + string field = 1; + string term = 2; + uint32 position = 3; + uint32 start = 4; + uint32 end = 5; + repeated uint32 array_positions = 6; +} + +message SearchRow { + string index = 1; + string id = 2; + double score = 3; + bytes explanation = 4; + // The result of calling result.locations.getAll(). + // If result.locations is missing, return nothing here. + repeated SearchRowLocation locations = 5; + map fragments = 6; + shared.ContentTypes fields = 7; +} + +message SearchMetrics { + int64 took_msec = 1; + int64 total_rows = 2; + double max_score = 3; + int64 total_partition_count = 4; + int64 success_partition_count = 5; + int64 error_partition_count = 6; +} + +message SearchMetaData { + SearchMetrics metrics = 1; + map errors = 2; +} + +message SearchFacetResult { + string name = 1; + string field = 2; + uint64 total = 3; + uint64 missing = 4; + uint64 other = 5; +} + +message SearchFacets { + map facets = 1; +} + +message BlockingSearchResult { + repeated SearchRow rows = 1; + SearchFacets facets = 2; + SearchMetaData meta_data = 3; +} + +message StreamingSearchResult { + // Not part of the SDK response - identifies what stream this is from. + string stream_id = 1; + + oneof result { + // Driver expects to see 0+ of these of stream. + SearchRow row = 2; + // Driver expects to see 0 or 1 of these, after all rows. + SearchFacets facets = 3; + // Driver expects to see exactly 1 of these, after all rows and facets. (Unless the stream errors.) + SearchMetaData meta_data = 4; + } +} + +enum HighlightStyle { + HIGHLIGHT_STYLE_HTML = 0; + HIGHLIGHT_STYLE_ANSI = 2; +} + +message Highlight { + optional HighlightStyle style = 1; + // There is no `optional repeated`, so if this field is empty, regard it as not provided. + // E.g. we cannot test providing sending an empty list of fields into the SDK. + repeated string fields = 2; +} + +enum SearchScanConsistency { + SEARCH_SCAN_CONSISTENCY_NOT_BOUNDED = 0; +} + +message SearchSortScore { + optional bool desc = 1; +} + +message SearchSortId { + optional bool desc = 1; +} + +message SearchSortField { + optional bool desc = 1; + string field = 2; + // possible values are "auto", "string", "number", "date" + optional string type = 3; + // possible values are "default", "min", "max" + optional string mode = 4; + // possible values are "first", "last" + optional string missing = 5; +} + +enum SearchGeoDistanceUnits { + SEARCH_GEO_DISTANCE_UNITS_METERS = 0; + SEARCH_GEO_DISTANCE_UNITS_MILES = 1; + SEARCH_GEO_DISTANCE_UNITS_CENTIMETERS = 2; + SEARCH_GEO_DISTANCE_UNITS_MILLIMETERS = 3; + SEARCH_GEO_DISTANCE_UNITS_NAUTICAL_MILES = 4; + SEARCH_GEO_DISTANCE_UNITS_KILOMETERS = 5; + SEARCH_GEO_DISTANCE_UNITS_FEET = 6; + SEARCH_GEO_DISTANCE_UNITS_YARDS = 7; + SEARCH_GEO_DISTANCE_UNITS_INCHES = 8; +} + +message Location { + float lon = 3; + float lat = 4; +} + +message SearchSortGeoDistance { + optional bool desc = 1; + string field = 2; + Location location = 3; + optional SearchGeoDistanceUnits unit = 4; +} + +message SearchSort { + oneof sort { + SearchSortScore score = 1; + SearchSortId id = 2; + SearchSortField field = 3; + SearchSortGeoDistance geo_distance = 4; + string raw = 5; + } +} + +message TermFacet { + string field = 1; + optional uint32 size = 2; +} + +message NumericRange { + string name = 1; + optional float min = 2; + optional float max = 3; +} + +message NumericRangeFacet { + string field = 1; + optional uint32 size = 2; + repeated NumericRange numeric_ranges = 3; +} + +message DateRange { + string name = 1; + optional google.protobuf.Timestamp start = 2; + optional google.protobuf.Timestamp end = 3; +} + +message DateRangeFacet { + string field = 1; + optional uint32 size = 2; + repeated DateRange date_ranges = 3; +} + +message SearchFacet { + oneof facet { + TermFacet term = 1; + NumericRangeFacet numeric_range = 2; + DateRangeFacet date_range = 3; + } +} + +message SearchOptions { + optional uint32 limit = 1; + optional uint32 skip = 2; + optional bool explain = 3; + optional Highlight highlight = 4; + // There is no `optional repeated`, so if this field is empty, regard it as not provided. + repeated string fields = 5; + optional SearchScanConsistency scan_consistency = 6; + optional protocol.shared.MutationState consistent_with = 7; + // There is no `optional repeated`, so if this field is empty, regard it as not provided. + repeated SearchSort sort = 8; + // There is no `optional map`, so if this field is empty, regard it as not provided. + map facets = 9; + optional int64 timeout_millis = 10; + optional string parent_span_id = 11; + map raw = 12; + optional bool include_locations = 13; + optional protocol.shared.JsonSerializer serialize = 14; +} + +enum MatchOperator { + SEARCH_MATCH_OPERATOR_OR = 0; + SEARCH_MATCH_OPERATOR_AND = 1; +} + +message MatchQuery { + string match = 1; + optional string field = 2; + optional string analyzer = 3; + optional uint32 prefix_length = 4; + optional uint32 fuzziness = 5; + optional float boost = 6; + optional MatchOperator operator = 7; +} + +message MatchPhraseQuery { + string match_phrase = 1; + optional string field = 2; + optional string analyzer = 3; + optional float boost = 4; +} + +message RegexpQuery { + string regexp = 1; + optional string field = 2; + optional float boost = 3; +} + +message QueryStringQuery { + string query = 1; + optional float boost = 2; +} + +message WildcardQuery { + string wildcard = 1; + optional string field = 2; + optional float boost = 3; +} + +message DocIdQuery { + repeated string ids = 1; + optional float boost = 2; +} + +message BooleanFieldQuery { + bool bool = 1; + optional string field = 2; + optional float boost = 3; +} + +message DateRangeQuery { + optional string start = 1; + optional bool inclusive_start = 2; + optional string end = 3; + optional bool inclusive_end = 4; + optional string datetime_parser = 5; + optional string field = 6; + optional float boost = 7; +} + +message NumericRangeQuery { + optional float min = 1; + optional bool inclusive_min = 2; + optional float max = 3; + optional bool inclusive_max = 4; + optional string field = 5; + optional float boost = 6; +} + +message TermRangeQuery { + optional string min = 1; + optional bool inclusive_min = 2; + optional string max = 3; + optional bool inclusive_max = 4; + optional string field = 5; + optional float boost = 6; +} + +message GeoDistanceQuery { + Location location = 1; + string distance = 2; + optional string field = 3; + optional float boost = 4; +} + +message GeoBoundingBoxQuery { + Location top_left = 1; + Location bottom_right = 2; + optional string field = 3; + optional float boost = 4; +} + +message ConjunctionQuery { + repeated SearchQuery conjuncts = 1; + optional float boost = 2; +} + +message DisjunctionQuery { + repeated SearchQuery disjuncts = 1; + optional uint32 min = 2; + optional float boost = 3; +} + +message BooleanQuery { + // Note these are all guaranteed to be ConjunctionQuerys. SearchQuery is sent + // instead as it makes it much easier to write recursive handling code in the performer. + repeated SearchQuery must = 1; + // All guaranteed to be DisjunctionQuerys. + repeated SearchQuery should = 2; + optional uint32 should_min = 3; + // All guaranteed to be DisjunctionQuerys. + repeated SearchQuery must_not = 4; + optional float boost = 5; +} + +message TermQuery { + string term = 1; + optional string field = 2; + optional uint32 fuzziness = 3; + optional uint32 prefix_length = 4; + optional float boost = 5; +} + +message PrefixQuery { + string prefix = 1; + optional string field = 2; + optional float boost = 3; +} + +message PhraseQuery { + repeated string terms = 1; + optional string field = 2; + optional float boost = 3; +} + +message MatchAllQuery { +} + +message MatchNoneQuery { +} + +message SearchQuery { + oneof query { + MatchQuery match = 1; + MatchPhraseQuery match_phrase = 2; + RegexpQuery regexp = 3; + QueryStringQuery query_string = 4; + WildcardQuery wildcard = 5; + DocIdQuery doc_id = 6; + // Cannot be named boolean_field as it causes issues in generated code for at least Java + BooleanFieldQuery search_boolean_field = 7; + DateRangeQuery date_range = 8; + NumericRangeQuery numeric_range = 9; + TermRangeQuery term_range = 20; + GeoDistanceQuery geo_distance = 10; + GeoBoundingBoxQuery geo_bounding_box = 11; + ConjunctionQuery conjunction = 12; + DisjunctionQuery disjunction = 13; + BooleanQuery boolean = 14; + TermQuery term = 15; + PrefixQuery prefix = 16; + PhraseQuery phrase = 17; + MatchAllQuery match_all = 18; + MatchNoneQuery match_none = 19; + } +} diff --git a/fit-performer/proto/sdk.workload.proto b/fit-performer/proto/sdk.workload.proto new file mode 100644 index 00000000..330fc5a7 --- /dev/null +++ b/fit-performer/proto/sdk.workload.proto @@ -0,0 +1,237 @@ +syntax = "proto3"; + +package protocol.sdk; +option csharp_namespace = "Couchbase.Grpc.Protocol.Sdk"; +option java_package = "com.couchbase.client.protocol.sdk"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/sdk"; +option ruby_package = "FIT::Protocol::SDK"; +option java_multiple_files = true; + +import "shared.bounds.proto"; +import "shared.cluster.proto"; +import "shared.exceptions.proto"; +import "sdk.kv.rangescan.top_level.proto"; +import "sdk.kv.commands.proto"; +import "sdk.kv.binary.commands.proto"; +import "shared.basic.proto"; +import "sdk.query.index_manager.proto"; +import "sdk.cluster.query.index_manager.proto"; +import "sdk.collection.query.index_manager.proto"; +import "shared.collection.proto"; +import "sdk.search.index_manager.proto"; +import "sdk.scope.search.index_manager.proto"; +import "sdk.cluster.search.index_manager.proto"; +import "sdk.search.proto"; +import "sdk.cluster.wait_until_ready.proto"; +import "sdk.cluster.bucket_manager.proto"; +import "sdk.cluster.eventing_function_manager.proto"; +import "sdk.query.proto"; +import "sdk.kv.lookup_in.proto"; +import "sdk.kv.mutate_in.proto"; +import "sdk.bucket.collection_manager.proto"; + +message ClusterLevelCommand { + oneof command { + // QueryIndexManager: cluster.queryIndexes() + cluster.query.index_manager.Command query_index_manager = 1; + + // cluster.searchQuery() - original FTS API + search.Search search = 2; + + // cluster.search() - updated FTS API + search.SearchWrapper search_v2 = 8; + + // SearchIndexManager: cluster.searchIndexes() + cluster.search.index_manager.Command search_index_manager = 3; + + cluster.wait_until_ready.WaitUntilReadyRequest wait_until_ready = 4; + + cluster.bucket_manager.Command bucket_manager = 5; + + cluster.eventing_function_manager.Command eventing_function_manager = 6; + + sdk.query.Command query = 7; + + // cluster.authenticator() + shared.Authenticator authenticator = 9; + } +} + +message BucketLevelCommand { + string bucket_name = 1; + + oneof command { + cluster.wait_until_ready.WaitUntilReadyRequest wait_until_ready = 2; + + bucket.collection_manager.Command collection_manager = 3; + } +} + +message ScopeLevelCommand { + protocol.shared.Scope scope = 1; + + oneof command { + // scope.search() - updated FTS API + search.SearchWrapper search_v2 = 5; + + // ScopeSearchIndexManager: scope.searchIndexes() + scope.search.index_manager.Command search_index_manager = 3; + + sdk.query.Command query = 4; + + // scope.searchQuery() - original FTS API + // Now deprecated as revision 10 of the RFC removes this (it was only implemented by one SDK). + // The SDK should implement only scope.search(), not scope.searchQuery(). + // FIT will not use this field. + search.Search search = 2 [deprecated = true]; + } +} + +// If the command specifies a docLocation use the collection within it, otherwise fall back on +// CollectionLevelCommand.collection. +message CollectionLevelCommand { + + // Should be passed in any collection level command that does not have a Location in the message for the command + optional protocol.shared.Collection collection = 1; + + oneof command { + // CollectionQueryIndexManager: collection.queryIndexes() + collection.query.index_manager.Command query_index_manager = 16; + + kv.lookup_in.LookupIn lookup_in = 17; + + kv.lookup_in.LookupInAllReplicas lookup_in_all_replicas = 18; + + kv.mutate_in.MutateIn mutate_in = 20; + + kv.GetAndLock get_and_lock = 21; + + kv.Unlock unlock = 25; + + kv.GetAndTouch get_and_touch = 22; + + kv.Exists exists = 23; + + kv.Touch touch = 24; + + kv.GetAnyReplica get_any_replica = 30; + + kv.GetAllReplicas get_all_replicas = 31; + + kv.lookup_in.LookupInAnyReplica lookup_in_any_replica = 19; + + BinaryCollectionLevelCommand binary = 32; + } +} + +// If the command specifies a docLocation use the collection within it, otherwise fall back on +// CollectionLevelCommand.collection. +message BinaryCollectionLevelCommand { + oneof command { + kv.Append append = 1; + + kv.Prepend prepend = 2; + + kv.Increment increment = 3; + + kv.Decrement decrement = 4; + } +} + +message Command { + oneof command { + // KV commands. These would be added to CollectionLevelCommand if added now. + kv.Insert insert = 1; + kv.Get get = 2; + kv.GetOrNull getOrNull = 9; + kv.Remove remove = 3; + kv.Replace replace = 4; + kv.Upsert upsert = 5; + kv.rangescan.Scan range_scan = 7; + + // With Serverless making it increasingly common that we have similar APIs at Cluster, Scope and/or Collection levels, + // it's becoming more important to separate them. GRPC added from now on should go into one of these four. + ClusterLevelCommand cluster_command = 17; + BucketLevelCommand bucket_command = 18; + ScopeLevelCommand scope_command = 19; + CollectionLevelCommand collection_command = 20; + } + + // Whether the returned CommandResult should contain a full result or just `success`. Used for performance tests + // when the result doesn't matter so the small cost of creating it can be eliminated. + // It only affects successful results. Failure should report the full exception, as that continues to be useful in + // performance tests. + bool return_result = 6; + + // The API to use to execute this command. + protocol.shared.API api = 8; +} + +// The result of executing a single sdk.Command. +message Result { + oneof result { + // If the small overhead of creating a full *Result is not required (e.g. for performance testing), can + // return this generic success instead. Controlled by SdkCommand.returnResult. + bool success = 1; + + // If the operation failed. + protocol.shared.Exception exception = 2; + + // If the SDK return null/none/undefined. + // Nb the SDK should very rarely do this! It is reserved for cases like getOrNull. + bool null_result = 21; + + // If the operation succeeded and sdk.Command.returnResult==true, return one of these. + kv.GetResult get_result = 3; + kv.MutationResult mutation_result = 4; + kv.rangescan.ScanResult range_scan_result = 5; + kv.ExistsResult exists_result = 16; + kv.CounterResult counter_result = 17; + kv.GetReplicaResult get_replica_result = 18; + kv.mutate_in.MutateInResult mutate_in_result = 19; + + // List of indexes returned by a GetAllIndexes request + query.index_manager.QueryIndexes query_indexes = 6; + + // Search results. + search.BlockingSearchResult search_blocking_result = 7; + search.StreamingSearchResult search_streaming_result = 8; + search.index_manager.Result search_index_manager_result = 9; + + //Bucket manager results + cluster.bucket_manager.Result bucket_manager_result = 10; + + //Eventing function manager results + cluster.eventing_function_manager.Result eventing_function_manager_result = 11; + + //Query results + query.QueryResult query_result = 12; + + // LookupIn results + kv.lookup_in.LookupInResult lookup_in_result = 13; + kv.lookup_in.LookupInAllReplicasResult lookup_in_all_replicas_result = 14; + kv.lookup_in.LookupInReplicaResult lookup_in_any_replica_result = 15; + + //Collection manager results + bucket.collection_manager.Result collection_manager_result = 20; + } +} + +message Workload { + // The commands to run. The performer should execute these in a loop until `bounds` is completed. + // + // If there are multiple commands, each should register as separate command for the purposes of bounding, and each + // should send back one CommandResult. + // + // For example, if the bounds specifies + // to run 10 commands total, and there are 3 Command specified here, the performer should execute the 3 commands + // 3 times, and then the first command once more. And 10 sdk.Results will be sent back. + // + // If the command fails, the performer should not propagate the failure (but should still send back an sdk.Result). + repeated Command command = 1; + + // Controls how the commands should be run. + // If it's not present, just run the commands through once. + optional protocol.shared.Bounds bounds = 2; +} + diff --git a/fit-performer/proto/shared.basic.proto b/fit-performer/proto/shared.basic.proto new file mode 100644 index 00000000..ae72fa9f --- /dev/null +++ b/fit-performer/proto/shared.basic.proto @@ -0,0 +1,95 @@ +syntax = "proto3"; + + +package protocol.shared; +option csharp_namespace = "Couchbase.Grpc.Protocol.Shared"; +option java_package = "com.couchbase.client.protocol.shared"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/shared"; +option ruby_package = "FIT::Protocol::Shared"; +option java_multiple_files = true; + +enum PersistTo { + PERSIST_TO_NONE = 0; + PERSIST_TO_ACTIVE = 1; + PERSIST_TO_ONE = 2; + PERSIST_TO_TWO = 3; + PERSIST_TO_THREE = 4; + PERSIST_TO_FOUR = 5; +} + +enum ReplicateTo { + REPLICATE_TO_NONE = 0; + REPLICATE_TO_ONE = 1; + REPLICATE_TO_TWO = 2; + REPLICATE_TO_THREE = 3; +} + +message Expiry { + oneof expiryType { + // The document will expire after the given duration. + int64 relativeSecs = 1; + // The document will expire at a specific time. Given in epoch time. + int64 absoluteEpochSecs = 2; + } +} + +message ObserveBased { + PersistTo persistTo = 1; + ReplicateTo replicateTo = 2; +} + + +enum Durability { + NONE = 0; + MAJORITY = 1; + MAJORITY_AND_PERSIST_TO_ACTIVE = 2; + PERSIST_TO_MAJORITY = 3; +} + +// Awkward name as Durability already taken +message DurabilityType { + oneof durability { + Durability durabilityLevel = 1; + ObserveBased observe = 2; + } +} + +enum ScanConsistency { + REQUEST_PLUS = 0; + NOT_BOUNDED = 1; +} + +message MutationToken { + // This is a short really, but GRPC doesn't have int16 + int32 partition_id = 1; + int64 partition_uuid = 2; + int64 sequence_number = 3; + string bucket_name = 4; +} + +message MutationState { + repeated MutationToken tokens = 1; +} + + +// When an implementation supports multiple APIs, this enum allows sending which is preferred to be used for a given test. +// This is an abstraction that allows us to test e.g. Java's blocking vs reactive APIs. +// +// Declaration in supportedApis: +// For implementations with just one API: Only declare DEFAULT. +// For implementations with separate blocking & async APIs: declare both. +// +// When the performer receives a preferred API: +// For implementations with just one API: ignore it. +// For implementations with separate blocking & async APIs: use the appropriate one. +enum API { + DEFAULT = 0; + ASYNC = 1; +} + +// Zone Aware Read from Replica options +enum ReadPreference { + NO_PREFERENCE = 0; + SELECTED_SERVER_GROUP = 1; + SELECTED_SERVER_GROUP_OR_ALL_AVAILABLE = 2; +} \ No newline at end of file diff --git a/fit-performer/proto/shared.bounds.proto b/fit-performer/proto/shared.bounds.proto new file mode 100644 index 00000000..5a3403dd --- /dev/null +++ b/fit-performer/proto/shared.bounds.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +package protocol.shared; +option csharp_namespace = "Couchbase.Grpc.Protocol.Shared"; +option java_package = "com.couchbase.client.protocol.shared"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/shared"; +option ruby_package = "FIT::Protocol::Shared"; +option java_multiple_files = true; + +// We may need the flexibility to do various forms of counter, e.g.: +// 1. Aim to do 1m operations across all threads. +// 2. Each thread does 1m operations. +// (1) is the simplest and most useful, so is the only one supported for now. And `counterId` lets us do (2) with it anyway. +message CounterGlobal { + // The performer should initialise the counter at this value, and will generally keep going until the counter reaches 0 + int32 count = 1; +} + +message Counter { + // Each counter is identified by its id, which the performer creates on-demand. + // Counters with the same id are shared across workloads and WorkloadRunRequests. Use unique ids + // when you want isolated counters (e.g. per-workload count bounds), and shared ids when you + // want a single counter to coordinate multiple workloads (e.g. counter_eq bounds with setCounter). + string counter_id = 1; + + oneof counter { + CounterGlobal global = 2; + } +} + +// The performer will run the bounded item until this much time has elapsed. +message ForTime { + int32 seconds = 1; +} + +// Controls how a given workload is bounded. +// A simple bounding would be to just run X operations as fast as possible, and then stop. +// In future we will support more advanced bounds, such as the performer maintaining a target throughput of 100 ops/sec and +// seeing what the latency is, for 60 seconds. +message Bounds { + oneof bounds { + // The performer decrements the counter and continues iff counter is now > 0 + Counter counter = 1; + ForTime for_time = 2; + // The performer should not change the counter, just continue iff its equal to the initial value + Counter counter_eq = 3; + + } +} + +// Empty but included anyway per GRPC best practices. +message SetCounterResponse { + +} + +// Clear all counters for a given performer. +message ClearAllCountersRequest { + +} + +// Empty but included anyway per GRPC best practices. +message ClearAllCountersResponse { + +} \ No newline at end of file diff --git a/fit-performer/proto/shared.cluster.proto b/fit-performer/proto/shared.cluster.proto new file mode 100644 index 00000000..38561ea5 --- /dev/null +++ b/fit-performer/proto/shared.cluster.proto @@ -0,0 +1,204 @@ +syntax = "proto3"; + +package protocol.shared; +option csharp_namespace = "Couchbase.Grpc.Protocol.Shared"; +option java_package = "com.couchbase.client.protocol.shared"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/shared"; +option ruby_package = "FIT::Protocol::Shared"; +option java_multiple_files = true; + +import "shared.transaction_config.proto"; +import "shared.transcoders.proto"; +import "observability.config.proto"; +import "sdk.circuit_breaker.config.proto"; + +message ClusterConfig { + // Will only be sent to ExtSDKIntegration-enabled performers. + optional TransactionsConfig transactions_config = 1; + + // The performer should use a custom JsonSerializer when creating the Cluster. + // This custom JsonSerializer is required to have specific serialization and deserialization logic, which the driver + // will validate. See the Java performer for implementation details. + bool use_custom_serializer = 2; + + // Used for TLS enabled clusters such as Capella clusters. + // Note that use_tls should only be used by SDKs that have such a field (e.g. Java). Many SDKs do not have a + // use_tls equivalent, and TLS is enabled via couchbases:// on the connection string. The driver will use the + // appropriate option. + bool use_tls = 3; + + // If specified, it's the path to an certificate in PEM format, that the SDK should apply in whatever way makes sense + // for it. + // If `cert_path` is specified, `cert` will not be. (They are not `oneof`, for backwards compatibility). + // Note that now FIT is mainly tested through Docker, `cert_path` is generally not usable since it's non-trivial + // to pass files between Docker containers. `cert` should generally be used instead. + optional string cert_path = 4; + + // If specified, it's an X509 certificate in PEM format, e.g. + // + //-----BEGIN CERTIFICATE----- + //MIIDKDCCAhCgAwIBAgIGAYfmPG2mMA0GCSqGSIb3DQEBCwUAMDYxNDAyBgNVBAMT + //K2FwcHMuY2xvdWQtbmF0aXZlLmZnMWIucDEub3BlbnNoaWZ0YXBwcy5jb20wHhcN + //... + //I+VkRC5NwrqMsnR3149LAVcj6Jen8hMjLM0wR8Frqu2zV9HW5yHMQmOTOKE= + //-----END CERTIFICATE----- + // + // The SDK should apply it in whatever way makes sense for it. Some SDKs (e.g. the C++ wrappers) cannot pass + // in such a certificate directly to the SDK. In this case they should write it to a temporary file first. + // + // If `cert_path` is specified, `cert` will not be. (They are not `oneof`, for backwards compatibility). + // `cert` can be used where it's not easy to pass `cert_path`, such as when the driver and performer are on + // separate nodes or Docker containers. + optional string cert = 26; + + // The SDK should skip certificate validation. It's SDK dependent how to implement this. On JVM it may involve + // applying InsecureTrustManager, while on a C++ wrapper it may require manipulating the connstr to pass "?tls_verify=none". + // Not every SDK can support this option against every backend. See internal discussion: + // https://couchbase.slack.com/archives/G9682CWN7/p1698333449931709 + // Behind cap CLUSTER_CONFIG_INSECURE. + optional bool insecure = 27; + + // Following taken from: + // https://github.com/couchbaselabs/sdk-rfcs/blob/master/rfc/0059-sdk3-foundation.md#configuration + optional int32 kv_connect_timeout_secs = 5; + optional int32 kv_timeout_millis = 6; + optional int32 kv_durable_timeout_millis = 7; + optional int32 view_timeout_secs = 8; + optional int32 query_timeout_secs = 9; + optional int32 analytics_timeout_secs = 10; + optional int32 search_timeout_secs = 11; + optional int32 management_timeout_secs = 12; + optional Transcoder transcoder = 13; + optional bool enable_mutation_tokens = 14; + optional int32 tcp_keep_alive_time_millis = 15; + optional bool enable_tcp_keep_alives = 16; + optional bool force_i_p_v4 = 17; + optional int32 config_poll_interval_secs = 18; + optional int32 config_poll_floor_interval_secs = 19; + optional int32 config_idle_redial_timeout_secs = 20; + optional int32 num_kv_connections = 21; + optional int32 max_http_connections = 22; + optional int32 idle_http_connection_timeout_secs = 23; + + // Whether some sort of observability (e.g. OpenTelemetry) output should be configured. The performer should pass this in to the cluster + // config's `requestTracer` and/or `meter` settings (depending on whether tracing and/or metrics is enabled). + // + // These will generally create OpenTelemetry SDK objects, e.g. SdkTracerProvider and SdkMeterProvider. + // These must be closed at the same time as this cluster connection is closed. Otherwise the SdkMeterProvider will + // continue outputting metrics until the application ends. + optional observability.Config observability_config = 24; + + optional int32 kv_scan_timeout_secs = 25; + + optional sdk.circuit_breaker.Config circuit_breaker_config = 28; + + optional string preferred_server_group = 29; + + // A Application telemetry endpoint WebSocket URI of the form "ws://:/" + optional string app_telemetry_endpoint = 30; + optional int32 app_telemetry_backoff_secs = 31; + optional bool enable_app_telemetry = 32; + optional int32 app_telemetry_ping_interval_secs = 33; + optional int32 app_telemetry_ping_timeout_secs = 34; +} + +message Authenticator { + message PasswordAuthenticator { + string username = 1; + string password = 2; + } + + message CertificateAuthenticator { + // An X509 certificate in PEM format, e.g. + // + //-----BEGIN CERTIFICATE----- + //MIIDKDCCAhCgAwIBAgIGAYfmPG2mMA0GCSqGSIb3DQEBCwUAMDYxNDAyBgNVBAMT + //K2FwcHMuY2xvdWQtbmF0aXZlLmZnMWIucDEub3BlbnNoaWZ0YXBwcy5jb20wHhcN + //... + //I+VkRC5NwrqMsnR3149LAVcj6Jen8hMjLM0wR8Frqu2zV9HW5yHMQmOTOKE= + //-----END CERTIFICATE----- + // + // The SDK should apply it in whatever way makes sense for it. Some SDKs (e.g. the C++ wrappers) cannot pass + // in such a certificate directly to the SDK. In this case they should write it to a temporary file first. + string cert = 1; + + // An X509 key in PEM format, e.g. + // + //-----BEGIN RSA PRIVATE KEY----- + //MIIJKAIBAAKCAgEAwiYPcb0zL9B5ZCn5mo/sF2QwdNr/aKjdfvnO6hx9h80klu5m + //... + //QZ57rjd7D6hDSzbSxNW54ktF0ZJrBZatfXJize2/WL1SzSAkm7rJovUWXKA= + //-----END RSA PRIVATE KEY----- + // + // The SDK should apply it in whatever way makes sense for it. Some SDKs (e.g. the C++ wrappers) cannot pass + // in such a certificate directly to the SDK. In this case they should write it to a temporary file first. + string key = 2; + } + + message JwtAuthenticator { + string jwt = 1; + } + + oneof authenticator { + PasswordAuthenticator password_auth = 1; + CertificateAuthenticator certificate_auth = 2; + JwtAuthenticator jwt_auth = 3; + } +} + +// Only sent to ExtSDKIntegration-supporting performers. +// Creates a cluster connection, with an optional cluster & transactions configuration. +message ClusterConnectionCreateRequest { + // The id to use for this connection. + string cluster_connection_id = 1; + + // Details of the cluster connection. + // cluster_hostname is the connection string. As with all fields (see rule 1 in README.md), the performer should + // pass this directly to the SDK without manipulation. E.g. it should not prepend couchbase:// or similar. + string cluster_hostname = 2; + + // Either these two fields will be set or authenticator will be set, but not both. + // These will always be set until the performer returns the SupportsAuthenticator capability. + string cluster_username = 3; + string cluster_password = 4; + + optional ClusterConfig cluster_config = 5; + + // Can apply SDK tunables that can't be represented generically in GRPC. + // + // One example is for performance testing, where we may want to experiment with various approaches in say the Java SDK. + // So one run may send "com.couchbase.executorMaxThreadCount"="10", and the next set it to "20". + // + // Interpretation of the tunables, and how to apply them inside the SDK, is completely SDK-dependent. + // + // The intent is that the tunables be associated with the cluster connection, but this may not be possible in all + // implementations - e.g. Java uses global system properties instead. + // + // Generally performers can ignore this field unless they are explicitly planning on using it. + map tunables = 6; + + // Either this will be set or username/password will be set, but not both. + // These will never be set until the performer returns the SupportsAuthenticator capability. + optional Authenticator authenticator = 7; +} + +message ClusterConnectionCreateResponse { + // The number of cluster connections the performer has, after opening this one. + int32 cluster_connection_count = 1; +} + +message ClusterConnectionCloseRequest { + string cluster_connection_id = 1; +} + +message ClusterConnectionCloseResponse { + // The number of cluster connections the performer has, after closing this one. + int32 cluster_connection_count = 2; +} + +message DisconnectConnectionsRequest { +} + +message DisconnectConnectionsResponse { +} + diff --git a/fit-performer/proto/shared.collection.proto b/fit-performer/proto/shared.collection.proto new file mode 100644 index 00000000..d9e9b6f1 --- /dev/null +++ b/fit-performer/proto/shared.collection.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package protocol.shared; +option csharp_namespace = "Couchbase.Grpc.Protocol.Shared"; +option java_package = "com.couchbase.client.protocol.shared"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/shared"; +option ruby_package = "FIT::Protocol::Shared"; +option java_multiple_files = true; + +message Scope { + string bucket_name = 1; + string scope_name = 2; +} + +message Collection { + string bucket_name = 1; + string scope_name = 2; + string collection_name = 3; +} diff --git a/fit-performer/proto/shared.content.proto b/fit-performer/proto/shared.content.proto new file mode 100644 index 00000000..c7e229b2 --- /dev/null +++ b/fit-performer/proto/shared.content.proto @@ -0,0 +1,130 @@ +syntax = "proto3"; + +package protocol.shared; +option csharp_namespace = "Couchbase.Grpc.Protocol.Shared"; +option java_package = "com.couchbase.client.protocol.shared"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/shared"; +option ruby_package = "FIT::Protocol::Shared"; +option java_multiple_files = true; + +import "shared.exceptions.proto"; + +message Content { + oneof content { + // This should be passed directly into the SDK, as string type - no initial conversion into a JsonObject or similar. + // How this is treated by the SDK will depend on the transcoder set (if any), which is sent separately in the options. + // (Usually it will be sent with RawJsonTranscoder). + string passthrough_string = 1; + + // The contents of this are JSON. + // The performer should convert to a platform-dependent representation of JSON, and then pass into the SDK. + // For Java, that would be the SDK's JsonObject. For Node, the result of JSON.parse(). + // Whatever is the most appropriate way to take a JSON blob and send it to the SDK without having to provide a + // transcoder, is the intent here. + bytes convert_to_json = 2; + + // Treat this content as a byte array and pass it directly through the SDK. + bytes byte_array = 3; + + // If this is sent then the performer should pass null/nil/none (depending on language) as the content. + bool null = 4; + } +} + +// The user is doing someResult.contentAs[Something]() - this allows specifying the Something. The protocol generally then +// requires that content to be converted into bytes to be streamed back. +message ContentAs { + oneof as { + // Intended to get the result as a String, then convert it into bytes from UTF-8. Something like this: + // var result = someResult.contentAs[String]() + // var returnOverGRPC = result.getBytes(StandardCharsets.UTF_8) + bool as_string = 1; + + // Intended to get the result as a byte array, then convert it into bytes. Something like this: + // var returnOverGRPC = someResult.contentAs[Array[Byte]]() + // var returnOverGRPC = someResult.contentAsBytes() + bool as_byte_array = 2; + + // JSON handling is very platform-dependent. The intent is to get the result in your platform's most standard/natural + // representation of JSON Objects, and then convert it into bytes, again using the most standard way of doing that. + // E.g. this is intended to represent how most users will use the SDK. + // var result = someResult.contentAs[JsonObject] + // var returnOverGRPC = result.toBytes() + bool as_json_object = 3; + + // JSON handling is very platform-dependent. The intent is to get the result in your platform's most standard/natural + // representation of JSON Arrays, and then convert it into bytes, again using the most standard way of doing that. + // E.g. this is intended to represent how most users will use the SDK. + // var result = someResult.contentAs[JsonArray] + // var returnOverGRPC = result.toBytes() + bool as_json_array = 4; + + // This is a way to represent an SDK deserializing content as a Boolean. + // Results where this is used should include a boolean in the oneof selection for content + bool as_boolean = 5; + + // The driver is requesting someResult.contentAs[Integer]. The performer is free + // to interpret `Integer` in a platform-specific way. Tests will only be + // expecting values well below the limits of a 32 bit signed integer, and the + // performer can safely use the platform-specific version of such a type to hold + // the value: for example, an `int` or `long` are both fine. + bool as_integer = 6; + + // The driver is requesting someResult.contentAs[Float]. The performer is free + // to interpret `Float` in a platform-specific way. Tests will only be + // expecting values well below the limits of a 32 bit signed floating point, and the + // performer can safely use the platform-specific version of such a type to hold + // the value: for example, a `float` or `double` are both fine. + bool as_floating_point = 7; + } +} + +message ContentOrError { + oneof result { + ContentTypes content = 1; + protocol.shared.Exception exception = 2; + } +} + +message ContentTypes { + message NullValue {} + + oneof content { + // Used for content as bytes and JSON + bytes content_as_bytes = 1; + + string content_as_string = 2; + + int64 content_as_int64 = 3; + + double content_as_double = 4; + + bool content_as_bool = 5; + + // content returned by sdk is supposed to be null + NullValue content_as_null = 6; + } +} + +// For subsystems (such as transactions) that rely on performer-side validation. +// Many result types have a `SomeResult.contentAs[SomeType]()` call, and this message controls how the performer +// executes those, and what validation it performs. +message ContentAsPerformerValidation { + // How the performer should perform SomeResult.contentAs[SomeType](). + shared.ContentAs content_as = 1; + + // If true, the contentAs operation is expected to succeed and the performer needs to validate this, raising a + // GRPC FAILED_PRECONDITION error if not. + // If false, the contentAs operation is expected to fail and similarly the performer needs to validate this and + // raise a GRPC INVALID_ARGUMENT error if validation fails. + // Specifically the operation would be expected to fail with an SDK3 DecodingFailureException. However, it is + // known that some SDKs do not follow the model, and so it is not a required part of the validation. The performer + // can choose to optionally validate it is a DecodingFailureException, for an extra level of regression safety. + bool expect_success = 2; + + // If specified, the performer will check the results of contentAs operation against this. + // The results of the contentAs operation should be converted to a byte[] first. + // Iff the expected content does not equal the actual content, the performer should raise a GRPC FAILED_PRECONDITION + // error. + optional bytes expected_content_bytes = 3; +} diff --git a/fit-performer/proto/shared.doc_location.proto b/fit-performer/proto/shared.doc_location.proto new file mode 100644 index 00000000..4ed71e47 --- /dev/null +++ b/fit-performer/proto/shared.doc_location.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +package protocol.shared; +option csharp_namespace = "Couchbase.Grpc.Protocol.Shared"; +option java_package = "com.couchbase.client.protocol.shared"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/shared"; +option ruby_package = "FIT::Protocol::Shared"; +option java_multiple_files = true; + +import "shared.collection.proto"; +import "shared.bounds.proto"; + +// The performer will access the document in this exact location +message DocLocationSpecific { + Collection collection = 1; + string id = 2; +} + +// The performer will generate a random UUID for the document id +message DocLocationUuid { + Collection collection = 1; +} + +enum RandomDistribution { + RANDOM_DISTRIBUTION_UNIFORM = 0; +} + +// Generate a random id in [0-poolSize). +message PoolSelectionStategyRandom { + RandomDistribution distribution = 1; +} + +// Use a counter to select the next document in the pool, starting at 0. +// The performer should increment the counter each time it uses it. +// The performer should modulo `poolSize`. E.g. `realCounterResult = currentCounterValue % poolSize`. +message PoolSelectionStrategyCounter { + Counter counter = 1; +} + +// The performer will pick a document from a pool. +// The performer should generate an integer in [0-poolSize) following selectionStrategy. Then append that to `idPreface` +// e.g. "${idPreface}${selectedInt}" to get the document id. +message DocLocationPool { + Collection collection = 1; + + // All documents in the pool start with this. + string id_preface = 2; + + // How many documents are in the pool. + int64 pool_size = 3; + + oneof poolSelectionStrategy { + PoolSelectionStategyRandom random = 4; + PoolSelectionStrategyCounter counter = 5; + } +} + +message DocLocation { + oneof location { + DocLocationSpecific specific = 1; + DocLocationUuid uuid = 2; + DocLocationPool pool = 3; + } +} diff --git a/fit-performer/proto/shared.echo.proto b/fit-performer/proto/shared.echo.proto new file mode 100644 index 00000000..fa2a6b0e --- /dev/null +++ b/fit-performer/proto/shared.echo.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package protocol.shared; +option csharp_namespace = "Couchbase.Grpc.Protocol.Shared"; +option java_package = "com.couchbase.client.protocol.shared"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/shared"; +option ruby_package = "FIT::Protocol::Shared"; +option java_multiple_files = true; + +// Request that the performer Echo a string to the performer logs. +message EchoRequest { + string message = 1; + string testName = 2; +} + +message EchoResponse { +} diff --git a/fit-performer/proto/shared.exceptions.proto b/fit-performer/proto/shared.exceptions.proto new file mode 100644 index 00000000..d739c1f2 --- /dev/null +++ b/fit-performer/proto/shared.exceptions.proto @@ -0,0 +1,148 @@ +syntax = "proto3"; + +package protocol.shared; +option csharp_namespace = "Couchbase.Grpc.Protocol.Shared"; +option java_package = "com.couchbase.client.protocol.shared"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/shared"; +option ruby_package = "FIT::Protocol::Shared"; +option java_multiple_files = true; + +// All errors derived from CouchbaseException. +// The performer should return the most specific error possible. E.g. not SDK_COUCHBASE_EXCEPTION, unless a raw CouchbaseException +// is what the SDK raised. +// From https://github.com/couchbaselabs/sdk-rfcs/blob/master/rfc/0058-error-handling.md +// +// +// Go specific notes: +// Go deviates from the error handling RFC in a few ways, and gets some special handling in the performer to map it into +// the RFC model so it can work with the FIT GRPC: +// +// 1. It doesn't really have `ErrorContext` in the same way. Instead it has `KeyValueError`, `QueryError` et al., which +// perform a similar purpose to `ErrorContext`. +// 2. It doesn't have a base `CouchbaseException` error. Errors don't derive from `CouchbaseException`, and generic +// errors that would be just a `CouchbaseException` in another SDK, become a `KeyValueError`, `QueryError` etc. +// (Assuming the error did originate from the SDK.) +// +// We accommodate (2)) in the FIT model by having the Go performer check for all specific exceptions first +// with e.g. `errors.Is(err, ErrTimeout)`. If these all fail, it then checks if the error is a `QueryError` etc. with +// `errors.As()`. If one of those succeeds, it's regarded as a generic CouchbaseException. +// If all those checks fail, the error is classified as `ExceptionOther`. +// +// (1) is accommodated by just converted `QueryError` et al. into a serialized `ErrorContext`-esque string, as they +// are serving very similar purposes. +enum CouchbaseExceptionType { + SDK_COUCHBASE_EXCEPTION = 0; + SDK_TIMEOUT_EXCEPTION = 1; + SDK_REQUEST_CANCELLED_EXCEPTION = 2; + SDK_INVALID_ARGUMENT_EXCEPTION = 3; + SDK_SERVICE_NOT_AVAILABLE_EXCEPTION = 4; + SDK_INTERNAL_SERVER_FAILURE_EXCEPTION = 5; + SDK_AUTHENTICATION_FAILURE_EXCEPTION = 6; + SDK_TEMPORARY_FAILURE_EXCEPTION = 7; + SDK_PARSING_FAILURE_EXCEPTION = 8; + SDK_CAS_MISMATCH_EXCEPTION = 9; + SDK_BUCKET_NOT_FOUND_EXCEPTION = 10; + SDK_COLLECTION_NOT_FOUND_EXCEPTION = 11; + SDK_UNSUPPORTED_OPERATION_EXCEPTION = 12; + SDK_AMBIGUOUS_TIMEOUT_EXCEPTION = 13; + SDK_UNAMBIGUOUS_TIMEOUT_EXCEPTION = 14; + SDK_FEATURE_NOT_AVAILABLE_EXCEPTION = 15; + SDK_SCOPE_NOT_FOUND_EXCEPTION = 16; + SDK_INDEX_NOT_FOUND_EXCEPTION = 17; + SDK_INDEX_EXISTS_EXCEPTION = 18; + SDK_ENCODING_FAILURE_EXCEPTION = 19; + SDK_DECODING_FAILURE_EXCEPTION = 20; + SDK_RATE_LIMITED_EXCEPTION = 21; + SDK_QUOTA_LIMITED_EXCEPTION = 22; + + // Key-value + SDK_DOCUMENT_NOT_FOUND_EXCEPTION = 101; + SDK_DOCUMENT_UNRETRIEVABLE_EXCEPTION = 102; + SDK_DOCUMENT_LOCKED_EXCEPTION = 103; + SDK_DOCUMENT_NOT_LOCKED_EXCEPTION = 131; + SDK_VALUE_TOO_LARGE_EXCEPTION = 104; + SDK_DOCUMENT_EXISTS_EXCEPTION = 105; + SDK_DURABILITY_LEVEL_NOT_AVAILABLE_EXCEPTION = 107; + SDK_DURABILITY_IMPOSSIBLE_EXCEPTION = 108; + SDK_DURABILITY_AMBIGUOUS_EXCEPTION = 109; + SDK_DURABLE_WRITE_IN_PROGRESS_EXCEPTION = 110; + SDK_DURABLE_WRITE_RECOMMIT_IN_PROGRESS_EXCEPTION = 111; + SDK_PATH_NOT_FOUND_EXCEPTION = 113; + SDK_PATH_MISMATCH_EXCEPTION = 114; + SDK_PATH_INVALID_EXCEPTION = 115; + SDK_PATH_TOO_BIG_EXCEPTION = 116; + SDK_PATH_TOO_DEEP_EXCEPTION = 117; + SDK_VALUE_TOO_DEEP_EXCEPTION = 118; + SDK_DOCUMENT_TOO_DEEP_EXCEPTION = 125; + SDK_VALUE_INVALID_EXCEPTION = 119; + SDK_DOCUMENT_NOT_JSON_EXCEPTION = 120; + SDK_NUMBER_TOO_BIG_EXCEPTION = 121; + SDK_DELTA_INVALID_EXCEPTION = 122; + SDK_PATH_EXISTS_EXCEPTION = 123; + SDK_XATTR_UNKNOWN_MACRO_EXCEPTION = 124; + SDK_XATTR_INVALID_KEY_COMBO_EXCEPTION = 126; + SDK_XATTR_UNKNOWN_VIRTUAL_ATTRIBUTE_EXCEPTION = 127; + SDK_XATTR_CANNOT_MODIFY_VIRTUAL_ATTRIBUTE_EXCEPTION = 128; + SDK_XATTR_NO_ACCESS_EXCEPTION = 130; + + // Query + SDK_PLANNING_FAILURE_EXCEPTION = 201; + SDK_INDEX_FAILURE_EXCEPTION = 202; + SDK_PREPARED_STATEMENT_FAILURE_EXCEPTION = 203; + SDK_DML_FAILURE_EXCEPTION = 204; + + // Analytics + SDK_COMPILATION_FAILURE_EXCEPTION = 301; + SDK_JOB_QUEUE_FULL_EXCEPTION = 302; + SDK_DATASET_NOT_FOUND_EXCEPTION = 303; + SDK_DATAVERSE_NOT_FOUND_EXCEPTION = 304; + SDK_DATASET_EXISTS_EXCEPTION = 305; + SDK_DATAVERSE_EXISTS_EXCEPTION = 306; + SDK_LINK_NOT_FOUND_EXCEPTION = 307; + + // View + SDK_VIEW_NOT_FOUND_EXCEPTION = 501; + SDK_DESIGN_DOCUMENT_NOT_FOUND_EXCEPTION = 502; + + // Management + SDK_COLLECTION_EXISTS_EXCEPTION = 601; + SDK_SCOPE_EXISTS_EXCEPTION = 602; + SDK_USER_NOT_FOUND_EXCEPTION = 603; + SDK_GROUP_NOT_FOUND_EXCEPTION = 604; + SDK_BUCKET_EXISTS_EXCEPTION = 605; + SDK_USER_EXISTS_EXCEPTION = 606; + SDK_BUCKET_NOT_FLUSHABLE_EXCEPTION = 607; +} + +// An exception derived from CouchbaseException +// "Ex" naming as transactions already has CouchbaseException in an enum. +message CouchbaseExceptionEx { + // The simple name e.g. "DocumentNotFoundException" + string name = 1; + + CouchbaseExceptionType type = 2; + optional Exception cause = 3; + + // Broadly this is the result of outputting the exception. E.g. for Java `ex.toString()`. + // Per the RFC it's expected to contain the serialized error context. + string serialized = 4; +} + + +// If the exception is not represented in SdkException, can return it in raw form here +message ExceptionOther { + // The simple name e.g. "InvalidArgumentException" + string name = 1; + + // Broadly this is the result of outputting the exception. E.g. for Java `ex.toString()`. + string serialized = 2; +} + +message Exception { + oneof exception { + CouchbaseExceptionEx couchbase = 1; + + ExceptionOther other = 2; + } +} + diff --git a/fit-performer/proto/shared.latch.proto b/fit-performer/proto/shared.latch.proto new file mode 100644 index 00000000..1115b67b --- /dev/null +++ b/fit-performer/proto/shared.latch.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package protocol.shared; +option csharp_namespace = "Couchbase.Grpc.Protocol.Shared"; +option java_package = "com.couchbase.client.protocol.shared"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/shared"; +option ruby_package = "FIT::Protocol::Shared"; +option java_multiple_files = true; + +// Note these latches don't work like real CountdownLatches (or equivalent). +// Each concurrent transaction gets its own version of the latch, and they are synchronized by the driver. +// +// Latches are bound to the rpc they are created in. E.g. it should be possible to have two concurrent `transactionStream` +// rpcs going on, each using the same latch name, but referring to two separate underlying CountdownLatches (or equivalents). +message Latch { + int32 initial_count = 1; + string name = 2; +} + diff --git a/fit-performer/proto/shared.transaction_config.proto b/fit-performer/proto/shared.transaction_config.proto new file mode 100644 index 00000000..b8cab7fd --- /dev/null +++ b/fit-performer/proto/shared.transaction_config.proto @@ -0,0 +1,51 @@ +// These are in shared package rather then transactions, to avoid problems with Go import cycles. +syntax = "proto3"; + +package protocol.shared; +option csharp_namespace = "Couchbase.Grpc.Protocol.Shared"; +option java_package = "com.couchbase.client.protocol.shared"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/shared"; +option ruby_package = "FIT::Protocol::Shared"; +option java_multiple_files = true; + +import "shared.basic.proto"; +import "hooks.transactions.proto"; +import "shared.collection.proto"; + +// Options that can be overridden at the individual transaction level. +// Note that pre-ExtSDKIntegration, this struct was empty. Not in the implementation, where PerTransactionConfig existed, +// but here at the FIT level. So, it will only be populated for ExtSDKIntegration performers. +message TransactionsConfig { + // Durability to use for all mutating KV operations, both in cleanup and all transactions initiated. + optional shared.Durability durability = 1; + + // Timeout (nee expirationTime) for all transactions. + optional int64 timeout_millis = 2; + + repeated hooks.transactions.Hook hook = 3; + + optional shared.Collection metadata_collection = 4; + + optional TransactionsCleanupConfig cleanup_config = 5; + + optional TransactionsConfigQuery query_config = 6; +} + +message TransactionsCleanupConfig { + // Whether to run the lost cleanup thread(s). + optional bool cleanup_lost_attempts = 1; + + // Whether to run the cleanup thread cleaning up attempts from this client. + optional bool cleanup_client_attempts = 2; + + // Length of the lost cleanup window, in millis. + optional int64 cleanup_window_millis = 3; + + // Additional collections to add to the cleanup set. + // Can be ignored if not ExtSDKIntegration. + repeated shared.Collection cleanup_collection = 4; +} + +message TransactionsConfigQuery { + optional shared.ScanConsistency scan_consistency = 1; +} diff --git a/fit-performer/proto/shared.transcoders.proto b/fit-performer/proto/shared.transcoders.proto new file mode 100644 index 00000000..73bdf525 --- /dev/null +++ b/fit-performer/proto/shared.transcoders.proto @@ -0,0 +1,98 @@ +syntax = "proto3"; + +package protocol.shared; +option csharp_namespace = "Couchbase.Grpc.Protocol.Shared"; +option java_package = "com.couchbase.client.protocol.shared"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/shared"; +option ruby_package = "FIT::Protocol::Shared"; +option java_multiple_files = true; + +message LegacyTranscoder {} +message JsonTranscoder {} +message RawJsonTranscoder {} +message RawStringTranscoder {} +message RawBinaryTranscoder {} + + +// All the transcoders required by https://github.com/couchbaselabs/sdk-rfcs/blob/master/rfc/0055-serializers-transcoders.md +message Transcoder { + oneof transcoder { + LegacyTranscoder legacy = 1; + JsonTranscoder json = 2; + RawJsonTranscoder raw_json = 3; + RawStringTranscoder raw_string = 4; + RawBinaryTranscoder raw_binary = 5; + } +} + +/** + * CustomSerializer defines different serialization strategies using a `oneof`. + * + * Currently, it includes a boolean option but is designed to be extended with + * other serialization methods in the future. + */ +message JsonSerializer { + oneof serializer { + // Explicitly use the SDK's default JsonSerializer + // (See https://github.com/couchbaselabs/sdk-rfcs/blob/master/rfc/0055-serializers-transcoders.md#default-jsonserializer) + // If the SDK does not have this concept, then it can instead not provide + // any serializer for this operation - which is expected to have the same + // effect. + bool default = 1; + + /** + * **Custom Serializer** + * + * Uses a custom `JsonSerializer` implementation with specific serialization and deserialization logic. + * + * **Serialization Logic:** + * ```java + * public byte[] serialize(Object input) { + * // Convert the input object to a JSON string + * String jsonString = convertObjectToJsonString(input); + * + * // Create a JSON object from the string + * JsonObject jsonObject = parseStringToJsonObject(jsonString); + * + * // Add the "Serialized": true field + * jsonObject.put("Serialized", true); + * + * // Convert the JSON object to a byte array and return + * return convertJsonObjectToByteArray(jsonObject); + * } + * ``` + * + * **Deserialization Logic:** + * ```java + * public T deserialize(Class target, byte[] input) { + * // Parse the byte array into a JSON object + * JsonObject jsonObject = parseByteArrayToJsonObject(input); + * + * // Upsert the "Serialized" field to false, "Serialized" key may or may not present while deserializing. + * jsonObject.put("Serialized", false); + * + * // Convert the JSON object back to the target type and return + * return convertJsonObjectToTargetType(jsonObject, target); + * } + * ``` + * + * **Implementation Details:** + * - **`convertObjectToJsonString(Object input)`**: + * - Converts the input object to its JSON string representation. + * - **`parseStringToJsonObject(String jsonString)`**: + * - Parses the JSON string into a `JsonObject`. + * - **`convertJsonObjectToByteArray(JsonObject jsonObject)`**: + * - Serializes the `JsonObject` into a byte array. + * - **`parseByteArrayToJsonObject(byte[] input)`**: + * - Deserializes the byte array back into a `JsonObject`. + * - **`convertJsonObjectToTargetType(JsonObject jsonObject, Class target)`**: + * - Converts the `JsonObject` to the specified target type `T`. + * + * **Notes for Performers:** + * - Implementations in other languages should follow the same logical steps using equivalent constructs. + * - Ensure that the `"Serialized"` flag is appropriately set during serialization and deserialization to track the state. + * - Proper error handling should be included to manage serialization/deserialization exceptions. + */ + bool custom_serializer = 2; + } +} \ No newline at end of file diff --git a/fit-performer/proto/streams.top_level.proto b/fit-performer/proto/streams.top_level.proto new file mode 100644 index 00000000..a7884134 --- /dev/null +++ b/fit-performer/proto/streams.top_level.proto @@ -0,0 +1,139 @@ +syntax = "proto3"; + +package protocol.streams; +option csharp_namespace = "Couchbase.Grpc.Protocol.Streams"; +option java_package = "com.couchbase.client.protocol.streams"; +option java_multiple_files = true; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/streams"; +option ruby_package = "FIT::Protocol::Streams"; + +import "shared.exceptions.proto"; + +enum Type { + STREAM_KV_RANGE_SCAN = 0; + STREAM_FULL_TEXT_SEARCH = 1; + STREAM_LOOKUP_IN_ALL_REPLICAS = 2; + STREAM_KV_GET_ALL_REPLICAS = 3; +} + +// Streams. +// Some operations (e.g. KV range scans, SQL++ queries) can stream back results. The performer needs to track these +// streams to allow testing their cancellation, backpressure, etc. This GRPC attempts to abstract over all forms of +// language-specific streams. +// +// Stream lifetimes: +// The performer must wait for all streams to complete or error, before completing or erroring the grpc that spawned those streams (as +// otherwise there is nothing to stream back on). +// There are top-level RPCs that allow cancelling and requesting more items on a stream. +// The corollary of these two is that the performer needs to track all streams globally, and associate each with a +// given `run` RPC - for example, by giving each run a unique runId. +// +// Stream lifecycles: +// In general, a golden path stream will return one streams.Created, zero or more run.Results corresponding to the actual +// rows/items on the stream, and then one streams.Complete. +// +// Cancellation: If the driver requests a stream be cancelled, it will raise a streams.Cancelled instead of the streams.Complete, +// and potentially instead of some of the run.Results. The driver will only cancel a stream that has returned streams.Created. + +// Errors: If the performer hits any error while processing the stream, it should raise a streams.Error instead of the streams.Complete, +// and potentially instead of some of the run.Results. It must have sent a streams.Created first. +// +// Safe assumptions: To simplify the performer logic, the performer can assume it's a bug if the driver requests anything on a stream that has +// sent back a Complete, Error or Cancelled signal. + +// A request for the performer to cancel a stream. Exactly what this entails is context and SDK dependent. If the +// SDK's stream has the concept of user-cancellation, this should be invoked. At minimum, the performer should stop +// streaming responses from that stream to the driver. +// The performer can assume that if the driver tries to cancel a stream that doesn't exist (e.g. has stopped streaming), +// this is a driver bug. +// As a corollary, the driver will only cancel `stream_when.OnDemand` streams. Otherwise it would be a race as to whether the stream +// completes or is cancelled first. +// On cancellation the performer should not proceed to the next stage of streaming. E.g. it should not return a +// streams.Complete. But instead return a streams.Cancelled, after it has finished cancelling. +message CancelRequest { + string stream_id = 1; +} + +// The performer has successfully cancelled the stream. Due to the asynchronous nature of streaming and cancelling, a +// few items may still be streamed back to the driver after this is sent. +message CancelResponse { + // Currently empty, but included per GRPC best practices. +} + +// The driver (pretending to be the user) is requesting more items. For each item requested, the performer should fetch +// one from the stream, and stream back an appropriate response. +// +// Once all items in the stream have been streamed, the performer should automatically proceed to the next stage of +// streaming (usually to return a streams.Complete). +// +// FIT is aware that some implementations (such as Go), don't have any lookahead, and that if the user/driver requests +// N items on a N item stream, the implementation does not know the stream is complete yet - the user/driver needs +// to request item N+1 first. +// +// To simplify the performer logic, the performer can assume that if the driver does any of these, it is a driver bug. +// It should raise a standard GRPC UNKNOWN error code: +// 1. If the driver tries to do this with a stream that doesn't exist (e.g. has stopped streaming), +// 2. If the driver requests more items before the performer has finished streaming back the previously requested items. +// +// If the driver requests more items than are currently in the stream, this is not an error, the performer should +// stream back as many as it can, and then wait for more items so it can satisfy the request. +// If the performer knows the stream has finished and the driver has still requested too many items, this is also not +// a bug. The performer should continue to the next stage of streaming (usually to return a streams.Complete). +message RequestItemsRequest { + string stream_id = 1; + int32 num_items = 2; +} + +message RequestItemsResponse { + // Currently empty, but included per GRPC best practices. +} + +// The performer has created a stream and is notifying the driver of it. +message Created { + string stream_id = 1; + Type type = 2; +} + +// A stream has finished sending back all results. +message Complete { + string stream_id = 1; +} + +// A stream raised an error. +message Error { + string stream_id = 1; + shared.Exception exception = 2; +} + +// A stream has finished being cancelled. +message Cancelled { + string stream_id = 1; +} + +message Signal { + oneof signal { + Created created = 1; + Complete complete = 2; + Error error = 3; + Cancelled cancelled = 4; + } +} + +// The performer should stream back responses as fast as it can. +message Automatically {} + +// The performer must wait for the driver to request rows before streaming them. +message OnDemand {} + +message Config { + // The performer must use this as the subsequent stream id. + // (Having stream ids generated driver-side will make it much easier for the driver to handle concurrent tests in future.) + string stream_id = 1; + + // When the performer should stream back results. + oneof stream_when { + Automatically automatically = 2; + OnDemand on_demand = 3; + } +} + diff --git a/fit-performer/proto/transactions.basic.proto b/fit-performer/proto/transactions.basic.proto new file mode 100644 index 00000000..6b602263 --- /dev/null +++ b/fit-performer/proto/transactions.basic.proto @@ -0,0 +1,28 @@ +// Contains basic low-level primitives that have no dependencies. +syntax = "proto3"; + +package protocol.transactions; +option csharp_namespace = "Couchbase.Grpc.Protocol.Transactions"; +option java_package = "com.couchbase.client.protocol.transactions"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/transactions"; +option ruby_package = "FIT::Protocol::Transactions"; +option java_multiple_files = true; + +enum AttemptStates { + NOTHING_WRITTEN=0; + PENDING=1; + ABORTED=2; + COMMITTED=3; + COMPLETED=4; + ROLLED_BACK=5; + UNKNOWN=6; +} + +// Exists for legacy reasons. Newer messages should use DocLocation. +message DocId { + string bucket_name = 1; + string scope_name = 2; + string collection_name = 3; + string doc_id = 4; +} + diff --git a/fit-performer/proto/transactions.cleanup.proto b/fit-performer/proto/transactions.cleanup.proto new file mode 100644 index 00000000..db61d0be --- /dev/null +++ b/fit-performer/proto/transactions.cleanup.proto @@ -0,0 +1,85 @@ +// Everything to do with transactions cleanup testing +syntax = "proto3"; + +package protocol.transactions; +option csharp_namespace = "Couchbase.Grpc.Protocol.Transactions"; +option java_package = "com.couchbase.client.protocol.transactions"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/transactions"; +option ruby_package = "FIT::Protocol::Transactions"; +option java_multiple_files = true; + +import "transactions.basic.proto"; +import "hooks.transactions.proto"; +import "shared.collection.proto"; + +message TransactionCleanupRequest { + DocId atr = 5; + // For backwards-compatibility with performers that don't support `atr` yet - will be removed eventually + string atr_id = 1 [ deprecated = true ]; // Field `atr` should be used instead. + string attempt_id = 2; + string transaction_id = 3; + // Any hooks can only use HookPoints in the CLEANUP_* range + repeated hooks.transactions.Hook hook = 4; + + // This is only needed for KV operations, so the driver will usually send the default cluster connection. + string cluster_connection_id = 6; +} + +message TransactionCleanupATRRequest { + // For backwards-compatibility with performers that don't support `atr` yet - will be removed eventually + string atr_id = 1 [ deprecated = true ]; // Field `atr` should be used instead. + DocId atr = 3; + // Any hooks can only use HookPoints in the CLEANUP_* range + repeated hooks.transactions.Hook hook = 2; + + string cluster_connection_id = 4; +} + +// The result of attempting to cleanup a single attempt entry in an ATR +message TransactionCleanupAttempt { + // Whether the attempt succeeded + bool success = 1; + + // All logs related to the attempt + repeated string logs = 2; + + // The state of the ATR entry, before cleanup. + // This field is optional. It will allow additional verification in some tests if it is provided. + AttemptStates state = 3; + + // The ATR's location + DocId atr = 7; + + // The key of the ATR document + // For backwards-compatibility with performers that don't support `atr` yet - aiming to remove eventually. + string atr_id = 4 [ deprecated = true ]; // Field `atr` should be used instead. + + // The name of the bucket the ATR is on + string atr_bucket_name = 5 [ deprecated = true ]; // Field `atr` should be used instead. + + // The ID of the transaction attempt that was being cleaned up + string attempt_id = 6; +} + +message TransactionCleanupATRResult { + repeated TransactionCleanupAttempt result = 1; + + // Total number of entries found in the ATR + int32 num_entries = 2; + + // Total number of expired entries found in the ATR + int32 num_expired_entries = 3; +} + +// The set of collections currently being cleaned up. +message CleanupSet { + repeated shared.Collection cleanup_set = 1; +} + +message CleanupSetFetchRequest { + string cluster_connection_id = 1; +} + +message CleanupSetFetchResponse { + CleanupSet cleanup_set = 1; +} \ No newline at end of file diff --git a/fit-performer/proto/transactions.client_record.proto b/fit-performer/proto/transactions.client_record.proto new file mode 100644 index 00000000..3bf5cdee --- /dev/null +++ b/fit-performer/proto/transactions.client_record.proto @@ -0,0 +1,40 @@ +// Everything to do with testing of _txn:client-record +syntax = "proto3"; + +package protocol.transactions; +option csharp_namespace = "Couchbase.Grpc.Protocol.Transactions"; +option java_package = "com.couchbase.client.protocol.transactions"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/transactions"; +option ruby_package = "FIT::Protocol::Transactions"; +option java_multiple_files = true; + +import "hooks.transactions.proto"; + +message ClientRecordProcessRequest { + string client_uuid = 1; + + // The collection where the client record should exist (this message was created before Collection existed) + string bucket_name = 2; + string scope_name = 3; + string collection_name = 4; + + // Any hooks can only use HookPoints in the CLIENT_RECORD_* range + repeated hooks.transactions.Hook hook = 5; + + // This is only needed for KV operations, so the driver will usually send the default cluster connection. + string cluster_connection_id = 6; +} + +message ClientRecordProcessResponse { + int32 num_active_clients = 1; + int32 index_of_this_client = 2; + repeated string expired_client_ids = 4; + int32 num_existing_clients = 5; + int32 num_expired_clients = 6; + bool override_enabled = 7; + bool override_active = 8; + int64 override_expires = 9; + int64 cas_now_nanos = 10; + string client_uuid = 11; + bool success = 12; +} diff --git a/fit-performer/proto/transactions.commands.proto b/fit-performer/proto/transactions.commands.proto new file mode 100644 index 00000000..f238d880 --- /dev/null +++ b/fit-performer/proto/transactions.commands.proto @@ -0,0 +1,526 @@ +syntax = "proto3"; + +package protocol.transactions; +option csharp_namespace = "Couchbase.Grpc.Protocol.Transactions"; +option java_package = "com.couchbase.client.protocol.transactions"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/transactions"; +option ruby_package = "FIT::Protocol::Transactions"; +option java_multiple_files = true; + +import "transactions.basic.proto"; +import "transactions.options.proto"; +import "transactions.legacy.proto"; +import "shared.collection.proto"; +import "shared.doc_location.proto"; +import "shared.content.proto"; +import "shared.transcoders.proto"; + +enum EventType { + EventCleanupFailedEvent = 0; + EventIllegalDocumentState = 1; + EventLostCleanupThreadEndedPrematurely = 2; + EventTransactionCleanupAttempt = 3; + EventTransactionCleanupEndRunEvent = 4; + EventTransactionCleanupStartRunEvent = 5; + EventTransactionsStarted = 6; + EventUnknown = 7; +} + +message Event { + EventType type=1; +} + +// The exception received by the application, if any +enum TransactionException { + NO_EXCEPTION_THROWN = 0; + EXCEPTION_FAILED = 1; + EXCEPTION_EXPIRED = 2; + EXCEPTION_UNKNOWN = 3; + EXCEPTION_COMMIT_AMBIGUOUS = 4; + EXCEPTION_FAILED_POST_COMMIT = 5; +} + +message ExpectedCause { + oneof cause { + // Tests should specify the expected cause wherever possible + bool do_not_check = 1; + ExternalException exception = 2; + } +} + +// This maps to the TransactionOperationFailed defined in the design doc +message ErrorWrapper { + // This is now deprecated and the performer can feel free to disable validation of this. Once all performers + // have removed validation, it will be removed. + ErrorClass error_class = 1 [ deprecated = true ]; + bool auto_rollback_attempt = 2; + bool retry_transaction = 3; + TransactionException to_raise = 4; + ExpectedCause cause = 5; +} + +// The root cause of the TransactionException. Will only be checked if TransactionException != NO_EXCEPTION_THROWN +enum ExternalException { + Unknown = 0; + ActiveTransactionRecordEntryNotFound = 1; + ActiveTransactionRecordFull = 2; + ActiveTransactionRecordNotFound = 3; + DocumentAlreadyInTransaction = 4; + DocumentExistsException = 5; + DocumentNotFoundException = 6; + NotSet = 7; + FeatureNotAvailableException = 8; + TransactionAbortedExternally = 9; + PreviousOperationFailed = 10; + ForwardCompatibilityFailure = 11; + ParsingFailure = 12; + IllegalStateException = 13; + CouchbaseException = 14; + ServiceNotAvailableException = 15; + RequestCanceledException = 16; + ConcurrentOperationsDetectedOnSameDocument = 17; + CommitNotPermitted = 18; + RollbackNotPermitted = 19; + TransactionAlreadyAborted = 20; + TransactionAlreadyCommitted = 21; + UnambiguousTimeoutException = 22; + AmbiguousTimeoutException = 23; + AuthenticationFailureException = 24; + // Added with EXT_REPLICA_FROM_PREFERRED_GROUP + DocumentUnretrievableException = 25; +} + +enum ErrorClass { + EC_FAIL_HARD = 0; + EC_FAIL_OTHER = 1; + EC_FAIL_TRANSIENT = 2; + EC_FAIL_AMBIGUOUS = 3; + EC_FAIL_DOC_ALREADY_EXISTS = 4; + EC_FAIL_DOC_NOT_FOUND = 5; + EC_FAIL_PATH_ALREADY_EXISTS = 10; + EC_FAIL_PATH_NOT_FOUND = 6; + EC_FAIL_CAS_MISMATCH = 7; + EC_FAIL_WRITE_WRITE_CONFLICT = 8; + EC_FAIL_ATR_FULL = 9; + EC_FAIL_EXPIRY = 11; + // Any ErrorClass is permitted - including the TransactionOperationFailed not having one + // Note that ErrorClass validation is now deprecated and performers should remove their + // validation logic. + EC_ANYTHING = 12; +} + +message ExpectedResult { + oneof result { + // If the command should succeed + bool success = 1; + + // If the command is permitted to do anything, e.g. succeed or throw errors. Required for some complex tests + // where it's tricky to calculate what a given op will do. + bool anything_allowed = 2; + + // The operation throws an ErrorWrapper/TransactionOperationFailed, the only exception an operation may throw. + // Update: as of EXT_QUERY, EXT_SDK_INTEGRATION and EXT_INSERTS_EXISTING, that is no longer the case, some operations can throw some other exceptions. + ErrorWrapper error = 3; + + // Require a non-TransactionOperationFailed exception. + ExternalException exception = 4; + } +} + +// This was only exposed in the Java API and is deprecated as of 1.1.3 +// Other implementations are not required to check this. +enum DocStatus { + NORMAL = 0; + IN_TXN_COMMITTED = 1; + IN_TXN_OTHER = 2; + OWN_WRITE = 3; + AMBIGUOUS = 4; + DO_NOT_CHECK = 5; + + option deprecated = true; +} + +message CommandGet { + DocId doc_id = 1; + + // What result(s) the command is allowed to return. Anything outside this will fail the test. + // Tests can generally just set one expected result. Multiple results are allowed to support tricky situations, + // such as an op generally failing with FAIL_TRANSIENT until it fails with FAIL_EXPIRY + repeated ExpectedResult expected_result = 2; + + // If not-null, the performer will check the fetched data equals this. Useful for read-your-own-write tests. + // This is superceded by content_as_validation, but that is only available with CONTENT_AS_PERFORMER_VALIDATION, so tests will + // continue to use this until that extension is fully rolled out. + string expected_content_json = 3; + + // Added with cap CONTENT_AS_PERFORMER_VALIDATION and will only be sent if performer declares support for that. + optional shared.ContentAsPerformerValidation content_as_validation = 6; + + // If not DO_NOT_CHECK, the performer will check the fetched document status equals this. + // Fully deprecated as of ExtSDKIntegration. The driver will only specify DO_NOT_CHECK now. + DocStatus expected_status = 4 [ deprecated = true ]; + + // Since day one the performer must have supported stashing a single document. Whenever a CommandGet or + // CommandGetOptional is executed (but not a transactions.Get), the result is automatically stashed. + // ExtMobileInterop adds the concept of stashing multiple documents. This allows some more sophisticated tests + // particularly around CAS mismatches. + // If this `stashInSlot` is set, the performer + // needs to update or create a map of StashSlot (Int) -> TransactionGetResult. This is stored per-transaction and + // should be deleted at the end of the transaction. Subsequent CommandReplace and CommandReplace operations may + // may reference to one of these stashed results. + optional int32 stash_in_slot = 5; + + // Added with EXT_BINARY_SUPPORT and will only be sent if performer declares support for that. + optional TransactionGetOptions options = 7; +} + +// There are two ways the implementation can implement getOptional: +// 1. An explicit ctx.getOptional() API. +// 2. ctx.get() throwing a catchable DocumentNotFoundException. +// Either is allowed. +message CommandGetOptional { + CommandGet get = 1; + + // Use this instead of passing EXPECT_FAIL_DOC_NOT_FOUND or EXPECT_FAIL_DOC_ALREADY_EXISTS + bool expect_doc_present = 2; + + // Added with cap CONTENT_AS_PERFORMER_VALIDATION and will only be sent if performer declares support for that. + optional shared.ContentAsPerformerValidation content_as_validation = 6; + + // Added with EXT_BINARY_SUPPORT and will only be sent if performer declares support for that. + optional TransactionGetOptions options = 4; +} + +message CommandGetReplicaFromPreferredServerGroup { + DocId doc_id = 1; + + // What result(s) the command is allowed to return. Anything outside this will fail the test. + repeated ExpectedResult expected_result = 2; + + // Added with cap CONTENT_AS_PERFORMER_VALIDATION and will only be sent if performer declares support for that. + optional shared.ContentAsPerformerValidation content_as_validation = 6; + + // Refer to `stashInSlot` in CommandGet for details. + // `useStashedResult` and `useStashedSlot` will not be sent together. + optional int32 stash_in_slot = 5; + + // Added with EXT_BINARY_SUPPORT and will only be sent if performer declares support for that. + optional TransactionGetOptions options = 7; +} + +// The SDK needs to implement TransactionGetMultiReplicasFromPreferredServerGroupOptions and TransactionGetMultiReplicaFromPreferredServerGroupSpec. +// But to save testing effort, and because the APIs are currently identical except for naming, FIT will use the +// same CommandGetMulti for both. If they diverge in future we will copy the messages then. +message CommandGetMulti { + message TransactionGetMultiSpec { + shared.DocLocation location = 1; + optional shared.Transcoder transcoder = 2; + + // The performer always needs to check if the doc is returned as Optional.None or present, against this. + bool expect_present = 3; + + // Determines if and how the performer should execute transactionGetResult.contentAs() on this individual spec result. + optional shared.ContentAsPerformerValidation content_as_validation = 6; + + // Each spec will result in a result in the output array. This contains the required position of the result + // in that array, which the SDK must validate. + int32 expected_result_position = 5; + } + + repeated TransactionGetMultiSpec specs = 1; + + optional TransactionGetMultiOptions options = 2; + + // What result(s) this individual spec is allowed to return. Anything outside this will fail the test. + // The performer must perform this validation. If this is empty, the performer skips validation. + repeated ExpectedResult expected_result = 3; + + // Iff true, performer should use ctx.getMultiReplicasFromPreferredServerGroup() instead of ctx.getMulti() + bool get_multi_replicas_from_preferred_server_group = 4; +} + +message CommandInsert { + DocId doc_id = 1; + + oneof content_option { + // This is superceded by `content`, but that is only available with EXT_BINARY_SUPPORT, so tests will + // continue to use this until that extension is fully rolled out. + string content_json = 2; + + // Added with EXT_BINARY_SUPPORT and will only be sent if performer declares support for that. + shared.Content content = 5; + } + + // What result(s) the command is allowed to return. Anything outside this will fail the test. + repeated ExpectedResult expected_result = 3; + + // Added with EXT_BINARY_SUPPORT and will only be sent if performer declares support for that. + optional TransactionInsertOptions options = 4; +} + +message CommandReplace { + DocId doc_id = 1; + + oneof content_option { + string content_json = 2; + + // Added with EXT_BINARY_SUPPORT and will only be sent if performer declares support for that. + shared.Content content = 7; + } + + // What result(s) the command is allowed to return. Anything outside this will fail the test. + repeated ExpectedResult expected_result = 3; + + // Whether to use the result of the last get operation. + // This will not be set inside a CommandBatch, due to the 'last' operation not being + // a valid concept during parallel ops. + bool use_stashed_result = 4; + + // Refer to `stashInSlot` in CommandGet for details. + // `useStashedResult` and `useStashedSlot` will not be sent together. + optional int32 use_stashed_slot = 5; + + // Added with EXT_BINARY_SUPPORT and will only be sent if performer declares support for that. + optional TransactionReplaceOptions options = 6; +} + +message CommandRemove { + DocId doc_id = 1; + + // What result(s) the command is allowed to return. Anything outside this will fail the test. + repeated ExpectedResult expected_result = 2; + + // Whether to use the result of the last get operation + // This will not be set inside a CommandBatch, due to the 'last' operation not being + // a valid concept during parallel ops. + bool use_stashed_result = 3; + + // Refer to `stashInSlot` in CommandGet for details. + // `useStashedResult` and `useStashedSlot` will not be sent together. + optional int32 use_stashed_slot = 4; +} + +// ExtSDKIntegration performers should ignore. +// Otherwise, perform an explicit ctx.commit() +message CommandCommit { + // What result(s) the command is allowed to return. Anything outside this will fail the test. + repeated ExpectedResult expected_result = 1; +} + +// ExtSDKIntegration performers: throw an exception (or raise an error) to cause an auto-rollback. +// Otherwise, perform an explicit ctx.rollback() +message CommandRollback { + // What result(s) the command is allowed to return. Anything outside this will fail the test. + repeated ExpectedResult expected_result = 1; +} + +message CommandWaitOnLatch { + string latch_name = 1; +} + +message CommandSetLatch { + string latch_name = 1; +} + +// A batch of commands, often to be performed in parallel. +// But not always, sometimes it's used so multiple threads can each run a series of commands in serial. +// (E.g. nested CommandBatches). So it's crucial that the performer respect the `parallelism` setting. +// Created long before HorizontalScaling. +// +// Note the semantics of this command aren't as defined as they could be - see some discussion on +// https://couchbase.slack.com/archives/C014WB8U2MQ/p1665062945395139 +// +// The requirements are: +// 1. The operations should be started in parallel (assuming `parallelism` > 1 of course) +// 2. The ~first error should be propagated out of the lambda. ("~first" as it's fine for this to be best-effort, e.g. +// it doesn't have to require atomic variables or locking. Because the first concurrent failure is anyway non-deterministic). +// +// What is not defined is what happens to the other operations if one of them fails. Are they left to run to completion, +// are they cancelled, is that cancellation waited on? These semantics probably need to be selectable. Currently it +// is acceptable to do whatever is easiest - e.g. Java reactor will automatically cancel the other operations, while in Go +// it's easiest to wait for them all to complete. +message CommandBatch { + // A limited subset of commands are supported here: + // CommandInsert, CommandReplace, CommandRemove and CommandGet. + // And replace/remove cannot use useStashedResult, as that is not a well-defined + // concept in parallel operations. + repeated TransactionCommand commands = 1; + + // If the commands are being performed in parallel, how many to do concurrently. + // The performer is required to have this many operations in flight at once. E.g. say this value + // is 3, and 10 operations are specified in `commands`. The performer should start the first 3 + // operations in `commands` in parallel. Once an operation finishes successfully the performer should + // immediately start the next operation in `commands`, in parallel with the still-running other 2. + // It should not 'batch' the operations and wait for them all to succeed before progressing with + // the next batch. + int32 parallelism = 2; +} + +// Replace a document using a regular KV operation. The command is expected to succeed. +// Created long before SdkCommandReplace. +message CommandReplaceRegularKV { + DocId doc_id = 1; + string content_json = 2; +} + +// Remove a document using a regular KV operation. The command is expected to succeed. +// Created long before SdkCommandRemove. +message CommandRemoveRegularKV { + DocId doc_id = 1; +} + +// Insert a document using a regular KV operation. The command is expected to succeed. +// Created long before SdkCommandInsert. +message CommandInsertRegularKV { + DocId doc_id = 1; + string content_json = 2; +} + +// Run a query command. +message CommandQuery { + string statement = 1; + + // What result(s) the command is allowed to return. Anything outside this will fail the test. + repeated ExpectedResult expected_result = 2; + + // Whether to check the returned rows against expectedRows + bool check_row_content = 3; + + // What rows the query should return, as JSON + repeated string expected_rows = 4; + + // Whether to check the number of returned rows against expectedRowCount + bool check_row_count = 6; + + // How many rows the query should return + int32 expected_row_count = 7; + + // Whether to check the returned mutations + bool check_mutations = 8; + + // How many mutations the query should create + int32 expected_mutations = 9; + + // Deprecated as of ExtSDKIntegration, replaced by queryOptions, which allows optional. + // FIT will send both for now, performers should remove support for queryParameters. + CommandQueryParameters query_parameters = 10 [ deprecated = true ]; + + optional TransactionQueryOptions query_options = 12; + + shared.Scope scope = 11; +} + +// If the performer executes this command, it should report a failure. This is usually used to check an earlier +// operation definitely failed the transaction. +message CommandTestFail { +} + +// Simulates the application throwing an exception inside the lambda. Could be used to simulate an application bug, +// or the application choosing to rollback this way (as opposed to ctx.rollback(), e.g. app-rollback). +// Application exceptions will be classified as FAIL_OTHER. +message CommandThrowException { +} + +message Insert { + shared.DocLocation location = 1; + shared.Content content = 2; + + // What result(s) the command is allowed to return. Anything outside this will fail the test. + // The performer must perform this validation. If this is empty, the performer skips validation. + repeated ExpectedResult expected_result = 3; + + // Added with EXT_BINARY_SUPPORT and will only be sent if performer declares support for that. + optional TransactionInsertOptions options = 4; +} + +message Get { + shared.DocLocation location = 1; + + // What result(s) the command is allowed to return. Anything outside this will fail the test. + // The performer must perform this validation. If this is empty, the performer skips validation. + repeated ExpectedResult expected_result = 2; + + // If this `stashInSlot` is set, the performer + // needs to update or create a map of StashSlot (Int) -> TransactionGetResult. This is stored per-transaction and + // should be deleted at the end of the transaction. Subsequent operations may + // may reference to one of these stashed results. + // transactions.CommandGet and transactions.Get share the same map. + optional int32 stash_in_slot = 3; + + // Determines if and how the performer should execute transactionGetResult.contentAs(). + // Added with CONTENT_AS_PERFORMER_VALIDATION and will only be sent if performer declares support for that. + optional shared.ContentAs content_as = 4; + + // Added with EXT_BINARY_SUPPORT and will only be sent if performer declares support for that. + optional TransactionGetOptions options = 5; +} + +message Remove { + shared.DocLocation location = 1; + + // What result(s) the command is allowed to return. Anything outside this will fail the test. + // The performer must perform this validation. If this is empty, the performer skips validation. + repeated ExpectedResult expected_result = 2; + + // Refer to `stashInSlot` in CommandGet for details. + optional int32 use_stashed_slot = 3; +} + +message Replace { + shared.DocLocation location = 1; + shared.Content content = 2; + + // What result(s) the command is allowed to return. Anything outside this will fail the test. + // The performer must perform this validation. If this is empty, the performer skips validation. + repeated ExpectedResult expected_result = 3; + + // Refer to `stashInSlot` in CommandGet for details. + optional int32 use_stashed_slot = 4; + + // Added with EXT_BINARY_SUPPORT and will only be sent if performer declares support for that. + optional TransactionReplaceOptions options = 5; +} + +message TransactionCommand { + oneof command { + CommandInsert insert = 1; + CommandReplace replace = 2; + CommandRemove remove = 3; + CommandCommit commit = 4; + CommandRollback rollback = 5; + CommandGet get = 6; + CommandGetOptional get_optional = 7; + CommandWaitOnLatch wait_on_latch = 8; + CommandSetLatch set_latch = 9; + + // All operations in the batch will be performed in parallel + CommandBatch parallelize = 10; + + CommandReplaceRegularKV replace_regular_kv = 11; + CommandRemoveRegularKV remove_regular_kv = 12; + CommandInsertRegularKV insert_regular_kv = 13; + + CommandThrowException throw_exception = 14; + + CommandQuery query = 15; + CommandTestFail test_fail = 16; + + // Why do we have transactions.CommandInsert and transactions.Insert? The former stems from the original days of FIT + // that just did integration testing of transactions. The latter is for modern FIT and supports the abstractions we + // need now, including for performance testing. + // These will only be sent if the performer declares support for TRANSACTIONS_WORKLOAD_1. + Insert insert_v2 = 17; + Get get_v2 = 18; + Remove remove_v2 = 19; + Replace replace_v2 = 20; + CommandGetMulti get_multi = 22; + + CommandGetReplicaFromPreferredServerGroup get_from_preferred_server_group = 21; + } + + // Added for TXNJ-249: make the application (erroneously) catch and not propagate the error. + bool do_not_propagate_error = 100; + + // If given, the performer will pause (blocking) for `wait_msecs` before executing the TransactionCommand + int32 wait_msecs = 101; +} diff --git a/fit-performer/proto/transactions.extensions.proto b/fit-performer/proto/transactions.extensions.proto new file mode 100644 index 00000000..c1f13611 --- /dev/null +++ b/fit-performer/proto/transactions.extensions.proto @@ -0,0 +1,52 @@ +syntax = "proto3"; + +package protocol.transactions; +option csharp_namespace = "Couchbase.Grpc.Protocol.Transactions"; +option java_package = "com.couchbase.client.protocol.transactions"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/transactions"; +option ruby_package = "FIT::Protocol::Transactions"; +option java_multiple_files = true; + +// Defined https://hackmd.io/foGjnSSIQmqfks2lXwNp8w#Protocol-Extensions +enum Caps { + // This is mandatory if PROTOCOL_VERSION_2_0 or higher is sent. + EXT_TRANSACTION_ID = 0; + + EXT_DEFERRED_COMMIT = 1; + + // At least one of EXT_MEMORY_OPT_UNSTAGING and EXT_TIME_OPT_UNSTAGING must be supported + EXT_MEMORY_OPT_UNSTAGING = 2; + EXT_TIME_OPT_UNSTAGING = 3; + + EXT_CUSTOM_METADATA_COLLECTION = 4; + EXT_BINARY_METADATA = 5; + EXT_QUERY = 6; + EXT_STORE_DURABILITY = 7; + BF_CBD_3787 = 8; + BF_CBD_3794 = 9; + BF_CBD_3705 = 10; + BF_CBD_3838 = 11; + EXT_REMOVE_COMPLETED = 12; + EXT_ALL_KV_COMBINATIONS = 13; + EXT_UNKNOWN_ATR_STATES = 14; + BF_CBD_3791 = 15; + EXT_SINGLE_QUERY = 16; + EXT_REPLACE_BODY_WITH_XATTR = 17; + EXT_THREAD_SAFE = 18; + EXT_SERIALIZATION = 19; + EXT_SDK_INTEGRATION = 20; + EXT_MOBILE_INTEROP = 21; + EXT_INSERT_EXISTING = 22; + EXT_OBSERVABILITY = 23; + EXT_QUERY_CONTEXT = 24; + EXT_PARALLEL_UNSTAGING = 25; + + // Support for Caps.CONTENT_AS_PERFORMER_VALIDATION is a pre-requisite for all following extensions. + EXT_BINARY_SUPPORT = 26; + EXT_REPLICA_FROM_PREFERRED_GROUP = 27; + EXT_GET_MULTI = 28; + EXT_REPLICA_FROM_PREFERRED_GROUP_PATCH1 = 29; + EXT_TTL = 30; + + // Remember to add a test to ForwardCompatibilityTest for each extension +} diff --git a/fit-performer/proto/transactions.legacy.proto b/fit-performer/proto/transactions.legacy.proto new file mode 100644 index 00000000..67c700c2 --- /dev/null +++ b/fit-performer/proto/transactions.legacy.proto @@ -0,0 +1,132 @@ +// Contains legacy or deprecated messages that we want to eventually remove. See performer.proto for the reasoning. +syntax = "proto3"; + +package protocol.transactions; +option csharp_namespace = "Couchbase.Grpc.Protocol.Transactions"; +option java_package = "com.couchbase.client.protocol.transactions"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/transactions"; +option ruby_package = "FIT::Protocol::Transactions"; +option java_multiple_files = true; + +import "hooks.transactions.proto"; +import "transactions.extensions.proto"; +import "transactions.basic.proto"; +import "transactions.cleanup.proto"; +import "shared.basic.proto"; + + + +message CreateTransactionFactoryResponse { + string cluster_connection_id = 1; +} + +message CloseTransactionsRequest { +} + +message CloseTransactionsResponse { +} + +message TransactionsFactoryCreateResponse { + string transactions_factory_ref = 1; +} + +message TransactionAttempt { + AttemptStates state = 1; + string attempt_id = 2; + DocId atr = 3; // Can be empty iff state=NOTHING_WRITTEN +} + +// Requests the performer creates a connection to a Couchbase cluster, and return performer caps. +// In the transactions package as it was part of the original transactions-only implementation, and has been replaced now. +message CreateConnectionRequest { + string cluster_hostname = 1; + string cluster_username = 2; + string cluster_password = 3; + + // As of ExtSDKIntegration this field will no longer be sent and should be ignored. It is unclear why it was ever + // required. + string bucket_name = 4 [ deprecated = true ]; + + // As of ExtSDKIntegration this field will no longer be sent and should be ignored. The driver will now always + // explicitly send a clusterConnectionId. + bool use_as_default_connection = 5 [ deprecated = true ]; + + // The performer should use a custom JsonSerializer when creating the Cluster. + // This custom JsonSerializer is required to have specific serialization and deserialization logic, which the driver + // will validate. See the Java performer for implementation details. + bool use_custom_serializer = 6; + + // Used for TLS enabled clusters such as Capella clusters + bool use_tls = 7; + optional string cert_path = 8; + + option deprecated = true; +} + +// Returns the capabilities of the performer (or more specifically, the transactions-implementation-under-test by that +// performer), along with a reference to the created cluster connection. +// As of ExtSDKIntegration, this is now legacy. Performers should implement getPerformerCaps and createClusterConnection instead. This +message CreateConnectionResponse { + repeated Caps performer_caps = 1; + + // Human-readable string identifying the performer. For example, "java". + string performer_user_agent = 2; + + // Identifies the version of the library under test. For example, "1.1.2". + string performer_library_version = 4; + + // Defined https://hackmd.io/foGjnSSIQmqfks2lXwNp8w#Protocol-Versions + // Must be "1.0", "2.0" or "2.1" any other values will cause the driver to abort. + string protocol_version = 3; + + // The APIs this implementation supports. This is primarily used to run tests on multiple APIs, so if an + // implementation only supports one, it should just return one - see comments for `API` for details. If the + // performer does not return anything, it will be assumed to support just API.DEFAULT. + repeated shared.API supported_apis = 6; + + string cluster_connection_id = 5; + + option deprecated = true; +} + + +message TransactionProcessCleanupQueueRequest { + string transactions_factory_ref = 1; + string cluster_connection_id = 2; + + option deprecated = true; +} + +message TransactionCleanupQueueResult { + repeated TransactionCleanupAttempt attempts = 1; + + option deprecated = true; +} + + +message ClientRecordRemoveRequest { + string client_uuid = 1; + // Any hooks can only use HookPoints in the CLIENT_RECORD_* range + repeated hooks.transactions.Hook hook = 2; + + // This is only needed for KV operations, so the driver will usually send the default cluster connection. + string cluster_connection_id = 8; + + option deprecated = true; +} + +message ClientRecordRemoveResponse { + bool success = 1; + + option deprecated = true; +} + +// Parameters for the Query Statement +message CommandQueryParameters { + option deprecated = true; + + // Extra deprecated as it was never possible to set timeout on TransactionQueryOptions + int32 timeout = 10 [ deprecated = true ]; + + shared.ScanConsistency scan_consistency = 11; +} diff --git a/fit-performer/proto/transactions.options.proto b/fit-performer/proto/transactions.options.proto new file mode 100644 index 00000000..82b91c2b --- /dev/null +++ b/fit-performer/proto/transactions.options.proto @@ -0,0 +1,85 @@ +// All options and config blocks +syntax = "proto3"; + +package protocol.transactions; +option csharp_namespace = "Couchbase.Grpc.Protocol.Transactions"; +option java_package = "com.couchbase.client.protocol.transactions"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/transactions"; +option ruby_package = "FIT::Protocol::Transactions"; +option java_multiple_files = true; + +import "hooks.transactions.proto"; +import "shared.collection.proto"; +import "shared.basic.proto"; +import "shared.transcoders.proto"; + +enum UnstagingMode { + TIME_OPTIMIZED = 0; + MEMORY_OPTIMIZED = 1; +} + +message TransactionQueryOptions { + optional shared.ScanConsistency scan_consistency = 1; + map< string, string > raw = 2; + optional bool adhoc = 3; + // "off", "phases", or "timings" + optional string profile = 4; + optional bool readonly = 5; + repeated string parameters_positional = 6; + map< string, string > parameters_named = 7; + optional bool flex_index = 8; + optional int32 pipeline_cap = 9; + optional int32 pipeline_batch = 10; + optional int32 scan_cap = 11; + optional int32 scan_wait_millis = 12; + optional string client_context_id = 13; +} + +// Options that can be overridden at the individual transaction level. +// Note that pre-ExtSDKIntegration, this struct was empty. Not in the implementation, where PerTransactionConfig existed, +// but here at the FIT level. So, it will only be populated for ExtSDKIntegration performers. +message TransactionOptions { + // Durability to use for all mutating KV operations in this transaction. + optional shared.Durability durability = 1; + + // Timeout (nee expirationTime) for this transaction. + optional int64 timeout_millis = 2; + + // Any hooks can only use HookPoints before the CLEANUP_* range. + repeated hooks.transactions.Hook hook = 3; + + // Will only be sent to ExtSDKIntegration-enabled performers. + optional shared.Collection metadata_collection = 4; + + // Note that PerTransactionQueryConfig is not sent, as it is removed in ExtSDKIntegration. + + // Identifies a tracing span to use as the parent for this transaction. + // Only sent to performers that declare support for Caps.OBSERVABILITY_1. + optional string parent_span_id = 5; +} + +message TransactionGetOptions { + optional shared.Transcoder transcoder = 1; +} + +message TransactionGetMultiOptions { + enum TransactionGetMultiMode { + PRIORITISE_LATENCY = 0; + DISABLE_READ_SKEW_DETECTION = 1; + PRIORITISE_READ_SKEW_DETECTION = 2; + } + + optional TransactionGetMultiMode mode = 1; +} + +message TransactionInsertOptions { + optional shared.Transcoder transcoder = 1; + // Added with EXT_TTL and will only be sent if performer declares support for that + optional shared.Expiry expiry = 2; +} + +message TransactionReplaceOptions { + optional shared.Transcoder transcoder = 1; + // Added with EXT_TTL and will only be sent if performer declares support for that + optional shared.Expiry expiry = 2; +} \ No newline at end of file diff --git a/fit-performer/proto/transactions.performer.proto b/fit-performer/proto/transactions.performer.proto new file mode 100644 index 00000000..00a412e8 --- /dev/null +++ b/fit-performer/proto/transactions.performer.proto @@ -0,0 +1,262 @@ +syntax = "proto3"; + +package protocol.transactions; +option csharp_namespace = "Couchbase.Grpc.Protocol.Transactions"; +option java_package = "com.couchbase.client.protocol.transactions"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/transactions"; +option ruby_package = "FIT::Protocol::Transactions"; +option java_multiple_files = true; + +import "transactions.commands.proto"; +import "hooks.transactions.proto"; +import "transactions.options.proto"; +import "transactions.legacy.proto"; +import "shared.latch.proto"; +import "shared.basic.proto"; +import "shared.collection.proto"; +import "sdk.query.proto"; +import "shared.transaction_config.proto"; + + +message TransactionStartRequest { +} + +message TransactionStreamDriverToPerformer { + oneof request { + // This will setup a TwoWayTransaction but not start it. + // Must only be one of these per stream.. + TransactionCreateRequest create = 1; + + // Starts a previously setup TwoWayTransaction + // Only one of these will be sent per stream, and it will be after the TransactionCreateRequest. + TransactionStartRequest start = 2; + + // A request broadcast from another TwoWayTransaction, that is being broadcast to the rest via the driver. + BroadcastToOtherConcurrentTransactionsRequest broadcast = 3; + } +} + +// This will be propagated to other concurrent transactions, e.g. it can be used for +// gating and communicating between concurrent transactions +message BroadcastToOtherConcurrentTransactionsRequest { + oneof request { + // Countdown a latch + CommandSetLatch latch_set = 2; + } +} + +message TransactionCreated { +} + +message TransactionStreamPerformerToDriver { + oneof response { + // The transaction is completed, and the stream is done + TransactionResult final_result = 1; + + // The transaction has been created e.g. as result of a TransactionCreateRequest, and is now ready + // to start e.g. a TransactionStartRequest + TransactionCreated created = 2; + + BroadcastToOtherConcurrentTransactionsRequest broadcast = 3; + } +} + +message TransactionCreateResponse { + string transaction_ref = 1; +} + +message TransactionGenericResponse { +} + +// The result of closing a transaction: whether the transaction succeeded, its logs, +// any exception thrown... +message TransactionResult { + // With CBD-3802 this is now only checked for the Java performer, other implementations should not provide this. + // It will eventually be removed. + repeated shared.MutationToken mutation_tokens = 1 [ deprecated = true ]; + + // Next three fields are all now deprecated. They are no longer used by FIT and performers do not need to return them. + // They will eventually be removed. + repeated TransactionAttempt attempts = 2 [ deprecated = true ]; + string atrCollection = 3 [ deprecated = true ]; + string atrId = 4 [ deprecated = true ]; + + // Any TransactionFailed error raised by the operation. + TransactionException exception = 5; + + // The cause field from any TransactionFailed error. + ExternalException exception_cause = 10; + + // The implementation's in-memory log. It's optional but helps greatly with debugging. + repeated string log = 6; + + string transaction_id = 7; + + // How many attempts need to be cleaned up, as a direct result of this transaction. + // This field is only valid if cleanupClientAttempts was set (transactions will not queue a cleanup attempt if + // there is nothing to handle it) + int32 cleanup_requests_pending = 8; + + // Whether the cleanupRequestsPending field is valid, e.g. whether it can be checked + bool cleanup_requests_valid = 9; + + bool unstaging_complete = 11; + + // Allows the performer to flag implementation/platform-specific validation failures that cannot be handled generically. + // E.g. if an erroneous event is detected it can be returned here. + // If present, the driver will fail the test, logging this message. + optional string performer_specific_validation = 12; + + // As of ExtSDKIntegration, this is deprecated, and the performer no longer needs to return it - it will be ignored. + // It's somewhat unclear why it was here in the first place. Appears to be used for creating a TransactionCleanupRequest, + // but that can use any clusterConnectionId, including the default. + string cluster_connection_id = 13 [ deprecated = true ]; +} + +// Initiates a single transaction. +message TransactionCreateRequest { + // Non-ExtSDKIntegration: A reference to a previously constructed Transactions object (e.g. factory). + // ExtSDKIntegration: ignore this (it will not be sent), and just use `clusterConnectionId` to get `cluster.transactions()`. + optional string transactions_factory_ref = 1; + + // A transaction can consist of multiple attempts. Usually we want each attempt to do the same thing, + // but there are a handful of tests where we need more complex logic and need to do different things + // on different attempts. + // If the transaction does more attempts than are in this field, the commands in the last attempt are + // what will be repeated. + repeated TransactionAttemptRequest attempts = 2; + + optional TransactionOptions options = 3; + + // Any latches that the transaction should create. + repeated shared.Latch latches = 4; + + // Just for debugging, name the test + string name = 5; + + // Instead of the performer returning any raised events, the driver specifies which are expected, or explicitly + // not expected. + // Event buses can be asynchronous so this prevents the performer having to wait for an unknown amount of + // time on each test. + // Also, each implementation handles events differently - some use an event bus, some simply log. So this allows + // the performer to also check as appropriate. + // For expectedAbsenceOfEvents, the performer should pick an appropriate amount of time to wait for events before + // deciding an event will definitely not arrive. + repeated Event expected_events = 6; + repeated Event expected_absence_of_events = 9; + + // Instead of the performer returning any raised events, the driver specifies which are explicitly not expected. + // Event buses can be asynchronous so this prevents the performer having to wait for an unknown amount of + // time on each test. + + // Used for ExtSDKIntegration to create the transaction, and both in that and in non-ExtSDKIntegration to access + // collections for KV operations. + string cluster_connection_id = 7; + + // The API to use for this transaction. See the comments for `API`. + shared.API api = 8; +} + +message TransactionAttemptRequest { + repeated TransactionCommand commands = 1; +} + +// Runs a single query transaction (tximplicit). +// ExtSDKIntegration: cluster.query() +// Non-ExtSDKIntegration: transactions.query() +message TransactionSingleQueryRequest { + // A reference to a previously constructed Transactions object (e.g. factory). + // Not sent to ExtSDKIntegration performers. + optional string transactions_factory_ref = 1; + + // A reference to a previously constructed cluster connection. + // Will always be sent, but is only used by ExtSDKIntegration performers. + string cluster_connection_id = 2; + + // The query to run + CommandQuery query = 3; + + // Config needs handling differently in the performer based on ExtSDKIntegration or not. The structure of these + // messages is based around how things work in ExtSDKIntegration, and need some finangling for non-ExtSDKIntegration + // performers. + // + // ExtSDKIntegration: + // - The user calls `(cluster|scope).query(String, queryOptions().asTransaction([SingleQueryTransactionOptions]))`. + // - Everything in GRPC QueryOptions should be passed to an SDK QueryOptions. + // - The nested GRPC SingleQueryTransactionOptions should be passed as the SDK SingleQueryTransactionOptions block on + // QueryOptions.asTransaction(), if it is present. Otherwise call QueryOptions.asTransaction() with no args. + // + // Non-ExtSDKIntegration: + // - The user calls `transactions().query([Scope], String, [SingleQueryTransactionConfig])` + // - SDK SingleQueryTransactionConfig contains an SDK TransactionQueryOptions. This should be populated from GRPC QueryOptions. + // -- If no GRPC QueryOptions settings are specified, no SDK TransactionQueryOptions should be created. + // -- The GRPC QueryOptions.timeout() field must be applied at the SDK SingleQueryTransactionConfig level instead. + // - SDK SingleQueryTransactionConfig is populated from GRPC QueryOptions.SingleQueryTransactionOptions. + // - GRPC SingleQueryTransactionOptions.metadataCollection can be ignored. That was added in ExtSDKIntegration. + optional sdk.query.QueryOptions query_options = 4; + + // The API to use for this transaction. See the comments for `API`. + shared.API api = 8; +} + +message TransactionSingleQueryResponse { + // Deprecated as this was from the initial version of ExtSingleQuery, pre SDK-integration. + // The replacement fields below are compatible with the SDK integration, when only a QueryResult (or + // TransactionFailed-derived error) is going to be returned to the user. + // If `result` is returned then FIT will validate that, else it will use the replacement fields. + TransactionResult result = 1 [ deprecated = true ]; + + // The fields below replace the deprecated `result`, which will ultimately be removed. + + // Any TransactionFailed error raised by the operation, at the point of performing the cluster.query()/transactions.query() + // If it threw a non-TransactionFailed exception, return EXCEPTION_UNKNOWN + TransactionException exception = 2; + + // If `exception` is EXCEPTION_UNKNOWN, then this is the exception raised at the point of performing the cluster.query()/transactions.query(). + // e.g. this is how non-TransactionFailed errors are indicated. + // Otherwise, it's the cause field of the TransactionFailed. + ExternalException exception_cause = 3; + + // Any TransactionFailed error raised by the operation, at the point of streaming the rows. + TransactionException exception_during_streaming = 4; + + // If `exceptionDuringStreaming` is NO_EXCEPTION_THROWN, then this is the exception raised while streaming the rows. + // e.g. this is how non-TransactionFailed errors are indicated. + // Otherwise, it's the cause field of the TransactionFailed. + ExternalException exception_cause_during_streaming = 5; + + // The implementation's in-memory log. It's optional (and not possible to return when the SDK integrated cluster.query() + // succeeds), but helps greatly with debugging on failures. + repeated string log = 6; + + // Allows the performer to flag implementation/platform-specific validation failures that cannot be handled generically. + // E.g. if an erroneous event is detected it can be returned here. + // If present, the driver will fail the test, logging this message. + optional string performer_specific_validation = 7; +} + +// Creates a Transactions object, e.g. a transactions factory +message TransactionsFactoryCreateRequest { + // The following fields are deprecated and replaced by the `config` field (which allows optional fields) + // They will still be sent by FIT, but performers should remove them while implementing the breaking change for + // ExtSDKIntegration + shared.Durability durability = 1 [ deprecated = true ]; + int32 expiration_millis = 2 [ deprecated = true ]; + bool cleanup_lost_attempts = 4 [ deprecated = true ]; + bool cleanup_client_attempts = 5 [ deprecated = true ]; + int64 cleanup_window_millis = 6 [ deprecated = true ]; + UnstagingMode unstaging_mode = 7 [ deprecated = true ]; + shared.Collection metadata_collection = 8 [ deprecated = true ]; + // Any hooks can only use HookPoints before the CLEANUP_* range + repeated hooks.transactions.Hook hook = 3 [ deprecated = true ]; + + // The Cluster to use for `Transactions.create()` + string cluster_connection_id = 9; + + optional shared.TransactionsConfig config = 10; +} + +// Shuts down a previously created Transactions (factory) +message TransactionsFactoryCloseRequest { + string transactions_factory_ref = 1; +} diff --git a/fit-performer/proto/transactions.workload.proto b/fit-performer/proto/transactions.workload.proto new file mode 100644 index 00000000..906082b6 --- /dev/null +++ b/fit-performer/proto/transactions.workload.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; + +package protocol.transactions; +option csharp_namespace = "Couchbase.Grpc.Protocol.Transactions"; +option java_package = "com.couchbase.client.protocol.transactions"; +option go_package = "github.com/couchbaselabs/transactions-fit-performer/protocol/transactions"; +option ruby_package = "FIT::Protocol::Transactions"; +option java_multiple_files = true; + +import "transactions.performer.proto"; +import "shared.bounds.proto"; + +message Workload { + // The commands to run. Each command should send back one TransactionResult. The commands should be executed in a loop + // following `bounds`. E.g. the performer may go through all the commands multiple times. + // + // If the command fails, the performer should not propagate the failure (but should still send back an TransactionResult). + // E.g. it should continue until bounds is exhausted (or it has run all commands, if bounds is not specified). + repeated TransactionCreateRequest command = 1; + + // Controls how the commands should be run. + // If it's not present, just run the commands through once. + optional shared.Bounds bounds = 2; + + // Performance mode indicates that this is a performance test, where the performer should disable various functionality. + // This includes: + // + // 1. Minimise the returned result + // A TransactionResult is heavy-weight, including things like a full log of the transaction. + // This requests that a minimal TransactionResult be returned, to reduce the performance impact. + // Specifically, on success the performer should only set exception=NO_EXCEPTION_THROWN, and populate no other fields. + // On failure, the performer should populate a full TransactionResult as normal - as it's expected that normal + // performance tests should not include failed transactions). + // + // 2. Reduce all performer logging to minimise impact. The transactions/SDK logging though should not be affected - this + // should be configured through the standard configuration paths. Because we will want to measure the impact of + // having it enabled. + bool performance_mode = 3; +} + diff --git a/fit-performer/server.rb b/fit-performer/server.rb new file mode 100644 index 00000000..1647a02a --- /dev/null +++ b/fit-performer/server.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +# Copyright 2026-Present Couchbase, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +$LOAD_PATH << File.expand_path('lib') + +require 'logger' +require 'optparse' +require 'grpc' +require 'fit/performer/service' + +$stdout.sync = true # For logging to appear when running in docker +logger = Logger.new($stdout) + +port = "0.0.0.0:8060" +server = GRPC::RpcServer.new +server.add_http2_port(port, :this_port_is_insecure) +logger.info("Listening on #{port}") +server.handle(FIT::Performer::Service.new) +server.run_till_terminated_or_interrupted([1, 'int', 'SIGQUIT']) diff --git a/task/grpc.rake b/task/grpc.rake index f6e0ffe8..47bd1ecd 100644 --- a/task/grpc.rake +++ b/task/grpc.rake @@ -15,26 +15,7 @@ # limitations under the License. require "tmpdir" - -def find_executable(cmd) - exts = RbConfig::CONFIG["EXECUTABLE_EXTS"].split | [RbConfig::CONFIG["EXEEXT"]] - ENV["PATH"].split(File::PATH_SEPARATOR).each do |path| - path.strip! - next if path.empty? - - path = File.join(path, cmd) - exts.each do |ext| - executable_path = path + ext - if File.executable?(executable_path) - puts("# found #{cmd}: #{executable_path}") - return executable_path - end - end - end - abort "#{cmd}: command not found" -end - -require "open-uri" +require "mkmf" desc "Re-generate gRPC client implementation" task :generate_grpc_client do