diff --git a/chasm/context_mock.go b/chasm/context_mock.go index 92b05f780b7..db6727a7bb4 100644 --- a/chasm/context_mock.go +++ b/chasm/context_mock.go @@ -46,6 +46,16 @@ type MockContext struct { registeredContextValues map[any]any } +// RegisterLibrary copies the context values that lib's components declare via +// [WithContextValues] into the mock, mirroring what [Registry] does in production. Use this in +// tests that reach into another library's component methods, so that library can keep its context +// keys unexported. +func (c *MockContext) RegisterLibrary(lib Library) { + for _, rc := range lib.Components() { + c.RegisterComponentContextValues(rc.contextValues) + } +} + func (c *MockContext) RegisterComponentContextValues( keyValues map[any]any, ) { @@ -138,7 +148,11 @@ func (c *MockContext) MetricsHandler() metrics.Handler { } func (c *MockContext) Value(key any) any { - return c.goContext().Value(key) + if v := c.goContext().Value(key); v != nil { + return v + } + + return c.registeredContextValues[key] } func (c *MockContext) Links(component Component) []*commonpb.Link { @@ -175,6 +189,8 @@ func (c *MockContext) withValue(key any, value any) Context { HandleLinks: c.HandleLinks, HandleRequestLinks: c.HandleRequestLinks, HandleUserMetadata: c.HandleUserMetadata, + + registeredContextValues: c.registeredContextValues, } } diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index 04a65799714..43939c27cbe 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -17,7 +17,6 @@ import ( "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" "go.temporal.io/server/chasm/lib/callback" - callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common" "go.temporal.io/server/common/contextutil" "go.temporal.io/server/common/metrics" @@ -326,24 +325,14 @@ func (a *Activity) addCompletionCallbacks( registrationTime := timestamppb.New(ctx.Now(a)) for idx, cb := range completionCallbacks { - chasmCB := &callbackspb.Callback{ - Links: cb.GetLinks(), - } - switch variant := cb.Variant.(type) { - case *commonpb.Callback_Nexus_: - chasmCB.Variant = &callbackspb.Callback_Nexus_{ - Nexus: &callbackspb.Callback_Nexus{ - Url: variant.Nexus.GetUrl(), - Header: variant.Nexus.GetHeader(), - }, - } - default: - return serviceerror.NewInvalidArgumentf("unsupported callback variant: %T", variant) + chasmCB, err := callback.FromAPICallback(cb) + if err != nil { + return err } // requestID (unique per API call) + idx (position within the request) ensures unique,idempotent callback IDs. id := fmt.Sprintf("%s-%d", requestID, idx) - callbackObj := callback.NewCallback(requestID, registrationTime, &callbackspb.CallbackState{}, chasmCB) + callbackObj := callback.NewCallback(requestID, registrationTime, chasmCB) a.Callbacks[id] = chasm.NewComponentField(ctx, callbackObj) } return nil diff --git a/chasm/lib/activity/config.go b/chasm/lib/activity/config.go index 311e4744f4b..a6551f49dd7 100644 --- a/chasm/lib/activity/config.go +++ b/chasm/lib/activity/config.go @@ -3,6 +3,7 @@ package activity import ( "go.temporal.io/server/chasm/lib/callback" "go.temporal.io/server/common" + "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/retrypolicy" ) @@ -40,6 +41,14 @@ var ( `Allows attaching completion callbacks to standalone activity executions.`, ) + EnabledCallbackKinds = dynamicconfig.NewNamespaceTypedSettingWithConverter( + "activity.enabledCallbackKinds", + callbacks.ConvertEnabledKinds, + []callbacks.Kind{callbacks.KindNexus}, + `The list of completion callback kinds that may be attached to a standalone activity execution. +Only consulted when activity.enableCallbacks is set.`, + ) + EnableStandaloneActivityOperatorCommands = dynamicconfig.NewNamespaceBoolSetting( "history.enableStandaloneActivityOperatorCommands", false, @@ -52,6 +61,7 @@ type Config struct { BlobSizeLimitWarn dynamicconfig.IntPropertyFnWithNamespaceFilter BreakdownMetricsByTaskQueue dynamicconfig.TypedPropertyFnWithTaskQueueFilter[bool] EnableCallbacks dynamicconfig.BoolPropertyFnWithNamespaceFilter + EnabledCallbackKinds dynamicconfig.TypedPropertyFnWithNamespaceFilter[[]callbacks.Kind] Enabled dynamicconfig.BoolPropertyFnWithNamespaceFilter EnableStandaloneActivityOperatorCommands dynamicconfig.BoolPropertyFnWithNamespaceFilter LongPollBuffer dynamicconfig.DurationPropertyFnWithNamespaceFilter @@ -73,6 +83,7 @@ func ConfigProvider(dc *dynamicconfig.Collection) *Config { BreakdownMetricsByTaskQueue: dynamicconfig.MetricsBreakdownByTaskQueue.Get(dc), DefaultActivityRetryPolicy: dynamicconfig.DefaultActivityRetryPolicy.Get(dc), EnableCallbacks: EnableCallbacks.Get(dc), + EnabledCallbackKinds: EnabledCallbackKinds.Get(dc), Enabled: Enabled.Get(dc), EnableStandaloneActivityOperatorCommands: EnableStandaloneActivityOperatorCommands.Get(dc), LongPollBuffer: LongPollBuffer.Get(dc), diff --git a/chasm/lib/activity/frontend.go b/chasm/lib/activity/frontend.go index 26596b08946..3bf32ec5964 100644 --- a/chasm/lib/activity/frontend.go +++ b/chasm/lib/activity/frontend.go @@ -418,7 +418,10 @@ func (h *frontendHandler) validateAndPopulateStartRequest( if !h.config.EnableCallbacks(req.GetNamespace()) { return nil, serviceerror.NewInvalidArgument("completion callbacks are not enabled for this namespace") } - if err := h.callbackValidator.Validate(ctx, req.GetNamespace(), cbs); err != nil { + opts := callbacks.ValidatorOptions{ + EnabledKinds: h.config.EnabledCallbackKinds(req.GetNamespace()), + } + if err := h.callbackValidator.Validate(ctx, req.GetNamespace(), cbs, opts); err != nil { return nil, err } } diff --git a/chasm/lib/activity/responses.go b/chasm/lib/activity/responses.go index 8aaa11103fb..340b21ca11c 100644 --- a/chasm/lib/activity/responses.go +++ b/chasm/lib/activity/responses.go @@ -4,15 +4,12 @@ import ( "fmt" apiactivitypb "go.temporal.io/api/activity/v1" //nolint:importas - callbackpb "go.temporal.io/api/callback/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" - "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" - callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -205,42 +202,16 @@ func (a *Activity) buildCallbackInfos(ctx chasm.Context) ([]*apiactivitypb.Callb for _, field := range a.Callbacks { cb := field.Get(ctx) - cbSpec, err := cb.ToAPICallback() + cbInfo, err := cb.ToAPICallbackInfo(ctx) if err != nil { return nil, err } - var state enumspb.CallbackState - switch cb.Status { - case callbackspb.CALLBACK_STATUS_UNSPECIFIED: - return nil, serviceerror.NewInternal("callback with UNSPECIFIED state") - case callbackspb.CALLBACK_STATUS_STANDBY: - state = enumspb.CALLBACK_STATE_STANDBY - case callbackspb.CALLBACK_STATUS_SCHEDULED: - state = enumspb.CALLBACK_STATE_SCHEDULED - case callbackspb.CALLBACK_STATUS_BACKING_OFF: - state = enumspb.CALLBACK_STATE_BACKING_OFF - case callbackspb.CALLBACK_STATUS_FAILED: - state = enumspb.CALLBACK_STATE_FAILED - case callbackspb.CALLBACK_STATUS_SUCCEEDED: - state = enumspb.CALLBACK_STATE_SUCCEEDED - default: - return nil, serviceerror.NewInternalf("unknown callback state: %v", cb.Status) - } - cbInfos = append(cbInfos, &apiactivitypb.CallbackInfo{ Trigger: &apiactivitypb.CallbackInfo_Trigger{ Variant: &apiactivitypb.CallbackInfo_Trigger_ActivityClosed{}, }, - Info: &callbackpb.CallbackInfo{ - Callback: cbSpec, - RegistrationTime: cb.RegistrationTime, - State: state, - Attempt: cb.Attempt, - LastAttemptCompleteTime: cb.LastAttemptCompleteTime, - LastAttemptFailure: cb.LastAttemptFailure, - NextAttemptScheduleTime: cb.NextAttemptScheduleTime, - }, + Info: cbInfo, }) } return cbInfos, nil diff --git a/chasm/lib/activity/validator_test.go b/chasm/lib/activity/validator_test.go index 95a725ef37f..8118308d071 100644 --- a/chasm/lib/activity/validator_test.go +++ b/chasm/lib/activity/validator_test.go @@ -536,6 +536,7 @@ func TestRequestIDGeneratedWhenMissing(t *testing.T) { func TestValidateAndPopulateStartRequest_CombinesRequestAndCallbackLinks(t *testing.T) { callbackValidator, err := callbacks.NewValidator(callbacks.ValidatorConfig{ MaxCallbacksPerExecution: func(string) int { return 2000 }, + MaxIDLengthLimit: func() int { return 1000 }, URLMaxLength: func(string) int { return 1000 }, HeaderMaxSize: func(string) int { return 2000 }, EndpointRules: func(string) callbacks.AddressMatchRules { @@ -545,6 +546,10 @@ func TestValidateAndPopulateStartRequest_CombinesRequestAndCallbackLinks(t *test }, } }, + MaxServiceNameLength: func(string) int { return 1000 }, + MaxOperationNameLength: func(string) int { return 1000 }, + WorkerSourceContextMaxSize: func(string) int { return 64 * 1024 }, + WorkerSourceContextAggregateMaxSize: func(string) int { return 2 * 1024 * 1024 }, }) require.NoError(t, err) @@ -554,6 +559,9 @@ func TestValidateAndPopulateStartRequest_CombinesRequestAndCallbackLinks(t *test BlobSizeLimitWarn: defaultBlobSizeLimitWarn, DefaultActivityRetryPolicy: getDefaultRetrySettings, EnableCallbacks: func(string) bool { return true }, + EnabledCallbackKinds: func(string) []callbacks.Kind { + return []callbacks.Kind{callbacks.KindNexus} + }, MaxIDLengthLimit: func() int { return defaultMaxIDLengthLimit }, MaxUserMetadataDetailsSize: defaultMaxUserMetadataDetailsSize, MaxUserMetadataSummarySize: defaultMaxUserMetadataSummarySize, @@ -580,8 +588,10 @@ func TestValidateAndPopulateStartRequest_CombinesRequestAndCallbackLinks(t *test }, }}, CompletionCallbacks: []*commonpb.Callback{{ - Variant: &commonpb.Callback_Internal_{ - Internal: &commonpb.Callback_Internal{}, + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: "http://localhost/cb", + }, }, Links: []*commonpb.Link{{ Variant: &commonpb.Link_BatchJob_{ diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 1ca220a3197..8ecca031dd6 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -3,16 +3,19 @@ package callback import ( "fmt" "maps" - "slices" "time" + callbackpb "go.temporal.io/api/callback/v1" commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/nexus/nexusrpc" queueserrors "go.temporal.io/server/service/history/queues/errors" + "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -37,7 +40,6 @@ type Callback struct { func NewCallback( requestID string, registrationTime *timestamppb.Timestamp, - state *callbackspb.CallbackState, cb *callbackspb.Callback, ) *Callback { return &Callback{ @@ -79,21 +81,35 @@ func (c *Callback) loadInvocationArgs( ctx chasm.Context, _ chasm.NoValue, ) (invocable, error) { - target := c.CompletionSource.Get(ctx) + // Reject unknown/unsupported callback variants. + switch c.GetCallback().GetVariant().(type) { + case *callbackspb.Callback_Nexus_, *callbackspb.Callback_Worker_: + // OK + default: + return nil, queueserrors.NewUnprocessableTaskError( + fmt.Sprintf("unprocessable callback variant: %T", c.GetCallback().GetVariant()), + ) + } + // Get the parent CHASM object's Nexus result to be delivered. + target := c.CompletionSource.Get(ctx) completion, err := target.GetNexusCompletion(ctx, c.RequestId) if err != nil { return nil, err } - callback := c.GetCallback().GetNexus() - if callback == nil { - return nil, queueserrors.NewUnprocessableTaskError( - fmt.Sprintf("unprocessable callback variant: %v", callback), - ) + if worker := c.GetCallback().GetWorker(); worker != nil { + return invocableWorker{ + callback: worker, + completion: completion, + startTime: ctx.Now(c), + requestID: c.RequestId, + attempt: c.Attempt, + }, nil } - if callback.Url == chasm.NexusCompletionHandlerURL { + callback := c.GetCallback().GetNexus() + if callback.GetUrl() == chasm.NexusCompletionHandlerURL { return invocableInternal{ callback: callback, attempt: c.Attempt, @@ -143,16 +159,22 @@ func (c *Callback) saveResult( } } +// SourceContextSize returns the size in bytes of the Worker source context this callback carries, +// or 0 for any other variant. Used to enforce an aggregate cap for an execution's callbacks. +func (c *Callback) SourceContextSize() int { + return c.GetCallback().GetWorker().GetSourceContext().Size() +} + // ToAPICallback converts a CHASM callback to API callback proto. func (c *Callback) ToAPICallback() (*commonpb.Callback, error) { // Convert CHASM callback proto to API callback proto chasmCB := c.GetCallback() res := &commonpb.Callback{ - Links: slices.Clone(chasmCB.GetLinks()), + Links: common.CloneProtoSlice(chasmCB.GetLinks()), } - // CHASM currently only supports Nexus callbacks - if variant, ok := chasmCB.Variant.(*callbackspb.Callback_Nexus_); ok { + switch variant := chasmCB.GetVariant().(type) { + case *callbackspb.Callback_Nexus_: res.Variant = &commonpb.Callback_Nexus_{ Nexus: &commonpb.Callback_Nexus{ Url: variant.Nexus.GetUrl(), @@ -160,10 +182,143 @@ func (c *Callback) ToAPICallback() (*commonpb.Callback, error) { }, } return res, nil + case *callbackspb.Callback_Worker_: + res.Variant = &commonpb.Callback_Worker_{ + Worker: &commonpb.Callback_Worker{ + TaskQueueName: variant.Worker.GetTaskQueueName(), + Service: variant.Worker.GetService(), + Operation: variant.Worker.GetOperation(), + SourceContext: common.CloneProto(variant.Worker.GetSourceContext()), + }, + } + return res, nil + default: + return nil, serviceerror.NewInternalf("unsupported CHASM callback type: %T", variant) + } +} + +// setResult populates the Result field of the supplied proto based on the Callback's state. +// (Including nil if the Callback has not completed.) +func (c *Callback) setResult(cbi *callbackpb.CallbackInfo) { + switch c.Status { + case callbackspb.CALLBACK_STATUS_SUCCEEDED: + cbi.Result = &callbackpb.CallbackInfo_Success{ + Success: &emptypb.Empty{}, + } + case callbackspb.CALLBACK_STATUS_FAILED: + // A callback can only fail on a non-retryable delivery error, recorded in LastAttemptFailure. + cbi.Result = &callbackpb.CallbackInfo_Failure{ + Failure: common.CloneProto(c.LastAttemptFailure), + } + default: + cbi.Result = nil } +} + +// APIState converts the CHASM callback status to the API CallbackState enum along with the relevant +// circuit breaker's blocking status. +func (c *Callback) APIState(ctx chasm.Context) (enumspb.CallbackState, string, error) { + state, err := c.apiStatus() + if err != nil { + return enumspb.CALLBACK_STATE_UNSPECIFIED, "", err + } + + // The circuit breaker is only relevant for scheduled callbacks. + if state != enumspb.CALLBACK_STATE_SCHEDULED { + return state, "", nil + } + + cbCtx := callbackContextFromChasm(ctx) + destination, err := c.Destination() + if err != nil { + return enumspb.CALLBACK_STATE_UNSPECIFIED, "", err + } + if !cbCtx.destinationBlocked(ctx.ExecutionKey().NamespaceID, destination) { + return state, "", nil + } + return enumspb.CALLBACK_STATE_BLOCKED, "The circuit breaker is open.", nil +} + +func (c *Callback) apiStatus() (enumspb.CallbackState, error) { + switch c.Status { + case callbackspb.CALLBACK_STATUS_STANDBY: + return enumspb.CALLBACK_STATE_STANDBY, nil + case callbackspb.CALLBACK_STATUS_SCHEDULED: + return enumspb.CALLBACK_STATE_SCHEDULED, nil + case callbackspb.CALLBACK_STATUS_BACKING_OFF: + return enumspb.CALLBACK_STATE_BACKING_OFF, nil + case callbackspb.CALLBACK_STATUS_FAILED: + return enumspb.CALLBACK_STATE_FAILED, nil + case callbackspb.CALLBACK_STATUS_SUCCEEDED: + return enumspb.CALLBACK_STATE_SUCCEEDED, nil + case callbackspb.CALLBACK_STATUS_UNSPECIFIED: + return enumspb.CALLBACK_STATE_UNSPECIFIED, serviceerror.NewInternal("callback with UNSPECIFIED state") + default: + return enumspb.CALLBACK_STATE_UNSPECIFIED, serviceerror.NewInternalf("unknown callback state: %v", c.Status) + } +} + +// ToAPICallbackInfo returns the API CallbackInfo based on the current state of the CHASM component. +func (c *Callback) ToAPICallbackInfo(ctx chasm.Context) (*callbackpb.CallbackInfo, error) { + apiCb, err := c.ToAPICallback() + if err != nil { + return nil, err + } + apiState, blockedReason, err := c.APIState(ctx) + if err != nil { + return nil, err + } + + info := &callbackpb.CallbackInfo{ + Callback: apiCb, + RegistrationTime: common.CloneProto(c.RegistrationTime), + State: apiState, + BlockedReason: blockedReason, + RequestId: c.RequestId, + Attempt: c.Attempt, + LastAttemptCompleteTime: common.CloneProto(c.LastAttemptCompleteTime), + LastAttemptFailure: common.CloneProto(c.LastAttemptFailure), + NextAttemptScheduleTime: common.CloneProto(c.NextAttemptScheduleTime), + } + c.setResult(info) + return info, nil +} + +// FromAPICallback converts an API callback into a CHASM callback proto. +func FromAPICallback(cb *commonpb.Callback) (*callbackspb.Callback, error) { + res := &callbackspb.Callback{ + Links: common.CloneProtoSlice(cb.GetLinks()), + } + + switch variant := cb.GetVariant().(type) { + case *commonpb.Callback_Nexus_: + res.Variant = &callbackspb.Callback_Nexus_{ + Nexus: &callbackspb.Callback_Nexus{ + Url: variant.Nexus.GetUrl(), + Header: maps.Clone(variant.Nexus.GetHeader()), + }, + } + return res, nil + case *commonpb.Callback_Worker_: + res.Variant = &callbackspb.Callback_Worker_{ + Worker: &callbackspb.Callback_Worker{ + TaskQueueName: variant.Worker.GetTaskQueueName(), + Service: variant.Worker.GetService(), + Operation: variant.Worker.GetOperation(), + SourceContext: common.CloneProto(variant.Worker.GetSourceContext()), + }, + } + return res, nil + default: + return nil, serviceerror.NewInvalidArgumentf("unsupported callback variant: %T", variant) + } +} - // This should not happen as CHASM only supports Nexus callbacks currently - return nil, serviceerror.NewInternal("unsupported CHASM callback type") +// Destination returns the destination this callback's invocation tasks are grouped under. Callers +// outside this package need it to look the callback up in per-destination structures such as the +// outbound queue's circuit breaker pool. +func (c *Callback) Destination() (string, error) { + return callbackDestination(c.GetCallback()) } // ScheduleStandbyCallbacks transitions all STANDBY callbacks to SCHEDULED state, diff --git a/chasm/lib/callback/component_test.go b/chasm/lib/callback/component_test.go new file mode 100644 index 00000000000..9f553e18c15 --- /dev/null +++ b/chasm/lib/callback/component_test.go @@ -0,0 +1,256 @@ +package callback + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + callbackpb "go.temporal.io/api/callback/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/testing/protorequire" + queueserrors "go.temporal.io/server/service/history/queues/errors" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestFromAPICallback(t *testing.T) { + // Set of API Callback variants to test. + apiCallbackVariants := map[string]struct { + callback *commonpb.Callback + // Whether CHASM can persist this variant. Persistable variants must round trip. + persistable bool + }{ + "nexus": { + callback: &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: "http://localhost:8080/cb", + Header: map[string]string{"key": "value"}, + }, + }, + }, + persistable: true, + }, + "worker": { + callback: &commonpb.Callback{ + Variant: &commonpb.Callback_Worker_{ + Worker: &commonpb.Callback_Worker{ + TaskQueueName: "completions-task-queue", + Service: "HTTPAdapter", + Operation: "DeliverAsWebhook", + SourceContext: &commonpb.Payload{Data: []byte("...")}, + }, + }, + }, + persistable: true, + }, + "internal": { + callback: &commonpb.Callback{ + Variant: &commonpb.Callback_Internal_{Internal: &commonpb.Callback_Internal{}}, + }, + // Not defined in the CHASM callback proto. + persistable: false, + }, + "unset": {callback: &commonpb.Callback{}}, + } + + t.Run("RoundTripped", func(t *testing.T) { + for name, tc := range apiCallbackVariants { + t.Run(name, func(t *testing.T) { + got, err := FromAPICallback(tc.callback) + + // Error case, for invalid or unknown API callbacks. + if !tc.persistable { + var invalidArgErr *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgErr) + require.ErrorContains(t, err, "unsupported callback variant") + return + } + require.NoError(t, err) + + // Verify round-tripping the proto produces the same result. + chasmComponent := &Callback{ + CallbackState: &callbackspb.CallbackState{ + Callback: got, + }, + } + roundTripped, err := chasmComponent.ToAPICallback() + require.NoError(t, err) + protorequire.ProtoEqual(t, tc.callback, roundTripped) + }) + } + }) + + t.Run("LinksPersistedForVariants", func(t *testing.T) { + links := []*commonpb.Link{ + { + Variant: &commonpb.Link_WorkflowEvent_{ + WorkflowEvent: &commonpb.Link_WorkflowEvent{Namespace: "ns", WorkflowId: "wf-id"}, + }, + }, + { + Variant: &commonpb.Link_Callback_{ + Callback: &commonpb.Link_Callback{ + Execution: &commonpb.Execution{ + Type: enumspb.EXECUTION_TYPE_NEXUS_OPERATION, + BusinessId: "nexus-operation-id", + RunId: "run-id", + }, + RequestId: "request-id", + }, + }, + }, + } + + for name, tc := range apiCallbackVariants { + // This test only applies to callbacks that can be converted. + if !tc.persistable { + continue + } + + t.Run(name, func(t *testing.T) { + cbWithLinks := common.CloneProto(tc.callback) + cbWithLinks.Links = links + + got, err := FromAPICallback(cbWithLinks) + require.NoError(t, err) + + // Verify links were converted in the process. + gotLinks := got.GetLinks() + require.Len(t, gotLinks, 2) + require.NotNil(t, gotLinks[0].GetWorkflowEvent()) + require.NotNil(t, gotLinks[1].GetCallback()) + + // Verify that a deep copy was used. (Different references.) + require.NotSame(t, links[0], gotLinks[0]) + require.NotSame(t, links[1], gotLinks[1]) + }) + } + }) + + // Confirm a malformed CHASM Callback fails to be converted into the api proto. + t.Run("UnsupportedVariant", func(t *testing.T) { + cb := &Callback{CallbackState: &callbackspb.CallbackState{ + Callback: &callbackspb.Callback{}, + }} + _, err := cb.ToAPICallback() + var internalErr *serviceerror.Internal + require.ErrorAs(t, err, &internalErr) + require.ErrorContains(t, err, "unsupported CHASM callback type") + }) +} + +// A callback whose variant this server doesn't know how to invoke can still be persisted (by a server that +// does, or by a future version), so its invocation task has to be rejected rather than crash. +func TestLoadInvocationArgsUnsupportedVariant(t *testing.T) { + cb := &Callback{ + CallbackState: &callbackspb.CallbackState{ + Callback: &callbackspb.Callback{}, + }, + } + _, err := cb.loadInvocationArgs(&chasm.MockMutableContext{}, nil) + + var unprocessableErr *queueserrors.UnprocessableTaskError + require.ErrorAs(t, err, &unprocessableErr) + require.ErrorContains(t, err, "unprocessable callback variant") +} + +// Confirm the request ID passed to NewCallback is perssited, and set in ToAPICallbackInfo. +func TestToAPICallbackInfoCarriesTheRequestID(t *testing.T) { + ctx := &chasm.MockContext{} + ctx.RegisterLibrary(NewNilLibrary()) + + cb := NewCallback( + "callback-request-id", + timestamppb.New(time.Unix(1, 0)), + &callbackspb.Callback{Variant: &callbackspb.Callback_Worker_{ + Worker: &callbackspb.Callback_Worker{ + TaskQueueName: "completions-task-queue", + Service: "HTTPAdapter", + Operation: "DeliverAsWebhook", + }, + }}, + ) + + info, err := cb.ToAPICallbackInfo(ctx) + require.NoError(t, err) + require.Equal(t, "callback-request-id", info.GetRequestId()) +} + +// Verify the setResult method sets the "result" field based on the Callback state. +func TestSetResult(t *testing.T) { + lastAttemptFailure := &failurepb.Failure{Message: "last attempt"} + + cases := []struct { + name string + + // Callback state to set. + status callbackspb.CallbackStatus + lastAttemptFailure *failurepb.Failure + + // The CallbackInfo expected after setResult. + want *callbackpb.CallbackInfo + }{ + { + name: "unspecified is non-terminal", + status: callbackspb.CALLBACK_STATUS_UNSPECIFIED, + want: &callbackpb.CallbackInfo{}, + }, + { + name: "standby is non-terminal", + status: callbackspb.CALLBACK_STATUS_STANDBY, + want: &callbackpb.CallbackInfo{}, + }, + { + name: "scheduled is non-terminal", + status: callbackspb.CALLBACK_STATUS_SCHEDULED, + want: &callbackpb.CallbackInfo{}, + }, + { + name: "backing off is non-terminal, even with a last attempt failure", + status: callbackspb.CALLBACK_STATUS_BACKING_OFF, + lastAttemptFailure: lastAttemptFailure, + want: &callbackpb.CallbackInfo{}, + }, + { + name: "succeeded", + status: callbackspb.CALLBACK_STATUS_SUCCEEDED, + want: &callbackpb.CallbackInfo{ + Result: &callbackpb.CallbackInfo_Success{Success: &emptypb.Empty{}}, + }, + }, + { + name: "failed reports the terminal failure", + status: callbackspb.CALLBACK_STATUS_FAILED, + lastAttemptFailure: lastAttemptFailure, + want: &callbackpb.CallbackInfo{ + Result: &callbackpb.CallbackInfo_Failure{Failure: lastAttemptFailure}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cb := &Callback{ + CallbackState: &callbackspb.CallbackState{ + Status: tc.status, + LastAttemptFailure: tc.lastAttemptFailure, + }, + } + + var cbInfo callbackpb.CallbackInfo + cb.setResult(&cbInfo) + + protorequire.ProtoEqual(t, tc.want, &cbInfo) + if gotFailure := cbInfo.GetFailure(); gotFailure != nil { + require.NotSame(t, tc.lastAttemptFailure, gotFailure) + } + }) + } +} diff --git a/chasm/lib/callback/config.go b/chasm/lib/callback/config.go index 7149c423871..59ec53bc04a 100644 --- a/chasm/lib/callback/config.go +++ b/chasm/lib/callback/config.go @@ -14,6 +14,20 @@ var MaxPerExecution = dynamicconfig.NewNamespaceIntSetting( `MaxPerExecution is the maximum number of callbacks that can be attached to an execution (workflow or standalone activity).`, ) +var WorkerSourceContextMaxSize = dynamicconfig.NewNamespaceIntSetting( + "callback.worker.sourceContext.maxSize", + 64*1024, + `The maximum allowed size, in bytes, of the opaque source context attached to a single Worker +completion callback. The server carries this payload to the callback's handler untouched.`, +) + +var WorkerSourceContextAggregateMaxSize = dynamicconfig.NewNamespaceIntSetting( + "callback.worker.sourceContext.aggregateMaxSize", + 2*1024*1024, + `The maximum allowed total size, in bytes, of the source context payloads carried by all Worker +completion callbacks on an execution.`, +) + var RequestTimeout = dynamicconfig.NewDestinationDurationSetting( "callback.request.timeout", time.Second*10, diff --git a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go index d998ef3fc8f..fd2d866d5dd 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go @@ -222,6 +222,7 @@ type Callback struct { // Types that are valid to be assigned to Variant: // // *Callback_Nexus_ + // *Callback_Worker_ Variant isCallback_Variant `protobuf_oneof:"variant"` Links []*v11.Link `protobuf:"bytes,100,rep,name=links,proto3" json:"links,omitempty"` unknownFields protoimpl.UnknownFields @@ -274,6 +275,15 @@ func (x *Callback) GetNexus() *Callback_Nexus { return nil } +func (x *Callback) GetWorker() *Callback_Worker { + if x != nil { + if x, ok := x.Variant.(*Callback_Worker_); ok { + return x.Worker + } + } + return nil +} + func (x *Callback) GetLinks() []*v11.Link { if x != nil { return x.Links @@ -289,8 +299,14 @@ type Callback_Nexus_ struct { Nexus *Callback_Nexus `protobuf:"bytes,2,opt,name=nexus,proto3,oneof"` } +type Callback_Worker_ struct { + Worker *Callback_Worker `protobuf:"bytes,4,opt,name=worker,proto3,oneof"` +} + func (*Callback_Nexus_) isCallback_Variant() {} +func (*Callback_Worker_) isCallback_Variant() {} + // Trigger for when the workflow is closed. type CallbackState_WorkflowClosed struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -385,6 +401,79 @@ func (x *Callback_Nexus) GetHeader() map[string]string { return nil } +// Forked from temporal.api.common.v1.Callback.Worker in the api repo, with abbreviated comments. +type Callback_Worker struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Nexus task queue the Temporal worker is listening on. + TaskQueueName string `protobuf:"bytes,1,opt,name=task_queue_name,json=taskQueueName,proto3" json:"task_queue_name,omitempty"` + // Target Nexus service. + Service string `protobuf:"bytes,2,opt,name=service,proto3" json:"service,omitempty"` + // Target operation. + Operation string `protobuf:"bytes,3,opt,name=operation,proto3" json:"operation,omitempty"` + // Arbitrary user-supplied data from the source operation's callsite. + SourceContext *v11.Payload `protobuf:"bytes,4,opt,name=source_context,json=sourceContext,proto3" json:"source_context,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Callback_Worker) Reset() { + *x = Callback_Worker{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Callback_Worker) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Callback_Worker) ProtoMessage() {} + +func (x *Callback_Worker) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Callback_Worker.ProtoReflect.Descriptor instead. +func (*Callback_Worker) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDescGZIP(), []int{1, 1} +} + +func (x *Callback_Worker) GetTaskQueueName() string { + if x != nil { + return x.TaskQueueName + } + return "" +} + +func (x *Callback_Worker) GetService() string { + if x != nil { + return x.Service + } + return "" +} + +func (x *Callback_Worker) GetOperation() string { + if x != nil { + return x.Operation + } + return "" +} + +func (x *Callback_Worker) GetSourceContext() *v11.Payload { + if x != nil { + return x.SourceContext + } + return nil +} + var File_temporal_server_chasm_lib_callback_proto_v1_message_proto protoreflect.FileDescriptor const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = "" + @@ -400,16 +489,22 @@ const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = " "\x1anext_attempt_schedule_time\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\x17nextAttemptScheduleTime\x12\x1d\n" + "\n" + "request_id\x18\t \x01(\tR\trequestId\x1a\x10\n" + - "\x0eWorkflowClosed\"\xde\x02\n" + + "\x0eWorkflowClosed\"\xea\x04\n" + "\bCallback\x12T\n" + - "\x05nexus\x18\x02 \x01(\v2<.temporal.server.chasm.lib.callbacks.proto.v1.Callback.NexusH\x00R\x05nexus\x122\n" + + "\x05nexus\x18\x02 \x01(\v2<.temporal.server.chasm.lib.callbacks.proto.v1.Callback.NexusH\x00R\x05nexus\x12W\n" + + "\x06worker\x18\x04 \x01(\v2=.temporal.server.chasm.lib.callbacks.proto.v1.Callback.WorkerH\x00R\x06worker\x122\n" + "\x05links\x18d \x03(\v2\x1c.temporal.api.common.v1.LinkR\x05links\x1a\xb6\x01\n" + "\x05Nexus\x12\x10\n" + "\x03url\x18\x01 \x01(\tR\x03url\x12`\n" + "\x06header\x18\x02 \x03(\v2H.temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntryR\x06header\x1a9\n" + "\vHeaderEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\t\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1a\xb0\x01\n" + + "\x06Worker\x12&\n" + + "\x0ftask_queue_name\x18\x01 \x01(\tR\rtaskQueueName\x12\x18\n" + + "\aservice\x18\x02 \x01(\tR\aservice\x12\x1c\n" + + "\toperation\x18\x03 \x01(\tR\toperation\x12F\n" + + "\x0esource_context\x18\x04 \x01(\v2\x1f.temporal.api.common.v1.PayloadR\rsourceContextB\t\n" + "\avariantJ\x04\b\x01\x10\x02*\xc9\x01\n" + "\x0eCallbackStatus\x12\x1f\n" + "\x1bCALLBACK_STATUS_UNSPECIFIED\x10\x00\x12\x1b\n" + @@ -432,33 +527,37 @@ func file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDescGZIP( } var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_goTypes = []any{ (CallbackStatus)(0), // 0: temporal.server.chasm.lib.callbacks.proto.v1.CallbackStatus (*CallbackState)(nil), // 1: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState (*Callback)(nil), // 2: temporal.server.chasm.lib.callbacks.proto.v1.Callback (*CallbackState_WorkflowClosed)(nil), // 3: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.WorkflowClosed (*Callback_Nexus)(nil), // 4: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus - nil, // 5: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry - (*timestamppb.Timestamp)(nil), // 6: google.protobuf.Timestamp - (*v1.Failure)(nil), // 7: temporal.api.failure.v1.Failure - (*v11.Link)(nil), // 8: temporal.api.common.v1.Link + (*Callback_Worker)(nil), // 5: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Worker + nil, // 6: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry + (*timestamppb.Timestamp)(nil), // 7: google.protobuf.Timestamp + (*v1.Failure)(nil), // 8: temporal.api.failure.v1.Failure + (*v11.Link)(nil), // 9: temporal.api.common.v1.Link + (*v11.Payload)(nil), // 10: temporal.api.common.v1.Payload } var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_depIdxs = []int32{ - 2, // 0: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.callback:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback - 6, // 1: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.registration_time:type_name -> google.protobuf.Timestamp - 0, // 2: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.status:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.CallbackStatus - 6, // 3: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 7, // 4: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure - 6, // 5: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp - 4, // 6: temporal.server.chasm.lib.callbacks.proto.v1.Callback.nexus:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus - 8, // 7: temporal.server.chasm.lib.callbacks.proto.v1.Callback.links:type_name -> temporal.api.common.v1.Link - 5, // 8: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.header:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry - 9, // [9:9] is the sub-list for method output_type - 9, // [9:9] is the sub-list for method input_type - 9, // [9:9] is the sub-list for extension type_name - 9, // [9:9] is the sub-list for extension extendee - 0, // [0:9] is the sub-list for field type_name + 2, // 0: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.callback:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback + 7, // 1: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.registration_time:type_name -> google.protobuf.Timestamp + 0, // 2: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.status:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.CallbackStatus + 7, // 3: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 8, // 4: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure + 7, // 5: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp + 4, // 6: temporal.server.chasm.lib.callbacks.proto.v1.Callback.nexus:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus + 5, // 7: temporal.server.chasm.lib.callbacks.proto.v1.Callback.worker:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Worker + 9, // 8: temporal.server.chasm.lib.callbacks.proto.v1.Callback.links:type_name -> temporal.api.common.v1.Link + 6, // 9: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.header:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry + 10, // 10: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Worker.source_context:type_name -> temporal.api.common.v1.Payload + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name } func init() { file_temporal_server_chasm_lib_callback_proto_v1_message_proto_init() } @@ -468,6 +567,7 @@ func file_temporal_server_chasm_lib_callback_proto_v1_message_proto_init() { } file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes[1].OneofWrappers = []any{ (*Callback_Nexus_)(nil), + (*Callback_Worker_)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -475,7 +575,7 @@ func file_temporal_server_chasm_lib_callback_proto_v1_message_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc), len(file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc)), NumEnums: 1, - NumMessages: 5, + NumMessages: 6, NumExtensions: 0, NumServices: 0, }, diff --git a/chasm/lib/callback/invocable_internal.go b/chasm/lib/callback/invocable_internal.go index b0249324475..b6698a7383c 100644 --- a/chasm/lib/callback/invocable_internal.go +++ b/chasm/lib/callback/invocable_internal.go @@ -12,13 +12,12 @@ import ( tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -93,7 +92,7 @@ func (c invocableInternal) Invoke( _, err = h.historyClient.CompleteNexusOperationChasm(ctx, request) if err != nil { msg := logInternalError(h.logger, "failed to complete Nexus operation", err) - if isRetryableRPCResponse(err) { + if common.IsRetryableRPCError(err) { return invocationResultRetry{err: msg} } return invocationResultFail{msg} @@ -102,33 +101,6 @@ func (c invocableInternal) Invoke( return invocationResultOK{} } -func isRetryableRPCResponse(err error) bool { - var st *status.Status - stGetter, ok := err.(interface{ Status() *status.Status }) - if ok { - st = stGetter.Status() - } else { - st, ok = status.FromError(err) - if !ok { - // Not a gRPC induced error - return false - } - } - // nolint:exhaustive - switch st.Code() { - case codes.Canceled, - codes.Unknown, - codes.Unavailable, - codes.DeadlineExceeded, - codes.ResourceExhausted, - codes.Aborted, - codes.Internal: - return true - default: - return false - } -} - func (c invocableInternal) getHistoryRequest( refBytes []byte, requestID string, @@ -158,17 +130,10 @@ func (c invocableInternal) getHistoryRequest( Completion: completion, } } else { - failure, err := nexusrpc.DefaultFailureConverter().ErrorToFailure(c.completion.Error) - if err != nil { - return nil, fmt.Errorf("failed to convert error to failure: %w", err) - } - // Unwrap the operation error, the handler on the other side is expecting to receive the underlying cause. - if failure.Cause != nil { - failure = *failure.Cause - } - apiFailure, err := commonnexus.NexusFailureToTemporalFailure(failure) + // Convert the nexus.OperationError into a failurepb.Failure. + apiFailure, err := commonnexus.OperationErrorToTemporalFailure(c.completion.Error) if err != nil { - return nil, fmt.Errorf("failed to convert failure type: %w", err) + return nil, err } req = &historyservice.CompleteNexusOperationChasmRequest{ diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index 41a02a55ece..3e1afb29f80 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -11,7 +11,6 @@ import ( callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" - "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" @@ -70,11 +69,8 @@ func (n invocableOutbound) Invoke( n.completion.Header = n.callback.Header err := client.CompleteOperation(ctx, n.callback.Url, n.completion) - namespaceTag := metrics.NamespaceTag(ns.Name().String()) - destTag := metrics.DestinationTag(taskAttr.Destination) - outcomeTag := metrics.OutcomeTag(outcomeTag(ctx, err)) - h.metricsHandler.Counter(RequestCounter.Name()).Record(1, namespaceTag, destTag, outcomeTag) - h.metricsHandler.Timer(RequestLatencyHistogram.Name()).Record(time.Since(startTime), namespaceTag, destTag, outcomeTag) + outcomeTag := outcomeTag(ctx, err) + h.emitMetrics(startTime, ns, taskAttr.Destination, outcomeTag) if err != nil { retryable := isRetryableCallError(err) diff --git a/chasm/lib/callback/invocable_worker.go b/chasm/lib/callback/invocable_worker.go new file mode 100644 index 00000000000..9c09a907ea0 --- /dev/null +++ b/chasm/lib/callback/invocable_worker.go @@ -0,0 +1,391 @@ +package callback + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/nexus-rpc/sdk-go/nexus" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + nexuspb "go.temporal.io/api/nexus/v1" + notificationpb "go.temporal.io/api/notificationservice/v1" + "go.temporal.io/api/serviceerror" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/chasm" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/namespace" + commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexusrpc" + "go.temporal.io/server/common/payload" + queueserrors "go.temporal.io/server/service/history/queues/errors" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// invocableWorker is an invocable that delivers a completion to a Temporal worker by dispatching a Nexus +// StartOperation task to the worker's task queue via MatchingService.DispatchNexusTask. +// +// Unlike invocableOutbound, which POSTs the completion to an arbitrary address, worker callbacks target a +// Nexus service registered on a worker polling within the source operation's own namespace. This is faster +// and more efficient than round tripping through the frontend's Nexus HTTP endpoint. +type invocableWorker struct { + callback *callbackspb.Callback_Worker + completion nexusrpc.CompleteOperationOptions + startTime time.Time + // requestID is sent as the Nexus request ID so that a redelivery of this callback is idempotent from + // the handler's perspective. + requestID string + attempt int32 +} + +func (n invocableWorker) WrapError(result invocationResult, err error) error { + // A DestinationDownError counts against the outbound queue's circuit breaker for this task + // queue, which holds back every callback targeting it — not just this one. So only failures that + // say something about the task queue itself are surfaced that way; a handler that is up and + // answering is the registering caller's problem, and the callback's own backoff already handles + // it. + if retry, ok := result.(invocationResultRetry); ok && isDestinationDown(retry.err) { + return queueserrors.NewDestinationDownError(retry.err.Error(), err) + } + return err +} + +// isDestinationDown reports whether a retryable delivery failure is a property of the task queue +// rather than of the handler polling it. +func isDestinationDown(err error) bool { + handlerErr, ok := errors.AsType[*nexus.HandlerError](err) + if !ok { + // Nothing a worker produced, so the RPC to matching itself failed. + return true + } + // Matching gave up waiting for a poller: nothing is serving this task queue. Every other + // retryable handler error is an answer from a worker that did receive the delivery. + return handlerErr.Type == nexus.HandlerErrorTypeUpstreamTimeout +} + +func (n invocableWorker) Invoke( + ctx context.Context, + ns *namespace.Namespace, + h *invocationTaskHandler, + task *callbackspb.InvocationTask, + taskAttr chasm.TaskAttributes, +) invocationResult { + logger := log.With(h.logger, + tag.WorkflowNamespace(ns.Name().String()), + tag.Operation("DispatchWorkerCallback"), + tag.NewStringTag("task-queue", n.callback.GetTaskQueueName()), + tag.Attempt(n.attempt), + ) + + result, outcome := n.dispatch(ctx, logger, h, ns, n.startTime) + h.emitMetrics(n.startTime, ns, taskAttr.Destination, outcome) + + return result +} + +// dispatch hands the completion to matching and returns the invocation result along with the metrics +// outcome tag to be recorded. +func (n invocableWorker) dispatch( + ctx context.Context, + logger log.Logger, + h *invocationTaskHandler, + ns *namespace.Namespace, + scheduledTime time.Time, +) (invocationResult, string) { + request, err := n.buildDispatchRequest(ns, scheduledTime) + if err != nil { + // No attempt can make this callback dispatchable, so fail it permanently. + logger.Error("Failed to build worker callback request", tag.Error(err)) + return invocationResultFail{err}, "invalid-request" + } + + resp, rpcErr := h.matchingClient.DispatchNexusTask(ctx, request) + return n.classifyDispatchResult(logger, resp, rpcErr) +} + +func (n invocableWorker) buildDispatchRequest( + ns *namespace.Namespace, + scheduledTime time.Time, +) (*matchingservice.DispatchNexusTaskRequest, error) { + taskQueueName := n.callback.GetTaskQueueName() + if taskQueueName == "" { + return nil, errors.New("worker callback is missing a task queue name") + } + + onComplete, err := n.buildOnCompleteRequest() + if err != nil { + return nil, err + } + // The handler is a lang-SDK Nexus operation, so encode the input with the standard Temporal payload + // format (json/protobuf) that its data converter decodes back into an OnCompleteRequest. + // + // TODO(chrsmith): This needs to be tagged in such a way that any client-side encryption will NOT + // attempt to decode the payload. (Because it was constructed by the Temporal server, and not the + // client.) This will be addressed in a follow-up PR. + input, err := payload.Encode(onComplete) + if err != nil { + return nil, fmt.Errorf("failed to encode worker callback input: %w", err) + } + // The size of the input is deliberately not checked here. The completion it carries already passed + // BlobSizeLimitError where it entered the server, and matching's gRPC limit is orders of magnitude + // above that, so a second check could only reject a completion that is legal everywhere else. + + req := &matchingservice.DispatchNexusTaskRequest{ + NamespaceId: ns.ID().String(), + // The delivery lands on whatever version the task queue currently routes to by default: + // DispatchNexusTaskRequest carries no versioning directive, so matching decides, and it + // applies the task queue's assignment rules to every Nexus task alike. + // + // There is no way to pin a callback to the version that registered it, which is what a + // pinned workflow gets for its own activities. Wiring that through would mean carrying the + // version on the callback and adding a directive to this request; until then, a handler + // receiving completions has to stay compatible across the versions it is rolled through. + TaskQueue: &taskqueuepb.TaskQueue{ + Name: taskQueueName, + Kind: enumspb.TASK_QUEUE_KIND_NORMAL, + }, + Request: &nexuspb.Request{ + ScheduledTime: timestamppb.New(scheduledTime), + Variant: &nexuspb.Request_StartOperation{ + StartOperation: &nexuspb.StartOperationRequest{ + Service: n.callback.GetService(), + Operation: n.callback.GetOperation(), + RequestId: n.requestID, + Payload: input, + // TODO(chrsmith): These links will be wrong. Backlinks to the source of the Nexus completion should be + // to the *callback attached* to the completion's source. Not the completion directly. + // e.g. a Link_Callback to "SANO xxx callback yyy", and not "SANO xxx". + Links: commonnexus.ConvertLinksToProto(n.completion.Links), + }, + }, + Capabilities: &nexuspb.Request_Capabilities{ + TemporalFailureResponses: true, + }, + }, + } + return req, nil +} + +// buildOnCompleteRequest builds the input delivered to the worker's completion handler from the source +// operation's outcome and the context the callback was registered with. +func (n invocableWorker) buildOnCompleteRequest() (*notificationpb.OnCompleteRequest, error) { + req := ¬ificationpb.OnCompleteRequest{ + SourceContext: common.CloneProto(n.callback.GetSourceContext()), + } + + if n.completion.Error != nil { + failure, err := commonnexus.OperationErrorToTemporalFailure(n.completion.Error) + if err != nil { + return nil, err + } + req.Result = ¬ificationpb.OnCompleteRequest_Failure{Failure: failure} + return req, nil + } + + var result *commonpb.Payload + switch typed := n.completion.Result.(type) { + case nil: + // No payload present. + case *commonpb.Payload: + result = typed + default: + return nil, fmt.Errorf("invalid result, expected a payload, got: %T", n.completion.Result) + } + + // A successful operation may legitimately have no result. The success variant always carries a + // payload on the wire, and a payload with no encoding fails the handler's data converter, so + // send the same binary/null representation of "no value" that the Nexus HTTP path produces. + if result == nil { + var err error + if result, err = payload.Encode(nil); err != nil { + return nil, fmt.Errorf("failed to encode empty worker callback result: %w", err) + } + } + + req.Result = ¬ificationpb.OnCompleteRequest_Success{Success: result} + return req, nil +} + +// isRequestRejection reports whether err is a callee rejecting the request as malformed or too +// large, rather than reporting a problem of its own. Only the former is safe to surface to the +// caller: a delivery request is built entirely out of the callback the caller registered and the +// completion of the execution it hangs off, so a rejection tells them what to fix and leaks nothing +// about the server. Everything else goes through logInternalError. +func isRequestRejection(err error) bool { + st, ok := common.GetRPCStatus(err) + if !ok { + return false + } + return st.Code() == codes.InvalidArgument || isOversizedRequest(err) +} + +// isOversizedRequest reports whether err is a gRPC message-size rejection, as opposed to one of the +// throttles that share its status code. A throttle carries a Temporal cause and clears on its own, +// so it stays retryable; a size rejection is a property of the bytes we sent and every retry sends +// the same bytes, so it must not. +func isOversizedRequest(err error) bool { + resourceExhausted, ok := errors.AsType[*serviceerror.ResourceExhausted](err) + if !ok { + st, hasStatus := common.GetRPCStatus(err) + // An error raised by the gRPC client itself, before any Temporal interceptor could convert + // it, is a size rejection on the send side. + return hasStatus && st.Code() == codes.ResourceExhausted + } + return resourceExhausted.Cause == enumspb.RESOURCE_EXHAUSTED_CAUSE_UNSPECIFIED +} + +// classifyDispatchResult maps the result of the dispatch RPC onto an invocation result and the "outcome" tag to emit in metrics. +func (n invocableWorker) classifyDispatchResult( + logger log.Logger, + resp *matchingservice.DispatchNexusTaskResponse, + rpcErr error, +) (invocationResult, string) { + if rpcErr != nil { + // The RPC to matching itself failed, e.g. matching is unavailable or rejected the request. + // A request matching will not accept for its size is the one exception to the status code's + // usual retryability: the bytes are fixed, so retrying only holds this task queue's circuit + // breaker open until the callback is abandoned. + retryable := common.IsRetryableRPCError(rpcErr) && !isOversizedRequest(rpcErr) + logger = log.With(logger, tag.Bool("retryable", retryable)) + + // A rejection describes the request we sent, which is built entirely out of what the caller + // registered, so it is theirs to fix and safe to surface. Everything else describes the state + // of the server and is blinded behind a reference ID. + var userFacingErr error + if isRequestRejection(rpcErr) { + logger.Error("Worker callback dispatch rejected", tag.Error(rpcErr)) + userFacingErr = rpcErr + } else { + userFacingErr = logInternalError(logger, "Worker callback dispatch failed", rpcErr) + } + + if retryable { + return invocationResultRetry{userFacingErr}, "internal-rpc-error" + } + return invocationResultFail{userFacingErr}, "internal-rpc-error" + } + + // There wasn't an RPC error, but any application-level (e.g. the end Handler) errors would be + // part of the response. + outcome, recognized := dispatchOutcomeTag(resp) + + // Note that an async response counts as delivered: the handler accepted the completion and started + // an operation to process it. The callback does not wait for that operation to finish. + err := commonnexus.MatchingDispatchResponseToError(resp) + if err == nil { + return invocationResultOK{}, outcome + } + + if !recognized { + // A response this server cannot interpret, e.g. an empty outcome or a variant added by a newer + // matching. There is nothing to act on and no attempt would produce an outcome we understand + // any better, so fail permanently rather than retry forever and hold the destination's circuit + // breaker open. Note that MatchingDispatchResponseToError reports this as a retryable internal + // handler error, so the check has to come before the retryability check below. + logger.Error("Worker callback received an unrecognized dispatch response", tag.Error(err)) + return invocationResultFail{err}, outcome + } + + if startOperationFailed(resp) { + // The worker received the completion but its operation failed. That outcome is the handler's + // answer, not a delivery problem, so the callback fails permanently instead of retrying. + logger.Error("Worker callback operation failed", tag.Error(err)) + return invocationResultFail{err}, outcome + } + + // Everything else is a delivery-level error: no worker polling the task queue (an upstream timeout) + // or a handler error returned by the worker. + // + // Only a handler error says whether another attempt is worthwhile. Anything else is a failure the + // worker chose to report, e.g. an application error sent via RespondNexusTaskFailed, and repeating + // the delivery would get the same answer, so the callback fails permanently. + handlerErr, ok := errors.AsType[*nexus.HandlerError](err) + retryable := ok && handlerErr.Retryable() + logger.Error("Worker callback request failed", tag.Error(err), tag.Bool("retryable", retryable)) + if retryable { + return invocationResultRetry{err}, outcome + } + return invocationResultFail{err}, outcome +} + +// dispatchOutcomeTag names a dispatch outcome for metrics. Values are hyphenated to match the ones +// invocableOutbound records on the same metric. +// +// The second return value reports whether the outcome is one this server knows how to interpret; see +// classifyDispatchResult for why that matters. +func dispatchOutcomeTag(resp *matchingservice.DispatchNexusTaskResponse) (string, bool) { + //revive:disable:enforce-switch-style // default would just return an error. + switch t := resp.GetOutcome().(type) { + case *matchingservice.DispatchNexusTaskResponse_Failure: + handlerFailure := t.Failure.GetNexusHandlerFailureInfo() + if handlerFailure == nil { + // The worker failed the task with something other than a handler error. + return "worker-failure", true + } + return "handler-error:" + handlerErrorTypeTag(handlerFailure.GetType()), true + case *matchingservice.DispatchNexusTaskResponse_HandlerError: //nolint:staticcheck // Deprecated, still sent by older workers. + //nolint:staticcheck // Deprecated field on a deprecated variant. + return "handler-error:" + handlerErrorTypeTag(t.HandlerError.GetErrorType()), true + case *matchingservice.DispatchNexusTaskResponse_RequestTimeout: + return "handler-timeout", true + case *matchingservice.DispatchNexusTaskResponse_Response: + switch t.Response.GetStartOperation().GetVariant().(type) { + case *nexuspb.StartOperationResponse_SyncSuccess: + return "sync-success", true + case *nexuspb.StartOperationResponse_AsyncSuccess: + return "async-success", true + case *nexuspb.StartOperationResponse_OperationError: //nolint:staticcheck // Deprecated, still sent by older workers. + return "operation-error", true + case *nexuspb.StartOperationResponse_Failure: + return "operation-failure", true + } + } + return "unrecognized-outcome", false +} + +// handlerErrorTypes are the handler error types that may appear in a metric tag. The type a worker +// reports is an arbitrary string it chose when constructing the handler error, so anything outside +// the Nexus spec is collapsed rather than given its own time series. +var handlerErrorTypes = map[string]struct{}{ + string(nexus.HandlerErrorTypeBadRequest): {}, + string(nexus.HandlerErrorTypeUnauthenticated): {}, + string(nexus.HandlerErrorTypeUnauthorized): {}, + string(nexus.HandlerErrorTypeNotFound): {}, + string(nexus.HandlerErrorTypeRequestTimeout): {}, + string(nexus.HandlerErrorTypeConflict): {}, + string(nexus.HandlerErrorTypeResourceExhausted): {}, + string(nexus.HandlerErrorTypeInternal): {}, + string(nexus.HandlerErrorTypeNotImplemented): {}, + string(nexus.HandlerErrorTypeUnavailable): {}, + string(nexus.HandlerErrorTypeUpstreamTimeout): {}, +} + +// handlerErrorTypeTag bounds the cardinality a worker can introduce into the outcome tag. +func handlerErrorTypeTag(errType string) string { + if _, ok := handlerErrorTypes[errType]; ok { + return errType + } + return "UNKNOWN" +} + +// startOperationFailed reports whether the worker handled the task and failed the operation, as opposed to +// failing to handle the task at all. +func startOperationFailed(resp *matchingservice.DispatchNexusTaskResponse) bool { + outcome, ok := resp.GetOutcome().(*matchingservice.DispatchNexusTaskResponse_Response) + if !ok { + return false + } + switch outcome.Response.GetStartOperation().GetVariant().(type) { + case *nexuspb.StartOperationResponse_Failure, + *nexuspb.StartOperationResponse_OperationError: //nolint:staticcheck // Deprecated, still sent by older workers. + return true + default: + return false + } +} diff --git a/chasm/lib/callback/invocable_worker_test.go b/chasm/lib/callback/invocable_worker_test.go new file mode 100644 index 00000000000..3fc0d9de069 --- /dev/null +++ b/chasm/lib/callback/invocable_worker_test.go @@ -0,0 +1,683 @@ +package callback + +import ( + "context" + "net/url" + "testing" + "time" + + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + nexuspb "go.temporal.io/api/nexus/v1" + notificationpb "go.temporal.io/api/notificationservice/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/api/matchingservicemock/v1" + "go.temporal.io/server/chasm" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "go.temporal.io/server/common/backoff" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace" + commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexusrpc" + "go.temporal.io/server/common/payload" + "go.temporal.io/server/common/testing/protorequire" + queueserrors "go.temporal.io/server/service/history/queues/errors" + "go.uber.org/mock/gomock" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + testWorkerTaskQueue = "completions-task-queue" + testWorkerService = "HTTPAdapter" + testWorkerOperation = "DeliverAsWebhook" + // The destination the invocation task is grouped under, mirroring what callbackDestination + // produces for testWorkerTaskQueue. + testWorkerDestination = "worker://completions-task-queue" +) + +func newWorkerCallback(t *testing.T) *Callback { + t.Helper() + + return &Callback{ + CallbackState: &callbackspb.CallbackState{ + RequestId: "request-id", + RegistrationTime: timestamppb.New(time.Now()), + Callback: &callbackspb.Callback{ + Variant: &callbackspb.Callback_Worker_{ + Worker: &callbackspb.Callback_Worker{ + TaskQueueName: testWorkerTaskQueue, + Service: testWorkerService, + Operation: testWorkerOperation, + SourceContext: &commonpb.Payload{Data: []byte("source-context")}, + }, + }, + }, + Status: callbackspb.CALLBACK_STATUS_SCHEDULED, + Attempt: 0, + }, + } +} + +// startOperationResponse builds the response matching returns when a worker handled the Nexus task and +// replied with the given StartOperation response. +func startOperationResponse(start *nexuspb.StartOperationResponse) *matchingservice.DispatchNexusTaskResponse { + return &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ + Response: &nexuspb.Response{ + Variant: &nexuspb.Response_StartOperation{StartOperation: start}, + }, + }, + } +} + +func syncSuccessResponse() *matchingservice.DispatchNexusTaskResponse { + return startOperationResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_SyncSuccess{ + SyncSuccess: &nexuspb.StartOperationResponse_Sync{}, + }, + }) +} + +// handlerFailureResponse builds the response matching returns when a worker fails the Nexus task itself, +// i.e. responds with RespondNexusTaskFailed. +func handlerFailureResponse(errType string) *matchingservice.DispatchNexusTaskResponse { + return &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ + Failure: &failurepb.Failure{ + Message: "handler error (" + errType + "): worker said no", + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: errType, + }, + }, + }, + }, + } +} + +// requireNilPayload asserts the payload is the binary/null representation of "no value", which is +// what a worker's data converter decodes into a nil result. +func requireNilPayload(t *testing.T, p *commonpb.Payload) { + t.Helper() + + expected, err := payload.Encode(nil) + require.NoError(t, err) + protorequire.ProtoEqual(t, expected, p) +} + +// wrappedOperationError builds the completion error a CHASM source component produces when its +// operation fails: the source's own Temporal failure is carried as the error's cause, and the +// enclosing nexus.OperationError is marked as a redundant wrapper so that consumers within the server +// unwrap it. See nexusrpc.MarkAsWrapperError, and Operation.getNexusCompletion in the nexusoperation +// library for the real thing. +func wrappedOperationError( + t *testing.T, + state nexus.OperationState, + message string, + cause *failurepb.Failure, +) *nexus.OperationError { + t.Helper() + + nexusFailure, err := commonnexus.TemporalFailureToNexusFailure(cause) + require.NoError(t, err) + + opErr := &nexus.OperationError{ + State: state, + Message: message, + Cause: &nexus.FailureError{Failure: nexusFailure}, + } + require.NoError(t, nexusrpc.MarkAsWrapperError(nexusrpc.DefaultFailureConverter(), opErr)) + return opErr +} + +// requireTerminalFailure asserts the callback permanently failed, recording a non-retryable failure +// whose message contains want. +func requireTerminalFailure(t *testing.T, cb *Callback, want string) { + t.Helper() + + require.Equal(t, callbackspb.CALLBACK_STATUS_FAILED, cb.Status) + require.Contains(t, cb.LastAttemptFailure.GetMessage(), want) + require.True(t, cb.LastAttemptFailure.GetApplicationFailureInfo().GetNonRetryable()) +} + +// TestExecuteInvocationTaskWorker_Outcomes runs the invocation task end to end against a CHASM tree with a +// mocked matching client, covering how each dispatch outcome maps onto the callback's state. +func TestExecuteInvocationTaskWorker_Outcomes(t *testing.T) { + cases := []struct { + name string + response *matchingservice.DispatchNexusTaskResponse + responseErr error + expectedMetricOutcome string + assertOutcome func(*testing.T, *Callback, error) + }{ + { + name: "sync-success", + response: syncSuccessResponse(), + expectedMetricOutcome: "sync-success", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + require.NoError(t, err) + require.Equal(t, callbackspb.CALLBACK_STATUS_SUCCEEDED, cb.Status) + }, + }, + { + // The handler accepted the completion and started an operation to process it. Delivery is + // done as far as the callback is concerned; it doesn't wait for that operation. + name: "async-success", + response: startOperationResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_AsyncSuccess{ + AsyncSuccess: &nexuspb.StartOperationResponse_Async{OperationToken: "operation-token"}, + }, + }), + expectedMetricOutcome: "async-success", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + require.NoError(t, err) + require.Equal(t, callbackspb.CALLBACK_STATUS_SUCCEEDED, cb.Status) + }, + }, + { + // The worker ran the completion handler and the operation failed. That verdict is + // deterministic, so the callback fails permanently rather than retrying. + name: "operation-failed", + response: startOperationResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_Failure{ + Failure: &failurepb.Failure{ + Message: "handler rejected the completion", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{}, + }, + }, + }, + }), + expectedMetricOutcome: "operation-failure", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + require.NoError(t, err) + requireTerminalFailure(t, cb, "handler rejected the completion") + }, + }, + { + // Older workers report a failed operation with the deprecated OperationError variant. It + // is just as deterministic as the Failure variant, so it must not be retried either. + name: "deprecated-operation-error", + response: startOperationResponse(&nexuspb.StartOperationResponse{ + //nolint:staticcheck // Deprecated, still sent by older workers. + Variant: &nexuspb.StartOperationResponse_OperationError{ + OperationError: &nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateFailed), + Failure: &nexuspb.Failure{Message: "handler rejected the completion"}, + }, + }, + }), + expectedMetricOutcome: "operation-error", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + require.NoError(t, err) + requireTerminalFailure(t, cb, "handler rejected the completion") + }, + }, + { + // Older workers report a handler error with the deprecated HandlerError outcome. Its type + // still decides whether the delivery is worth retrying. + name: "deprecated-non-retryable-handler-error", + response: &matchingservice.DispatchNexusTaskResponse{ + //nolint:staticcheck // Deprecated, still sent by older workers. + Outcome: &matchingservice.DispatchNexusTaskResponse_HandlerError{ + HandlerError: &nexuspb.HandlerError{ + ErrorType: "BAD_REQUEST", + Failure: &nexuspb.Failure{Message: "worker said no"}, + }, + }, + }, + expectedMetricOutcome: "handler-error:BAD_REQUEST", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + require.NoError(t, err) + requireTerminalFailure(t, cb, "BAD_REQUEST") + }, + }, + { + // A worker can fail the task with something other than a handler error, e.g. an + // application error. Only a handler error says whether retrying is worthwhile, so + // anything else is taken as the worker's final answer. + name: "non-handler-task-failure", + response: &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ + Failure: &failurepb.Failure{ + Message: "worker rejected the task", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{Type: "SomeError"}, + }, + }, + }, + }, + expectedMetricOutcome: "worker-failure", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + require.NoError(t, err) + requireTerminalFailure(t, cb, "worker rejected the task") + }, + }, + { + name: "retryable-handler-error", + response: handlerFailureResponse("INTERNAL"), + expectedMetricOutcome: "handler-error:INTERNAL", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + // A worker answered, so this says nothing about the task queue. It must not trip the + // circuit breaker, which would hold back every other callback delivering there. + require.NoError(t, err) + require.Equal(t, callbackspb.CALLBACK_STATUS_BACKING_OFF, cb.Status) + + // The failure is recorded, but not as a terminal one: another attempt is scheduled. + require.Contains(t, cb.LastAttemptFailure.GetMessage(), "INTERNAL") + require.False(t, cb.LastAttemptFailure.GetApplicationFailureInfo().GetNonRetryable()) + require.NotNil(t, cb.NextAttemptScheduleTime) + }, + }, + { + name: "non-retryable-handler-error", + response: handlerFailureResponse("BAD_REQUEST"), + expectedMetricOutcome: "handler-error:BAD_REQUEST", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + require.NoError(t, err) + requireTerminalFailure(t, cb, "BAD_REQUEST") + }, + }, + { + // Nobody is polling the task queue (or the worker died holding the task), so matching gave + // up waiting. A worker may show up later, so keep retrying. + name: "no-poller", + response: &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_RequestTimeout{ + RequestTimeout: &matchingservice.DispatchNexusTaskResponse_Timeout{}, + }, + }, + expectedMetricOutcome: "handler-timeout", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + var destDownErr *queueserrors.DestinationDownError + require.ErrorAs(t, err, &destDownErr) + require.Equal(t, callbackspb.CALLBACK_STATUS_BACKING_OFF, cb.Status) + }, + }, + { + name: "retryable-rpc-error", + responseErr: status.Error(codes.Unavailable, "matching unavailable"), + expectedMetricOutcome: "internal-rpc-error", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + var destDownErr *queueserrors.DestinationDownError + require.ErrorAs(t, err, &destDownErr) + require.Equal(t, callbackspb.CALLBACK_STATUS_BACKING_OFF, cb.Status) + }, + }, + { + // Matching rejecting the request describes the callback the caller registered, so it is + // surfaced verbatim rather than blinded. + name: "rejected-rpc-request", + responseErr: status.Error(codes.InvalidArgument, "malformed task queue name"), + expectedMetricOutcome: "internal-rpc-error", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + require.NoError(t, err) + requireTerminalFailure(t, cb, "malformed task queue name") + }, + }, + { + // The request is too large for matching to receive. Every retry sends the same bytes, so + // retrying could only hold this task queue's circuit breaker open until the callback is + // abandoned. It also describes the caller's own payload, so it is surfaced verbatim. + name: "oversized-rpc-request", + responseErr: status.Error(codes.ResourceExhausted, + "grpc: received message larger than max (5242880 vs. 4194304)"), + expectedMetricOutcome: "internal-rpc-error", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + require.NoError(t, err) + requireTerminalFailure(t, cb, "received message larger than max") + }, + }, + { + // A throttle shares the status code but is a property of the server's state rather than + // of our bytes, so it clears on its own and stays retryable. + name: "throttled-rpc-request", + responseErr: serviceerror.NewResourceExhausted( + enumspb.RESOURCE_EXHAUSTED_CAUSE_RPS_LIMIT, "namespace rps limit exceeded"), + expectedMetricOutcome: "internal-rpc-error", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + var destDownErr *queueserrors.DestinationDownError + require.ErrorAs(t, err, &destDownErr) + require.Equal(t, callbackspb.CALLBACK_STATUS_BACKING_OFF, cb.Status) + }, + }, + { + // Any other RPC failure describes the state of the server, so it is hidden behind a + // reference ID and only the shape is asserted. + name: "non-retryable-rpc-error", + responseErr: status.Error(codes.NotFound, "namespace not found"), + expectedMetricOutcome: "internal-rpc-error", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + require.NoError(t, err) + require.NotContains(t, cb.LastAttemptFailure.GetMessage(), "namespace not found") + requireTerminalFailure(t, cb, "internal error, reference-id:") + }, + }, + { + // A response this server cannot interpret is not actionable and no retry would make it + // so, so the callback fails permanently instead of retrying forever. + name: "unrecognized-outcome", + response: &matchingservice.DispatchNexusTaskResponse{}, + expectedMetricOutcome: "unrecognized-outcome", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + require.NoError(t, err) + require.Equal(t, callbackspb.CALLBACK_STATUS_FAILED, cb.Status) + require.True(t, cb.LastAttemptFailure.GetApplicationFailureInfo().GetNonRetryable()) + }, + }, + { + // A handler error type outside the Nexus spec is collapsed so a worker cannot introduce + // unbounded metric cardinality. + name: "handler-error-with-an-unknown-type", + response: handlerFailureResponse("SOMETHING_MADE_UP"), + expectedMetricOutcome: "handler-error:UNKNOWN", + assertOutcome: func(t *testing.T, cb *Callback, err error) { + // An unrecognized handler error type is retryable per the Nexus spec, so the delivery + // is retried. The type is collapsed only in the metric tag; the recorded failure keeps + // what the worker actually said. + require.NoError(t, err) + require.Equal(t, callbackspb.CALLBACK_STATUS_BACKING_OFF, cb.Status) + require.Contains(t, cb.LastAttemptFailure.GetMessage(), "SOMETHING_MADE_UP") + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + ns := newTestNamespace(t) + + metricsHandler := metrics.NewMockHandler(ctrl) + counter := metrics.NewMockCounterIface(ctrl) + timer := metrics.NewMockTimerIface(ctrl) + metricsHandler.EXPECT().Counter(RequestCounter.Name()).Return(counter) + counter.EXPECT().Record(int64(1), + metrics.NamespaceTag("namespace-name"), + metrics.DestinationTag(testWorkerDestination), + metrics.OutcomeTag(tc.expectedMetricOutcome)) + metricsHandler.EXPECT().Timer(RequestLatencyHistogram.Name()).Return(timer) + timer.EXPECT().Record(gomock.Any(), + metrics.NamespaceTag("namespace-name"), + metrics.DestinationTag(testWorkerDestination), + metrics.OutcomeTag(tc.expectedMetricOutcome)) + + matchingClient := matchingservicemock.NewMockMatchingServiceClient(ctrl) + matchingClient.EXPECT().DispatchNexusTask(gomock.Any(), gomock.Any()). + Return(tc.response, tc.responseErr) + + nsRegistry := namespace.NewMockRegistry(ctrl) + nsRegistry.EXPECT().GetNamespaceByID(gomock.Any()).Return(ns, nil) + + handler := &invocationTaskHandler{ + config: &Config{ + RequestTimeout: dynamicconfig.GetDurationPropertyFnFilteredByDestination(time.Second), + RetryPolicy: func() backoff.RetryPolicy { + return backoff.NewExponentialRetryPolicy(time.Second) + }, + }, + namespaceRegistry: nsRegistry, + metricsHandler: metricsHandler, + logger: log.NewTestLogger(), + matchingClient: matchingClient, + } + + callback := newWorkerCallback(t) + engineCtx, callbackRef := newInvocationTaskTest(t, handler, callback, nexusrpc.CompleteOperationOptions{}) + + executeErr := handler.Execute( + engineCtx, + callbackRef, + chasm.TaskAttributes{Destination: testWorkerDestination}, + &callbackspb.InvocationTask{Attempt: 0}, + ) + + readCallbackState(t, engineCtx, callbackRef, func(_ chasm.Context, c *Callback) { + t.Helper() + tc.assertOutcome(t, c, executeErr) + }) + }) + } +} + +// TestExecuteInvocationTaskWorker_DispatchedRequest covers what the worker actually receives: the task is +// addressed to the callback's task queue, service, and operation, and its input carries the source +// operation's outcome. +func TestExecuteInvocationTaskWorker_DispatchedRequest(t *testing.T) { + sourceURL, err := url.Parse("temporal:///namespaces/ns-name/operations/op-id/runs/run-id") + require.NoError(t, err) + sourceLink := nexus.Link{URL: sourceURL, Type: "temporal.api.common.v1.Link.NexusOperation"} + + for _, tc := range []struct { + name string + completion nexusrpc.CompleteOperationOptions + assertOn func(*testing.T, *notificationpb.OnCompleteRequest) + }{ + { + name: "successful-completion", + completion: nexusrpc.CompleteOperationOptions{ + Result: &commonpb.Payload{Data: []byte("result-data")}, + Links: []nexus.Link{sourceLink}, + }, + assertOn: func(t *testing.T, req *notificationpb.OnCompleteRequest) { + require.Equal(t, []byte("result-data"), req.GetSuccess().GetData()) + require.Nil(t, req.GetFailure()) + }, + }, + { + // A successful operation without a result still reports success, carrying the + // binary/null representation of "no value". + name: "successful-completion-without-a-result", + completion: nexusrpc.CompleteOperationOptions{ + Links: []nexus.Link{sourceLink}, + }, + assertOn: func(t *testing.T, req *notificationpb.OnCompleteRequest) { + require.IsType(t, ¬ificationpb.OnCompleteRequest_Success{}, req.GetResult()) + requireNilPayload(t, req.GetSuccess()) + }, + }, + { + // Completion sources report an absent result as a nil *commonpb.Payload rather than an + // untyped nil, which must be treated the same as no result at all. + name: "successful-completion-with-a-typed-nil-result", + completion: nexusrpc.CompleteOperationOptions{ + Result: (*commonpb.Payload)(nil), + Links: []nexus.Link{sourceLink}, + }, + assertOn: func(t *testing.T, req *notificationpb.OnCompleteRequest) { + require.IsType(t, ¬ificationpb.OnCompleteRequest_Success{}, req.GetResult()) + requireNilPayload(t, req.GetSuccess()) + }, + }, + { + // The completion of a failed source operation is a nexus.OperationError wrapping the + // source's own Temporal failure. Since the OnCompleteRequest carries a Temporal failure + // anyway, that wrapper is redundant, so the source marks it (see + // nexusrpc.MarkAsWrapperError) and the handler unwraps it: the worker is handed the + // failure the source operation actually failed with, not the "operation failed" wrapper. + name: "failed-completion-unwraps-the-operation-error", + completion: nexusrpc.CompleteOperationOptions{ + Error: wrappedOperationError(t, nexus.OperationStateFailed, "operation failed", &failurepb.Failure{ + Message: "widget exploded", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: "WidgetError", + NonRetryable: true, + }, + }, + }), + Links: []nexus.Link{sourceLink}, + }, + assertOn: func(t *testing.T, req *notificationpb.OnCompleteRequest) { + require.Nil(t, req.GetSuccess()) + + // What the worker sees is the source failure verbatim: its message, its application + // failure type, and no enclosing "operation failed" layer. + failure := req.GetFailure() + require.Equal(t, "widget exploded", failure.GetMessage()) + require.Equal(t, "WidgetError", failure.GetApplicationFailureInfo().GetType()) + require.True(t, failure.GetApplicationFailureInfo().GetNonRetryable()) + require.Nil(t, failure.GetCause()) + }, + }, + { + // A canceled source operation is delivered the same way, and unwrapping preserves the + // failure's Temporal type: the worker sees a canceled failure rather than an application + // failure that merely says "operation canceled". + name: "canceled-completion-unwraps-the-operation-error", + completion: nexusrpc.CompleteOperationOptions{ + Error: wrappedOperationError(t, nexus.OperationStateCanceled, "operation canceled", &failurepb.Failure{ + Message: "operation canceled by the caller", + FailureInfo: &failurepb.Failure_CanceledFailureInfo{ + CanceledFailureInfo: &failurepb.CanceledFailureInfo{}, + }, + }), + Links: []nexus.Link{sourceLink}, + }, + assertOn: func(t *testing.T, req *notificationpb.OnCompleteRequest) { + failure := req.GetFailure() + require.Equal(t, "operation canceled by the caller", failure.GetMessage()) + require.NotNil(t, failure.GetCanceledFailureInfo()) + }, + }, + { + // An OperationError that was not marked as a wrapper is delivered as-is, since dropping a + // layer nobody said was redundant would lose the only thing that identifies the outcome as + // a failed operation. This is what an error originating outside the Temporal server looks + // like, e.g. relayed from a third-party Nexus handler. + // + // The worker gets the wrapper as a non-retryable ApplicationFailure of type + // "OperationError", and the handler's own failure hangs off it as the cause. + name: "failed-completion-keeps-an-unmarked-operation-error", + completion: nexusrpc.CompleteOperationOptions{ + Error: &nexus.OperationError{ + State: nexus.OperationStateFailed, + Message: "operation failed", + Cause: &nexus.FailureError{Failure: nexus.Failure{Message: "widget exploded"}}, + }, + Links: []nexus.Link{sourceLink}, + }, + assertOn: func(t *testing.T, req *notificationpb.OnCompleteRequest) { + failure := req.GetFailure() + require.Equal(t, "operation failed", failure.GetMessage()) + require.Equal(t, "OperationError", failure.GetApplicationFailureInfo().GetType()) + require.True(t, failure.GetApplicationFailureInfo().GetNonRetryable()) + require.Equal(t, "widget exploded", failure.GetCause().GetMessage()) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + var dispatched *matchingservice.DispatchNexusTaskRequest + matchingClient := matchingservicemock.NewMockMatchingServiceClient(ctrl) + matchingClient.EXPECT().DispatchNexusTask(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, req *matchingservice.DispatchNexusTaskRequest, _ ...grpc.CallOption) (*matchingservice.DispatchNexusTaskResponse, error) { + dispatched = req + return syncSuccessResponse(), nil + }) + + nsRegistry := namespace.NewMockRegistry(ctrl) + nsRegistry.EXPECT().GetNamespaceByID(gomock.Any()).Return(newTestNamespace(t), nil) + + handler := &invocationTaskHandler{ + config: &Config{ + RequestTimeout: dynamicconfig.GetDurationPropertyFnFilteredByDestination(time.Second), + RetryPolicy: func() backoff.RetryPolicy { + return backoff.NewExponentialRetryPolicy(time.Second) + }, + }, + namespaceRegistry: nsRegistry, + metricsHandler: metrics.NoopMetricsHandler, + logger: log.NewTestLogger(), + matchingClient: matchingClient, + } + + callback := newWorkerCallback(t) + engineCtx, callbackRef := newInvocationTaskTest(t, handler, callback, tc.completion) + dispatchStart := time.Now() + require.NoError(t, handler.Execute( + engineCtx, + callbackRef, + chasm.TaskAttributes{Destination: testWorkerDestination}, + &callbackspb.InvocationTask{Attempt: 0}, + )) + + require.NotNil(t, dispatched) + require.Equal(t, "namespace-id", dispatched.GetNamespaceId()) + // The worker's poller measures task latencies against the scheduled time, so it has to + // reflect when this delivery attempt started. + require.WithinDuration(t, dispatchStart, dispatched.GetRequest().GetScheduledTime().AsTime(), time.Minute) + require.Equal(t, testWorkerTaskQueue, dispatched.GetTaskQueue().GetName()) + require.Equal(t, enumspb.TASK_QUEUE_KIND_NORMAL, dispatched.GetTaskQueue().GetKind()) + + start := dispatched.GetRequest().GetStartOperation() + require.Equal(t, testWorkerService, start.GetService()) + require.Equal(t, testWorkerOperation, start.GetOperation()) + // The callback's request ID doubles as the Nexus request ID, so a redelivery is idempotent + // from the handler's perspective. + require.Equal(t, "request-id", start.GetRequestId()) + // The source operation is identified to the handler by the completion's links. + require.Len(t, start.GetLinks(), 1) + require.Equal(t, sourceLink.URL.String(), start.GetLinks()[0].GetUrl()) + + var onComplete notificationpb.OnCompleteRequest + require.NoError(t, payload.Decode(start.GetPayload(), &onComplete)) + protorequire.ProtoEqual(t, &commonpb.Payload{Data: []byte("source-context")}, onComplete.GetSourceContext()) + tc.assertOn(t, &onComplete) + }) + } +} + +// A Worker callback that cannot be dispatched at all fails permanently, since no further attempt +// would change the outcome. +func TestInvocableWorkerCannotDispatch(t *testing.T) { + for _, tc := range []struct { + name string + callback *callbackspb.Callback_Worker + completion nexusrpc.CompleteOperationOptions + wantMessage string + }{ + { + name: "without a task queue", + callback: &callbackspb.Callback_Worker{Service: testWorkerService}, + wantMessage: "missing a task queue name", + }, + { + name: "with a result that isn't a payload", + callback: &callbackspb.Callback_Worker{TaskQueueName: testWorkerTaskQueue, Service: testWorkerService}, + completion: nexusrpc.CompleteOperationOptions{Result: "not-a-payload"}, + wantMessage: "invalid result, expected a payload", + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + handler := &invocationTaskHandler{ + config: &Config{}, + metricsHandler: metrics.NoopMetricsHandler, + logger: log.NewTestLogger(), + // Dispatch must not be attempted, so the mock is left without expectations. + matchingClient: matchingservicemock.NewMockMatchingServiceClient(ctrl), + } + invocable := invocableWorker{callback: tc.callback, completion: tc.completion} + + result := invocable.Invoke(context.Background(), newTestNamespace(t), handler, nil, chasm.TaskAttributes{}) + + require.IsType(t, invocationResultFail{}, result) + require.ErrorContains(t, result.error(), tc.wantMessage) + }) + } +} diff --git a/chasm/lib/callback/library.go b/chasm/lib/callback/library.go index 5d9b4f0003e..34db2936adf 100644 --- a/chasm/lib/callback/library.go +++ b/chasm/lib/callback/library.go @@ -2,15 +2,42 @@ package callback import ( "go.temporal.io/server/chasm" + "go.uber.org/fx" "google.golang.org/grpc" ) +// InvocationTaskGroup is the outbound queue task group that callback invocation tasks are +// scheduled under. The queue's per-destination rate limiters and circuit breakers are keyed by it. +var InvocationTaskGroup = chasm.FullyQualifiedName(chasm.CallbackLibraryName, "invoke") + +// DestinationBlockedFn reports whether the outbound queue is currently holding back callback +// deliveries to the given destination, i.e. whether its circuit breaker is open. +type DestinationBlockedFn func(namespaceID string, destination string) bool + +type ctxKeyCallbackContextType struct{} + +var ctxKeyCallbackContext = ctxKeyCallbackContextType{} + +// callbackContext holds the dependencies injected into the chasm.Context for use by Callback methods. +type callbackContext struct { + destinationBlocked DestinationBlockedFn +} + +// callbackContextFromChasm extracts the callbackContext from a chasm.Context. +// Panics if the context value is missing, which indicates a library registration bug. +func callbackContextFromChasm(ctx chasm.Context) *callbackContext { + //nolint:revive // unchecked-type-assertion: intentional panic on missing context value + return ctx.Value(ctxKeyCallbackContext).(*callbackContext) +} + type ( Library struct { chasm.UnimplementedLibrary InvocationTaskHandler *invocationTaskHandler BackoffTaskHandler *backoffTaskHandler + + destinationBlocked DestinationBlockedFn } ) @@ -20,13 +47,21 @@ func NewNilLibrary() *Library { return &Library{} } -func newLibrary( - InvocationTaskHandler *invocationTaskHandler, - BackoffTaskHandler *backoffTaskHandler, -) *Library { +type libraryParams struct { + fx.In + + InvocationTaskHandler *invocationTaskHandler + BackoffTaskHandler *backoffTaskHandler + // Only the history service runs the outbound queue, so only it can report whether a + // destination is blocked. Elsewhere callbacks are simply never reported as blocked. + DestinationBlocked DestinationBlockedFn `optional:"true"` +} + +func newLibrary(params libraryParams) *Library { return &Library{ - InvocationTaskHandler: InvocationTaskHandler, - BackoffTaskHandler: BackoffTaskHandler, + InvocationTaskHandler: params.InvocationTaskHandler, + BackoffTaskHandler: params.BackoffTaskHandler, + destinationBlocked: params.DestinationBlocked, } } @@ -35,10 +70,20 @@ func (l *Library) Name() string { } func (l *Library) Components() []*chasm.RegistrableComponent { + destinationBlocked := l.destinationBlocked + if destinationBlocked == nil { + // Processes that don't run the outbound queue never report a destination as blocked. + destinationBlocked = func(string, string) bool { return false } + } return []*chasm.RegistrableComponent{ chasm.NewRegistrableComponent[*Callback]( chasm.CallbackComponentName, chasm.WithDetached(), + chasm.WithContextValues(map[any]any{ + ctxKeyCallbackContext: &callbackContext{ + destinationBlocked: destinationBlocked, + }, + }), ), } } diff --git a/chasm/lib/callback/proto/v1/message.proto b/chasm/lib/callback/proto/v1/message.proto index 057e5c470e0..7d72185e678 100644 --- a/chasm/lib/callback/proto/v1/message.proto +++ b/chasm/lib/callback/proto/v1/message.proto @@ -61,9 +61,22 @@ message Callback { map header = 2; } + // Forked from temporal.api.common.v1.Callback.Worker in the api repo, with abbreviated comments. + message Worker { + // Nexus task queue the Temporal worker is listening on. + string task_queue_name = 1; + // Target Nexus service. + string service = 2; + // Target operation. + string operation = 3; + // Arbitrary user-supplied data from the source operation's callsite. + temporal.api.common.v1.Payload source_context = 4; + } + reserved 1; // For a generic callback mechanism to be added later. oneof variant { Nexus nexus = 2; + Worker worker = 4; } repeated temporal.api.common.v1.Link links = 100; diff --git a/chasm/lib/callback/statemachine.go b/chasm/lib/callback/statemachine.go index 779b9773989..637a677221a 100644 --- a/chasm/lib/callback/statemachine.go +++ b/chasm/lib/callback/statemachine.go @@ -12,6 +12,30 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +// callbackDestination returns the "destination" the callback is targeting. On the outbound queue +// this keys the per-(namespace, destination) rate limits and circuit breaking. It also picks the +// queue: a task with an empty destination goes to the transfer queue instead, see taskCategory +// in chasm/tree.go. +func callbackDestination(cb *callbackspb.Callback) (string, error) { + switch variant := cb.GetVariant().(type) { + case *callbackspb.Callback_Nexus_: + u, err := url.Parse(variant.Nexus.GetUrl()) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %v: %w", cb, err) + } + return u.Scheme + "://" + u.Host, nil + case *callbackspb.Callback_Worker_: + // Use a new "worker" scheme to avoid colliding with any other type of callback variant. + return "worker://" + variant.Worker.GetTaskQueueName(), nil + default: + // Only reachable if a newer server persisted a variant this build doesn't know about. + // Returning an error would fail the whole transaction that completed the execution, + // which is worse. An empty destination is valid: the task lands on the transfer queue, + // which runs CHASM side-effect tasks just fine. + return "", nil + } +} + // EventScheduled is triggered when the callback is meant to be scheduled for the first time - when its Trigger // condition is met. type EventScheduled struct{} @@ -20,11 +44,11 @@ var TransitionScheduled = chasm.NewTransition( []callbackspb.CallbackStatus{callbackspb.CALLBACK_STATUS_STANDBY}, callbackspb.CALLBACK_STATUS_SCHEDULED, func(cb *Callback, ctx chasm.MutableContext, event EventScheduled) error { - u, err := url.Parse(cb.Callback.GetNexus().GetUrl()) + destination, err := callbackDestination(cb.GetCallback()) if err != nil { - return fmt.Errorf("failed to parse URL: %v: %w", cb.Callback, err) + return err } - ctx.AddTask(cb, chasm.TaskAttributes{Destination: u.Scheme + "://" + u.Host}, &callbackspb.InvocationTask{}) + ctx.AddTask(cb, chasm.TaskAttributes{Destination: destination}, &callbackspb.InvocationTask{}) return nil }, ) @@ -37,13 +61,13 @@ var TransitionRescheduled = chasm.NewTransition( callbackspb.CALLBACK_STATUS_SCHEDULED, func(cb *Callback, ctx chasm.MutableContext, event EventRescheduled) error { cb.NextAttemptScheduleTime = nil - u, err := url.Parse(cb.Callback.GetNexus().Url) + destination, err := callbackDestination(cb.GetCallback()) if err != nil { - return fmt.Errorf("failed to parse URL: %v: %w", cb.Callback, err) + return err } ctx.AddTask( cb, - chasm.TaskAttributes{Destination: u.Scheme + "://" + u.Host}, + chasm.TaskAttributes{Destination: destination}, &callbackspb.InvocationTask{Attempt: cb.Attempt}, ) return nil diff --git a/chasm/lib/callback/statemachine_test.go b/chasm/lib/callback/statemachine_test.go index dbefaf5d96c..3aeca7c977d 100644 --- a/chasm/lib/callback/statemachine_test.go +++ b/chasm/lib/callback/statemachine_test.go @@ -12,6 +12,78 @@ import ( "google.golang.org/protobuf/proto" ) +func TestCallbackDestination(t *testing.T) { + for _, tc := range []struct { + name string + cb *callbackspb.Callback + want string + wantErr string + }{ + { + name: "nexus", + cb: &callbackspb.Callback{Variant: &callbackspb.Callback_Nexus_{ + Nexus: &callbackspb.Callback_Nexus{Url: "http://address:666/path/to/callback?query=string"}, + }}, + want: "http://address:666", + }, + { + name: "nexus with invalid url", + cb: &callbackspb.Callback{Variant: &callbackspb.Callback_Nexus_{ + Nexus: &callbackspb.Callback_Nexus{Url: "http://invalid url/path"}, + }}, + wantErr: "failed to parse URL:", + }, + { + name: "worker", + cb: &callbackspb.Callback{Variant: &callbackspb.Callback_Worker_{ + Worker: &callbackspb.Callback_Worker{TaskQueueName: "completions-task-queue"}, + }}, + want: "worker://completions-task-queue", + }, + { + // A variant this server does not recognize, e.g. one persisted by a newer server. + name: "unrecognized variant", + cb: &callbackspb.Callback{}, + want: "", + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := callbackDestination(tc.cb) + if tc.wantErr == "" { + require.NoError(t, err) + require.Equal(t, tc.want, got) + } else { + require.Error(t, err) + require.ErrorContains(t, err, tc.wantErr) + } + }) + } +} + +// Scheduling a Worker callback succeeds and routes its invocation task to the target task queue. +// Invoking it is not implemented yet, so the invocation task itself fails; scheduling must not, +// since it runs as part of the execution's close transaction. +func TestTransitionScheduled_Worker(t *testing.T) { + cb := &Callback{ + CallbackState: &callbackspb.CallbackState{ + Callback: &callbackspb.Callback{ + Variant: &callbackspb.Callback_Worker_{ + Worker: &callbackspb.Callback_Worker{TaskQueueName: "completions-task-queue"}, + }, + }, + }, + } + cb.SetStateMachineState(callbackspb.CALLBACK_STATUS_STANDBY) + + mctx := &chasm.MockMutableContext{} + require.NoError(t, TransitionScheduled.Apply(cb, mctx, EventScheduled{})) + + require.Equal(t, callbackspb.CALLBACK_STATUS_SCHEDULED, cb.StateMachineState()) + require.Len(t, mctx.Tasks, 1) + require.IsType(t, &callbackspb.InvocationTask{}, mctx.Tasks[0].Payload) + require.Equal(t, "worker://completions-task-queue", mctx.Tasks[0].Attributes.Destination) +} + func TestValidTransitions(t *testing.T) { // Setup currentTime := time.Now().UTC() diff --git a/chasm/lib/callback/tasks.go b/chasm/lib/callback/tasks.go index c54e065e073..d69ebe8f321 100644 --- a/chasm/lib/callback/tasks.go +++ b/chasm/lib/callback/tasks.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "time" "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" @@ -79,6 +80,7 @@ type invocationTaskHandlerOptions struct { HTTPCallerProvider HTTPCallerProvider HTTPTraceProvider commonnexus.HTTPClientTraceProvider HistoryClient resource.HistoryClient + MatchingClient resource.MatchingClient } type invocationTaskHandler struct { @@ -90,6 +92,7 @@ type invocationTaskHandler struct { httpCallerProvider HTTPCallerProvider httpTraceProvider commonnexus.HTTPClientTraceProvider historyClient resource.HistoryClient + matchingClient resource.MatchingClient } func newInvocationTaskHandler(opts invocationTaskHandlerOptions) *invocationTaskHandler { @@ -101,6 +104,7 @@ func newInvocationTaskHandler(opts invocationTaskHandlerOptions) *invocationTask httpCallerProvider: opts.HTTPCallerProvider, httpTraceProvider: opts.HTTPTraceProvider, historyClient: opts.HistoryClient, + matchingClient: opts.MatchingClient, } } @@ -148,6 +152,21 @@ func (h *invocationTaskHandler) Execute( return invokable.WrapError(result, saveErr) } +// emitMetrics emits the Count and Latency metrics for the invocation. +func (h *invocationTaskHandler) emitMetrics( + startTime time.Time, + ns *namespace.Namespace, + destination string, + outcome string) { + namespaceTag := metrics.NamespaceTag(ns.Name().String()) + destinationTag := metrics.DestinationTag(destination) + outcomeTag := metrics.OutcomeTag(outcome) + tags := []metrics.Tag{namespaceTag, destinationTag, outcomeTag} + + h.metricsHandler.Counter(RequestCounter.Name()).Record(1, tags...) + h.metricsHandler.Timer(RequestLatencyHistogram.Name()).Record(time.Since(startTime), tags...) +} + type backoffTaskHandler struct { chasm.PureTaskHandlerBase } diff --git a/chasm/lib/callback/tasks_test.go b/chasm/lib/callback/tasks_test.go index 71e5c881a64..ac3840400a8 100644 --- a/chasm/lib/callback/tasks_test.go +++ b/chasm/lib/callback/tasks_test.go @@ -80,6 +80,94 @@ func (l *mockNexusCompletionGetterLibrary) Components() []*chasm.RegistrableComp } } +func newTestNamespace(t *testing.T) *namespace.Namespace { + t.Helper() + + factory := namespace.NewDefaultReplicationResolverFactory() + detail := &persistencespb.NamespaceDetail{ + Info: &persistencespb.NamespaceInfo{ + Id: "namespace-id", + Name: "namespace-name", + }, + Config: &persistencespb.NamespaceConfig{}, + } + ns, err := namespace.FromPersistentState(detail, factory(detail)) + require.NoError(t, err) + return ns +} + +// newInvocationTaskTest builds a CHASM tree holding cb underneath a completion source that returns the +// given completion, and returns an engine context plus a ref to the callback to invoke task handlers with. +func newInvocationTaskTest( + t *testing.T, + handler *invocationTaskHandler, + cb *Callback, + completion nexusrpc.CompleteOperationOptions, +) (context.Context, chasm.ComponentRef) { + t.Helper() + + chasmRegistry := chasm.NewRegistry(log.NewTestLogger()) + require.NoError(t, chasmRegistry.Register(&Library{InvocationTaskHandler: handler})) + require.NoError(t, chasmRegistry.Register(&mockNexusCompletionGetterLibrary{})) + + executionKey := chasm.ExecutionKey{ + NamespaceID: "namespace-id", + BusinessID: "workflow-id", + RunID: "run-id", + } + engineCtx := chasm.NewEngineContext(context.Background(), chasmtest.NewEngine(t, chasmRegistry)) + _, err := chasm.StartExecution( + engineCtx, + executionKey, + func(ctx chasm.MutableContext, _ struct{}) (*mockNexusCompletionGetterComponent, error) { + return &mockNexusCompletionGetterComponent{ + completion: completion, + Callback: chasm.NewComponentField(ctx, cb), + }, nil + }, + struct{}{}, + ) + require.NoError(t, err) + + rootRef := chasm.NewComponentRef[*mockNexusCompletionGetterComponent](executionKey) + callbackRef, err := chasm.ReadComponent( + engineCtx, + rootRef, + func(_ *mockNexusCompletionGetterComponent, chasmCtx chasm.Context, _ struct{}) (chasm.ComponentRef, error) { + serialized, err := chasmCtx.Ref(cb) + if err != nil { + return chasm.ComponentRef{}, err + } + return chasm.DeserializeComponentRef(serialized) + }, + struct{}{}, + ) + require.NoError(t, err) + return engineCtx, callbackRef +} + +// readCallbackState runs assert against the persisted callback state, so that assertions see what the task +// handler committed rather than the in-memory component it was handed. +func readCallbackState( + t *testing.T, + engineCtx context.Context, + ref chasm.ComponentRef, + assert func(chasm.Context, *Callback), +) { + t.Helper() + + _, err := chasm.ReadComponent( + engineCtx, + ref, + func(c *Callback, chasmCtx chasm.Context, _ struct{}) (struct{}, error) { + assert(chasmCtx, c) + return struct{}{}, nil + }, + struct{}{}, + ) + require.NoError(t, err) +} + // Test the full executeInvocationTask flow with direct handler calls func TestExecuteInvocationTaskNexus_Outcomes(t *testing.T) { cases := []struct { @@ -141,17 +229,7 @@ func TestExecuteInvocationTaskNexus_Outcomes(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - // Setup namespace - factory := namespace.NewDefaultReplicationResolverFactory() - detail := &persistencespb.NamespaceDetail{ - Info: &persistencespb.NamespaceInfo{ - Id: "namespace-id", - Name: "namespace-name", - }, - Config: &persistencespb.NamespaceConfig{}, - } - ns, err := namespace.FromPersistentState(detail, factory(detail)) - require.NoError(t, err) + ns := newTestNamespace(t) // Setup metrics expectations metricsHandler := metrics.NewMockHandler(ctrl) @@ -191,14 +269,6 @@ func TestExecuteInvocationTaskNexus_Outcomes(t *testing.T) { }, } - chasmRegistry := chasm.NewRegistry(logger) - err = chasmRegistry.Register(&Library{ - InvocationTaskHandler: handler, - }) - require.NoError(t, err) - err = chasmRegistry.Register(&mockNexusCompletionGetterLibrary{}) - require.NoError(t, err) - callback := &Callback{ CallbackState: &callbackspb.CallbackState{ RequestId: "request-id", @@ -215,43 +285,7 @@ func TestExecuteInvocationTaskNexus_Outcomes(t *testing.T) { }, } - // Create completion - completion := nexusrpc.CompleteOperationOptions{} - - executionKey := chasm.ExecutionKey{ - NamespaceID: "namespace-id", - BusinessID: "workflow-id", - RunID: "run-id", - } - testEngine := chasmtest.NewEngine(t, chasmRegistry) - engineCtx := chasm.NewEngineContext(context.Background(), testEngine) - _, err = chasm.StartExecution( - engineCtx, - executionKey, - func(ctx chasm.MutableContext, _ struct{}) (*mockNexusCompletionGetterComponent, error) { - return &mockNexusCompletionGetterComponent{ - completion: completion, - Callback: chasm.NewComponentField(ctx, callback), - }, nil - }, - struct{}{}, - ) - require.NoError(t, err) - - rootRef := chasm.NewComponentRef[*mockNexusCompletionGetterComponent](executionKey) - callbackRef, err := chasm.ReadComponent( - engineCtx, - rootRef, - func(_ *mockNexusCompletionGetterComponent, chasmCtx chasm.Context, _ struct{}) (chasm.ComponentRef, error) { - serialized, err := chasmCtx.Ref(callback) - if err != nil { - return chasm.ComponentRef{}, err - } - return chasm.DeserializeComponentRef(serialized) - }, - struct{}{}, - ) - require.NoError(t, err) + engineCtx, callbackRef := newInvocationTaskTest(t, handler, callback, nexusrpc.CompleteOperationOptions{}) executeErr := handler.Execute( engineCtx, @@ -261,16 +295,9 @@ func TestExecuteInvocationTaskNexus_Outcomes(t *testing.T) { ) // Verify outcome by reading component state directly. - resultCallback, err := chasm.ReadComponent( - engineCtx, - callbackRef, - func(c *Callback, _ chasm.Context, _ struct{}) (*Callback, error) { - return c, nil - }, - struct{}{}, - ) - require.NoError(t, err) - tc.assertOutcome(t, resultCallback, executeErr) + readCallbackState(t, engineCtx, callbackRef, func(chasmCtx chasm.Context, c *Callback) { + tc.assertOutcome(t, c, executeErr) + }) }) } } diff --git a/chasm/lib/nexusoperation/callbacks_test.go b/chasm/lib/nexusoperation/callbacks_test.go new file mode 100644 index 00000000000..2f0942f56d8 --- /dev/null +++ b/chasm/lib/nexusoperation/callbacks_test.go @@ -0,0 +1,677 @@ +package nexusoperation + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/callback" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + nexusoperationpb "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/testing/protorequire" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func newCallbackTestContext() *chasm.MockMutableContext { + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { return defaultTime }, + HandleExecutionKey: func() chasm.ExecutionKey { + return chasm.ExecutionKey{NamespaceID: "ns-id", BusinessID: "op-id", RunID: "run-id"} + }, + HandleNamespaceEntry: func() *namespace.Namespace { + return namespace.NewNamespaceForTest(&persistencespb.NamespaceInfo{Name: "ns-name"}, nil, false, nil, 0) + }, + HandleExecutionInfo: func() chasm.ExecutionInfo { + return chasm.ExecutionInfo{CloseTime: defaultTime} + }, + GoCtx: context.WithValue(context.Background(), OperationContextKey, &OperationContext{ + MetricTagConfig: dynamicconfig.GetTypedPropertyFn(NexusMetricTagConfig{}), + }), + }, + } + // Attaching completion callbacks runs Callback component code, which reads its own values off + // the chasm context. Register them the same way the framework does in production, so the + // callback library keeps its context key unexported. + ctx.RegisterLibrary(callback.NewNilLibrary()) + return ctx +} + +func testCallbackLimits() callbackLimits { + return callbackLimits{maxCount: 10, maxSourceContextSize: 2 * 1024 * 1024} +} + +func TestNewStandaloneOperationAttachesCompletionCallbacks(t *testing.T) { + t.Parallel() + + newStartReq := func(cbs ...*commonpb.Callback) *nexusoperationpb.StartNexusOperationRequest { + return &nexusoperationpb.StartNexusOperationRequest{ + EndpointId: "endpoint-id", + FrontendRequest: &workflowservice.StartNexusOperationExecutionRequest{ + Namespace: "ns-name", + OperationId: "op-id", + RequestId: "req-id", + Endpoint: "test-endpoint", + Service: "test-service", + Operation: "test-operation", + CompletionCallbacks: cbs, + }, + } + } + + t.Run("WithCallbacks", func(t *testing.T) { + ctx := newCallbackTestContext() + + req := newStartReq(newNexusCallback()) + op, err := newStandaloneOperation(ctx, req, testCallbackLimits(), newTestLinkValidator(10, 10)) + require.NoError(t, err) + require.Equal(t, nexusoperationpb.OPERATION_STATUS_SCHEDULED, op.Status) + + // Callbacks start in STANDBY, only transitioning to SCHEDULED when the SANO completes. + require.Len(t, op.Callbacks, 1) + attachedCB := op.Callbacks["req-id-0"].Get(ctx) + require.Equal(t, callbackspb.CALLBACK_STATUS_STANDBY, attachedCB.Status) + }) + + t.Run("WithoutCallbacks", func(t *testing.T) { + ctx := newCallbackTestContext() + + op, err := newStandaloneOperation(ctx, newStartReq(), testCallbackLimits(), newTestLinkValidator(10, 10)) + require.NoError(t, err) + require.Nil(t, op.Callbacks) + }) + + t.Run("EnforcesTheCallersLimit", func(t *testing.T) { + ctx := newCallbackTestContext() + + callbackLimits := testCallbackLimits() + callbackLimits.maxCount = 1 + + _, err := newStandaloneOperation(ctx, newStartReq( + newNexusCallback(), + newNexusCallback(), + ), callbackLimits, newTestLinkValidator(10, 10)) + var failedPreconditionErr *serviceerror.FailedPrecondition + require.ErrorAs(t, err, &failedPreconditionErr) + require.ErrorContains(t, err, "cannot attach more than 1 callbacks") + }) +} + +func TestAddCompletionCallbacks(t *testing.T) { + t.Parallel() + + t.Run("AttachesCallbacksInStandby", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + + cb1 := newNexusCallback() + cb1.GetNexus().Url = "https://example.com/callback-1" + + // Set data on the second CB, we confirm is added to the Operation. + cb2 := newNexusCallback() + cb2.GetNexus().Url = "https://example.com/callback-2" + cb2.GetNexus().Header = map[string]string{ + "key": "xxx", + } + cb2.Links = []*commonpb.Link{{Variant: &commonpb.Link_WorkflowEvent_{ + WorkflowEvent: &commonpb.Link_WorkflowEvent{Namespace: "ns-name", WorkflowId: "wf-id"}, + }}} + + cbs := []*commonpb.Callback{ + cb1, + cb2, + } + + err := op.addCompletionCallbacks(ctx, "req-id", cbs, testCallbackLimits()) + require.NoError(t, err) + require.Len(t, op.Callbacks, 2) + + first, ok := op.Callbacks["req-id-0"] + require.True(t, ok) + firstCb := first.Get(ctx) + require.Equal(t, "https://example.com/callback-1", firstCb.GetCallback().GetNexus().GetUrl()) + + second, ok := op.Callbacks["req-id-1"] + require.True(t, ok) + secondCb := second.Get(ctx) + require.Equal(t, callbackspb.CALLBACK_STATUS_STANDBY, secondCb.Status) + require.Equal(t, defaultTime, secondCb.RegistrationTime.AsTime()) + require.Equal(t, "https://example.com/callback-2", secondCb.GetCallback().GetNexus().GetUrl()) + require.Equal(t, map[string]string{"key": "xxx"}, secondCb.GetCallback().GetNexus().GetHeader()) + require.Len(t, secondCb.GetCallback().GetLinks(), 1) + + // Each callback gets its own, unique request ID. + require.NotEqual(t, "req-id", firstCb.RequestId) + require.NotEqual(t, "req-id", secondCb.RequestId) + require.NotEqual(t, firstCb.RequestId, secondCb.RequestId) + + // STANDBY means no invocation task yet; only the scheduled transition's tasks are present. + for _, task := range ctx.Tasks { + _, isInvocation := task.Payload.(*callbackspb.InvocationTask) + require.False(t, isInvocation, "callbacks must not be invoked while in STANDBY") + } + }) + + t.Run("EmptyListIsNoOp", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + + require.NoError(t, op.addCompletionCallbacks(ctx, "req-id", nil, testCallbackLimits())) + require.Nil(t, op.Callbacks) + }) + + t.Run("ReAttachingTheSameRequestIsIdempotent", func(t *testing.T) { + // A retried start (or a retried on_conflict_options attach) must not duplicate callbacks. + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + cbs := []*commonpb.Callback{newNexusCallback()} + + require.NoError(t, op.addCompletionCallbacks(ctx, "req-id", cbs, testCallbackLimits())) + require.NoError(t, op.addCompletionCallbacks(ctx, "req-id", cbs, testCallbackLimits())) + require.Len(t, op.Callbacks, 1) + }) + + t.Run("ReAttachingTheSameRequestIsIdempotentAfterClose", func(t *testing.T) { + // A start request can reach addCompletionCallbacks twice: once creating the operation, then + // again if the client retries and the engine dedups on request ID. The operation may have closed + // in between, and the retry must still report success for callbacks that are already persisted + // rather than FailedPrecondition. + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + cbs := []*commonpb.Callback{newNexusCallback()} + + require.NoError(t, op.addCompletionCallbacks(ctx, "req-id", cbs, testCallbackLimits())) + require.NoError(t, TransitionSucceeded.Apply(op, ctx, EventSucceeded{})) + require.Equal(t, callbackspb.CALLBACK_STATUS_SCHEDULED, op.Callbacks["req-id-0"].Get(ctx).Status) + + tasksBefore := len(ctx.Tasks) + require.NoError(t, op.addCompletionCallbacks(ctx, "req-id", cbs, testCallbackLimits())) + + // The retry must leave the already-scheduled callback alone: re-attaching would reset it to + // STANDBY, stranding a callback the terminal transition had already released for delivery. + require.Len(t, op.Callbacks, 1) + require.Equal(t, callbackspb.CALLBACK_STATUS_SCHEDULED, op.Callbacks["req-id-0"].Get(ctx).Status) + require.Len(t, ctx.Tasks, tasksBefore) + }) + + t.Run("DistinctRequestsAccumulate", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + cbs := []*commonpb.Callback{newNexusCallback()} + + require.NoError(t, op.addCompletionCallbacks(ctx, "req-1", cbs, testCallbackLimits())) + require.NoError(t, op.addCompletionCallbacks(ctx, "req-2", cbs, testCallbackLimits())) + require.Len(t, op.Callbacks, 2) + }) + + t.Run("RejectsExceedingTheLimit", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + cbs := []*commonpb.Callback{newNexusCallback(), newNexusCallback()} + + callbackLimits := testCallbackLimits() + callbackLimits.maxCount = 1 + + err := op.addCompletionCallbacks(ctx, "req-id", cbs, callbackLimits) + var failedPreconditionErr *serviceerror.FailedPrecondition + require.ErrorAs(t, err, &failedPreconditionErr) + require.Contains(t, err.Error(), "cannot attach more than 1 callbacks") + require.Empty(t, op.Callbacks) + }) + + t.Run("RejectsExceedingTheLimitWithAlreadyAttachedCallbacks", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + + callbackLimits := testCallbackLimits() + callbackLimits.maxCount = 2 + + require.NoError(t, op.addCompletionCallbacks(ctx, "req-1", []*commonpb.Callback{ + newNexusCallback(), + }, callbackLimits)) + + err := op.addCompletionCallbacks(ctx, "req-2", []*commonpb.Callback{ + newNexusCallback(), + newNexusCallback(), + }, callbackLimits) + var failedPreconditionErr *serviceerror.FailedPrecondition + require.ErrorAs(t, err, &failedPreconditionErr) + require.Contains(t, err.Error(), "1 callbacks already attached") + require.Len(t, op.Callbacks, 1) + }) + + // The frontend bounds the source context on one request. Only this check bounds what accumulates + // across the several requests an on-conflict attach can make. + t.Run("RejectsExceedingTheSourceContextLimitWithAlreadyAttachedCallbacks", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + limits := callbackLimits{maxCount: 10, maxSourceContextSize: 1500} + + // Each request is within the limit on its own. + require.NoError(t, op.addCompletionCallbacks(ctx, "req-1", []*commonpb.Callback{ + newWorkerCallback(900), + }, limits)) + + err := op.addCompletionCallbacks(ctx, "req-2", []*commonpb.Callback{ + newWorkerCallback(900), + }, limits) + var failedPreconditionErr *serviceerror.FailedPrecondition + require.ErrorAs(t, err, &failedPreconditionErr) + require.ErrorContains(t, err, "cannot attach more than 1500 bytes of callback source_context") + require.ErrorContains(t, err, "bytes already attached") + // The rejected request attached nothing. + require.Len(t, op.Callbacks, 1) + }) + + t.Run("RejectsAClosedOperation", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + require.NoError(t, TransitionSucceeded.Apply(op, ctx, EventSucceeded{})) + + err := op.addCompletionCallbacks(ctx, "req-id", []*commonpb.Callback{ + newNexusCallback(), + }, testCallbackLimits()) + var failedPreconditionErr *serviceerror.FailedPrecondition + require.ErrorAs(t, err, &failedPreconditionErr) + require.Contains(t, err.Error(), "cannot attach callbacks to a closed nexus operation") + require.Empty(t, op.Callbacks) + }) + + t.Run("RejectsAnEmptyRequestID", func(t *testing.T) { + // Callback IDs are derived from the request ID, so without one two distinct requests would + // produce colliding keys and silently overwrite each other. The frontend always supplies one + // (validator.normalizeRequestID); this guards a history-side caller that did not. + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + + err := op.addCompletionCallbacks(ctx, "", []*commonpb.Callback{ + newNexusCallback(), + }, testCallbackLimits()) + var invalidArgErr *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgErr) + require.Contains(t, err.Error(), "without a request ID") + require.Empty(t, op.Callbacks) + }) +} + +func TestScheduleCompletionCallbacksOnTerminalTransition(t *testing.T) { + t.Parallel() + + timeoutFailure := &failurepb.Failure{ + Message: "timed out", + FailureInfo: &failurepb.Failure_TimeoutFailureInfo{ + TimeoutFailureInfo: &failurepb.TimeoutFailureInfo{ + TimeoutType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, + }, + }, + } + + // In all scenarios, we expect the CHASM callbacks to be scheduled once the + // SANO transitions to a terminal state. + for _, tc := range []struct { + name string + fromStatus nexusoperationpb.OperationStatus + apply func(*Operation, *chasm.MockMutableContext) error + expectedStatus nexusoperationpb.OperationStatus + }{ + { + name: "Succeeded", + apply: func(o *Operation, ctx *chasm.MockMutableContext) error { + return TransitionSucceeded.Apply(o, ctx, EventSucceeded{}) + }, + expectedStatus: nexusoperationpb.OPERATION_STATUS_SUCCEEDED, + }, + { + name: "Failed", + apply: func(o *Operation, ctx *chasm.MockMutableContext) error { + return TransitionFailed.Apply(o, ctx, EventFailed{ + Failure: &failurepb.Failure{Message: "boom"}, + }) + }, + expectedStatus: nexusoperationpb.OPERATION_STATUS_FAILED, + }, + { + name: "Canceled", + apply: func(o *Operation, ctx *chasm.MockMutableContext) error { + return TransitionCanceled.Apply(o, ctx, EventCanceled{ + Failure: &failurepb.Failure{Message: "canceled"}, + }) + }, + expectedStatus: nexusoperationpb.OPERATION_STATUS_CANCELED, + }, + { + name: "TimedOut", + apply: func(o *Operation, ctx *chasm.MockMutableContext) error { + return TransitionTimedOut.Apply(o, ctx, EventTimedOut{Failure: timeoutFailure}) + }, + expectedStatus: nexusoperationpb.OPERATION_STATUS_TIMED_OUT, + }, + { + name: "Terminated", + apply: func(o *Operation, ctx *chasm.MockMutableContext) error { + _, err := o.Terminate(ctx, chasm.TerminateComponentRequest{ + RequestID: "terminate-req-id", + Reason: "because", + }) + return err + }, + expectedStatus: nexusoperationpb.OPERATION_STATUS_TERMINATED, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + require.NoError(t, op.addCompletionCallbacks(ctx, "req-id", []*commonpb.Callback{ + newNexusCallback(), + }, testCallbackLimits())) + + tasksBefore := len(ctx.Tasks) + require.NoError(t, tc.apply(op, ctx)) + require.Equal(t, tc.expectedStatus, op.Status) + + cb := op.Callbacks["req-id-0"].Get(ctx) + require.Equal(t, callbackspb.CALLBACK_STATUS_SCHEDULED, cb.Status) + + // Closing must emit exactly one callback invocation task, routed to the callback's host. + newTasks := ctx.Tasks[tasksBefore:] + require.Len(t, newTasks, 1) + require.IsType(t, &callbackspb.InvocationTask{}, newTasks[0].Payload) + + // The task's Destination attribute for Nexus callbacks is the hostname, which + // is fixed in newNexusCallback. + const wantHost = "https://nexus.ex.xxxxx.cluster.tmprl.cloud:7243" + require.Equal(t, wantHost, newTasks[0].Attributes.Destination) + }) + } + + t.Run("NoCallbacksIsANoOp", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + + tasksBefore := len(ctx.Tasks) + require.NoError(t, TransitionSucceeded.Apply(op, ctx, EventSucceeded{})) + require.Len(t, ctx.Tasks, tasksBefore) + }) +} + +// TestTerminateRejectedForClosedOperation guards the source-state list of TransitionTerminated, +// ensuring you cannot terminate an already terminal SANO. +func TestTerminateRejectedForClosedOperation(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + close func(*testing.T, *Operation, *chasm.MockMutableContext) + expectedStatus nexusoperationpb.OperationStatus + expectedResult *commonpb.Payload + expectedFailure *failurepb.Failure + }{ + { + name: "Canceled", + close: func(t *testing.T, o *Operation, ctx *chasm.MockMutableContext) { + require.NoError(t, TransitionCanceled.Apply(o, ctx, EventCanceled{ + Failure: &failurepb.Failure{Message: "canceled by handler"}, + })) + }, + expectedStatus: nexusoperationpb.OPERATION_STATUS_CANCELED, + expectedFailure: &failurepb.Failure{Message: "canceled by handler"}, + }, + { + name: "Failed", + close: func(t *testing.T, o *Operation, ctx *chasm.MockMutableContext) { + require.NoError(t, TransitionFailed.Apply(o, ctx, EventFailed{ + Failure: &failurepb.Failure{Message: "boom"}, + })) + }, + expectedStatus: nexusoperationpb.OPERATION_STATUS_FAILED, + expectedFailure: &failurepb.Failure{Message: "boom"}, + }, + { + name: "TimedOut", + close: func(t *testing.T, o *Operation, ctx *chasm.MockMutableContext) { + require.NoError(t, TransitionTimedOut.Apply(o, ctx, EventTimedOut{ + Failure: &failurepb.Failure{ + Message: "timed out", + FailureInfo: &failurepb.Failure_TimeoutFailureInfo{ + TimeoutFailureInfo: &failurepb.TimeoutFailureInfo{ + TimeoutType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, + }, + }, + }, + })) + }, + expectedStatus: nexusoperationpb.OPERATION_STATUS_TIMED_OUT, + expectedFailure: &failurepb.Failure{ + Message: "timed out", + FailureInfo: &failurepb.Failure_TimeoutFailureInfo{ + TimeoutFailureInfo: &failurepb.TimeoutFailureInfo{ + TimeoutType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, + }, + }, + }, + }, + { + name: "Succeeded", + close: func(t *testing.T, o *Operation, ctx *chasm.MockMutableContext) { + require.NoError(t, TransitionSucceeded.Apply(o, ctx, EventSucceeded{ + Result: mustToPayload(t, "result"), + })) + }, + expectedStatus: nexusoperationpb.OPERATION_STATUS_SUCCEEDED, + expectedResult: mustToPayload(t, "result"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := newCallbackTestContext() + op := newScheduledTestOperation(t, ctx) + + op.RequestData = chasm.NewDataField(ctx, &nexusoperationpb.OperationRequestData{}) + op.Visibility = chasm.NewComponentField(ctx, chasm.NewVisibilityWithData(ctx, nil, nil)) + require.NoError(t, op.addCompletionCallbacks(ctx, "req-id", []*commonpb.Callback{ + newNexusCallback(), + }, testCallbackLimits())) + + tc.close(t, op, ctx) + tasksAfterClose := len(ctx.Tasks) + + _, err := op.Terminate(ctx, chasm.TerminateComponentRequest{ + RequestID: "terminate-req-id", + Reason: "because", + Identity: "test-identity", + }) + // Terminate rejects a closed operation itself rather than letting the transition do it, + // so the caller gets the same error the cancel path uses instead of the state machine's + // "invalid transition from OPERATION_STATUS_X". + require.ErrorIs(t, err, ErrOperationAlreadyCompleted) + var failedPreconditionErr *serviceerror.FailedPrecondition + require.ErrorAs(t, err, &failedPreconditionErr) + require.NotErrorIs(t, err, chasm.ErrInvalidTransition) + + // The closed state, its outcome, and its already-scheduled callback must all survive. + require.Equal(t, tc.expectedStatus, op.Status) + require.Nil(t, op.TerminateState) + require.Len(t, ctx.Tasks, tasksAfterClose) + + resp, err := op.buildDescribeResponse(ctx, &nexusoperationpb.DescribeNexusOperationRequest{ + FrontendRequest: &workflowservice.DescribeNexusOperationExecutionRequest{IncludeOutcome: true}, + }) + require.NoError(t, err) + protorequire.ProtoEqual(t, tc.expectedFailure, resp.GetFrontendResponse().GetFailure()) + protorequire.ProtoEqual(t, tc.expectedResult, resp.GetFrontendResponse().GetResult()) + }) + } +} + +func TestBuildCompletionCallbackInfos(t *testing.T) { + t.Parallel() + + t.Run("NoCallbacks", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newTestOperation() + + infos, err := op.buildCompletionCallbackInfos(ctx) + require.NoError(t, err) + require.Nil(t, infos) + }) + + t.Run("ReportsStateAndOutcomePerCallback", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newTestOperation() + + newCB := func(url string, status callbackspb.CallbackStatus) *callback.Callback { + cb := callback.NewCallback( + "req-id", + timestamppb.New(defaultTime), + &callbackspb.Callback{ + Variant: &callbackspb.Callback_Nexus_{ + Nexus: &callbackspb.Callback_Nexus{Url: url}, + }, + }, + ) + cb.SetStateMachineState(status) + return cb + } + + failed := newCB("http://localhost:8080/failed", callbackspb.CALLBACK_STATUS_FAILED) + failed.Attempt = 3 + failed.LastAttemptFailure = &failurepb.Failure{Message: "boom"} + + op.Callbacks = chasm.Map[string, *callback.Callback]{ + "req-id-0": chasm.NewComponentField(ctx, newCB("http://localhost:8080/standby", callbackspb.CALLBACK_STATUS_STANDBY)), + "req-id-1": chasm.NewComponentField(ctx, newCB("http://localhost:8080/succeeded", callbackspb.CALLBACK_STATUS_SUCCEEDED)), + "req-id-2": chasm.NewComponentField(ctx, failed), + } + + infos, err := op.buildCompletionCallbackInfos(ctx) + require.NoError(t, err) + require.Len(t, infos, 3) + + // Ordering follows the sorted callback IDs, not the (randomized) map iteration order. + require.Equal(t, "http://localhost:8080/standby", infos[0].GetInfo().GetCallback().GetNexus().GetUrl()) + require.Equal(t, enumspb.CALLBACK_STATE_STANDBY, infos[0].GetInfo().GetState()) + require.Nil(t, infos[0].GetInfo().GetResult()) + require.Equal(t, defaultTime, infos[0].GetInfo().GetRegistrationTime().AsTime()) + // Every callback on a standalone operation is triggered by the operation completing. + require.NotNil(t, infos[0].GetTrigger().GetOperationCompleted()) + + require.Equal(t, enumspb.CALLBACK_STATE_SUCCEEDED, infos[1].GetInfo().GetState()) + require.NotNil(t, infos[1].GetInfo().GetSuccess()) + + require.Equal(t, enumspb.CALLBACK_STATE_FAILED, infos[2].GetInfo().GetState()) + require.Equal(t, int32(3), infos[2].GetInfo().GetAttempt()) + protorequire.ProtoEqual(t, + &failurepb.Failure{Message: "boom"}, + infos[2].GetInfo().GetFailure()) + }) +} + +// TestCompletionCallbacksRoundTripThroughTheTree exercises the Callbacks map against a real CHASM tree +// rather than a mock context, so that a missing component registration or an unserializable field shows +// up here instead of at runtime. +func TestCompletionCallbacksRoundTripThroughTheTree(t *testing.T) { + logger := log.NewNoopLogger() + registry := chasm.NewRegistry(logger) + require.NoError(t, registry.Register(&chasm.CoreLibrary{})) + require.NoError(t, registry.Register(&Library{ + componentOnlyLibrary: componentOnlyLibrary{ + metricTagConfig: dynamicconfig.GetTypedPropertyFn(NexusMetricTagConfig{}), + }, + })) + require.NoError(t, registry.Register(callback.NewNilLibrary())) + + timeSource := clock.NewEventTimeSource() + timeSource.Update(defaultTime) + nodeBackend := &chasm.MockNodeBackend{ + HandleNextTransitionCount: func() int64 { return 2 }, + HandleGetCurrentVersion: func() int64 { return 1 }, + HandleCurrentVersionedTransition: func() *persistencespb.VersionedTransition { + return &persistencespb.VersionedTransition{NamespaceFailoverVersion: 1, TransitionCount: 1} + }, + HandleGetNamespaceEntry: func() *namespace.Namespace { + return namespace.NewNamespaceForTest(&persistencespb.NamespaceInfo{Name: "ns-name"}, nil, false, nil, 0) + }, + } + root := chasm.NewEmptyTree(registry, timeSource, nodeBackend, chasm.DefaultPathEncoder, logger, metrics.NoopMetricsHandler) + ctx := chasm.NewMutableContext(context.Background(), root) + + op := NewOperation(&nexusoperationpb.OperationState{ + Status: nexusoperationpb.OPERATION_STATUS_STARTED, + Endpoint: "test-endpoint", + ScheduledTime: timestamppb.New(defaultTime), + }) + op.RequestData = chasm.NewDataField(ctx, &nexusoperationpb.OperationRequestData{}) + op.Visibility = chasm.NewComponentField(ctx, chasm.NewVisibilityWithData(ctx, nil, nil)) + require.NoError(t, op.addCompletionCallbacks(ctx, "req-id", []*commonpb.Callback{ + newNexusCallback(), + }, testCallbackLimits())) + require.NoError(t, root.SetRootComponent(op)) + _, err := root.CloseTransaction() + require.NoError(t, err) + + ctx = chasm.NewMutableContext(context.Background(), root) + require.Len(t, op.Callbacks, 1) + cb := op.Callbacks["req-id-0"].Get(ctx) + require.Equal(t, callbackspb.CALLBACK_STATUS_STANDBY, cb.Status) + + // The callback resolves its parent via a ParentPtr, so the Operation must satisfy + // callback.CompletionSource from inside the tree, not just as a compile-time assertion. + require.NoError(t, TransitionSucceeded.Apply(op, ctx, EventSucceeded{Result: mustToPayload(t, "result")})) + require.Equal(t, callbackspb.CALLBACK_STATUS_SCHEDULED, cb.Status) + + completion, err := cb.CompletionSource.Get(ctx).GetNexusCompletion(ctx, cb.RequestId) + require.NoError(t, err) + require.Nil(t, completion.Error) +} + +// TestDescribeResponseIncludesCompletionCallbacks covers the plumbing from the component into the +// DescribeNexusOperationExecution response. +func TestDescribeResponseIncludesCompletionCallbacks(t *testing.T) { + t.Parallel() + + newOp := func(ctx chasm.MutableContext) *Operation { + op := newTestOperation() + op.RequestData = chasm.NewDataField(ctx, &nexusoperationpb.OperationRequestData{}) + op.Visibility = chasm.NewComponentField(ctx, chasm.NewVisibilityWithData(ctx, nil, nil)) + return op + } + req := &nexusoperationpb.DescribeNexusOperationRequest{ + FrontendRequest: &workflowservice.DescribeNexusOperationExecutionRequest{}, + } + + t.Run("WithCallbacks", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newOp(ctx) + require.NoError(t, op.addCompletionCallbacks(ctx, "req-id", []*commonpb.Callback{ + newNexusCallback(), + }, testCallbackLimits())) + + resp, err := op.buildDescribeResponse(ctx, req) + require.NoError(t, err) + + cbs := resp.GetFrontendResponse().GetCompletionCallbacks() + require.Len(t, cbs, 1) + require.Equal(t, enumspb.CALLBACK_STATE_STANDBY, cbs[0].GetInfo().GetState()) + }) + + t.Run("WithoutCallbacks", func(t *testing.T) { + ctx := newCallbackTestContext() + op := newOp(ctx) + + resp, err := op.buildDescribeResponse(ctx, req) + require.NoError(t, err) + require.Empty(t, resp.GetFrontendResponse().GetCompletionCallbacks()) + }) +} diff --git a/chasm/lib/nexusoperation/config.go b/chasm/lib/nexusoperation/config.go index 8d2b6b6c9ec..882d0030127 100644 --- a/chasm/lib/nexusoperation/config.go +++ b/chasm/lib/nexusoperation/config.go @@ -6,8 +6,10 @@ import ( "text/template" "time" + "go.temporal.io/server/chasm/lib/callback" "go.temporal.io/server/common" "go.temporal.io/server/common/backoff" + "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/config" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" @@ -35,6 +37,13 @@ var Enabled = dynamicconfig.NewNamespaceBoolSetting( `Toggles standalone Nexus operation functionality on the server.`, ) +var EnabledCallbackKinds = dynamicconfig.NewNamespaceTypedSettingWithConverter( + "nexusoperation.enabledCallbackKinds", + callbacks.ConvertEnabledKinds, + []callbacks.Kind{}, // i.e. callbacks not enabled at all. + `The list of completion callback kinds that may be attached to a standalone Nexus operation execution.`, +) + var EnableChasmWorkflowOperations = dynamicconfig.NewNamespaceBoolSetting( "nexusoperation.enableChasmWorkflowOperations", false, @@ -243,6 +252,9 @@ Added for safety. Defaults to true. Likely to be removed in future server versio type Config struct { Enabled dynamicconfig.BoolPropertyFnWithNamespaceFilter EnableChasm dynamicconfig.BoolPropertyFnWithNamespaceFilter + EnabledCallbackKinds dynamicconfig.TypedPropertyFnWithNamespaceFilter[[]callbacks.Kind] + MaxCallbacksPerExecution dynamicconfig.IntPropertyFnWithNamespaceFilter + WorkerSourceContextAggregateMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter EnableChasmNexusWorkflowOperations dynamicconfig.BoolPropertyFnWithNamespaceFilter ChasmNexusWorkflowOperationsRolloutPercent dynamicconfig.IntPropertyFnWithNamespaceFilter NumHistoryShards int32 @@ -273,32 +285,35 @@ type Config struct { func configProvider(dc *dynamicconfig.Collection, cfg *config.Persistence) *Config { return &Config{ - Enabled: Enabled.Get(dc), - EnableChasm: dynamicconfig.EnableChasm.Get(dc), - EnableChasmNexusWorkflowOperations: EnableChasmWorkflowOperations.Get(dc), + Enabled: Enabled.Get(dc), + EnableChasm: dynamicconfig.EnableChasm.Get(dc), + EnabledCallbackKinds: EnabledCallbackKinds.Get(dc), + MaxCallbacksPerExecution: callback.MaxPerExecution.Get(dc), + WorkerSourceContextAggregateMaxSize: callback.WorkerSourceContextAggregateMaxSize.Get(dc), + EnableChasmNexusWorkflowOperations: EnableChasmWorkflowOperations.Get(dc), ChasmNexusWorkflowOperationsRolloutPercent: ChasmWorkflowOperationsRolloutPercent.Get(dc), - NumHistoryShards: cfg.NumHistoryShards, - LongPollBuffer: LongPollBuffer.Get(dc), - LongPollTimeout: LongPollTimeout.Get(dc), - RequestTimeout: RequestTimeout.Get(dc), - MinRequestTimeout: MinRequestTimeout.Get(dc), - MaxConcurrentOperationsPerWorkflow: MaxConcurrentOperationsPerWorkflow.Get(dc), - MaxServiceNameLength: MaxServiceNameLength.Get(dc), - MaxOperationNameLength: MaxOperationNameLength.Get(dc), - MaxOperationTokenLength: MaxOperationTokenLength.Get(dc), - MaxOperationHeaderSize: MaxOperationHeaderSize.Get(dc), - DisallowedOperationHeaders: DisallowedOperationHeaders.Get(dc), - MaxOperationScheduleToCloseTimeout: MaxOperationScheduleToCloseTimeout.Get(dc), - PayloadSizeLimit: dynamicconfig.BlobSizeLimitError.Get(dc), - PayloadSizeLimitWarn: dynamicconfig.BlobSizeLimitWarn.Get(dc), - MaxUserMetadataSummarySize: dynamicconfig.MaxUserMetadataSummarySize.Get(dc), - MaxUserMetadataDetailsSize: dynamicconfig.MaxUserMetadataDetailsSize.Get(dc), - CallbackURLTemplate: CallbackURLTemplate.Get(dc), - UseSystemCallbackURL: UseSystemCallbackURL.Get(dc), - UseNewFailureWireFormat: UseNewFailureWireFormat.Get(dc), - VisibilityMaxPageSize: dynamicconfig.FrontendVisibilityMaxPageSize.Get(dc), - MaxIDLengthLimit: dynamicconfig.MaxIDLengthLimit.Get(dc), - MaxReasonLength: MaxReasonLength.Get(dc), - RetryPolicy: RetryPolicy.Get(dc), + NumHistoryShards: cfg.NumHistoryShards, + LongPollBuffer: LongPollBuffer.Get(dc), + LongPollTimeout: LongPollTimeout.Get(dc), + RequestTimeout: RequestTimeout.Get(dc), + MinRequestTimeout: MinRequestTimeout.Get(dc), + MaxConcurrentOperationsPerWorkflow: MaxConcurrentOperationsPerWorkflow.Get(dc), + MaxServiceNameLength: MaxServiceNameLength.Get(dc), + MaxOperationNameLength: MaxOperationNameLength.Get(dc), + MaxOperationTokenLength: MaxOperationTokenLength.Get(dc), + MaxOperationHeaderSize: MaxOperationHeaderSize.Get(dc), + DisallowedOperationHeaders: DisallowedOperationHeaders.Get(dc), + MaxOperationScheduleToCloseTimeout: MaxOperationScheduleToCloseTimeout.Get(dc), + PayloadSizeLimit: dynamicconfig.BlobSizeLimitError.Get(dc), + PayloadSizeLimitWarn: dynamicconfig.BlobSizeLimitWarn.Get(dc), + MaxUserMetadataSummarySize: dynamicconfig.MaxUserMetadataSummarySize.Get(dc), + MaxUserMetadataDetailsSize: dynamicconfig.MaxUserMetadataDetailsSize.Get(dc), + CallbackURLTemplate: CallbackURLTemplate.Get(dc), + UseSystemCallbackURL: UseSystemCallbackURL.Get(dc), + UseNewFailureWireFormat: UseNewFailureWireFormat.Get(dc), + VisibilityMaxPageSize: dynamicconfig.FrontendVisibilityMaxPageSize.Get(dc), + MaxIDLengthLimit: dynamicconfig.MaxIDLengthLimit.Get(dc), + MaxReasonLength: MaxReasonLength.Get(dc), + RetryPolicy: RetryPolicy.Get(dc), } } diff --git a/chasm/lib/nexusoperation/frontend.go b/chasm/lib/nexusoperation/frontend.go index 673b345d7a0..6187960b228 100644 --- a/chasm/lib/nexusoperation/frontend.go +++ b/chasm/lib/nexusoperation/frontend.go @@ -10,6 +10,7 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" nexusoperationpb "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" + "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/log" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" @@ -49,13 +50,15 @@ func NewFrontendHandler( endpointRegistry commonnexus.EndpointRegistry, saMapperProvider searchattribute.MapperProvider, saValidator *searchattribute.Validator, + callbackValidator callbacks.Validator, + linkValidator *linkValidator, ) FrontendHandler { return &frontendHandler{ client: client, config: config, namespaceRegistry: namespaceRegistry, endpointRegistry: endpointRegistry, - validator: newValidator(config, logger, saMapperProvider, saValidator), + validator: newValidator(config, logger, saMapperProvider, saValidator, callbackValidator, linkValidator), } } @@ -72,7 +75,7 @@ func (h *frontendHandler) StartNexusOperationExecution( return nil, err } - if err := h.validator.validateAndNormalizeStartRequest(req); err != nil { + if err := h.validator.validateAndNormalizeStartRequest(ctx, req); err != nil { return nil, err } diff --git a/chasm/lib/nexusoperation/fx.go b/chasm/lib/nexusoperation/fx.go index 53728ee56d7..ef5a5eb834a 100644 --- a/chasm/lib/nexusoperation/fx.go +++ b/chasm/lib/nexusoperation/fx.go @@ -30,6 +30,7 @@ const nexusCallbackSourceHeader = "Nexus-Callback-Source" var Module = fx.Module( "chasm.lib.nexusoperation", fx.Provide(configProvider), + fx.Provide(linkValidatorProvider), fx.Provide(commonnexus.NewCallbackTokenGenerator), fx.Provide(endpointRegistryProvider), fx.Invoke(endpointRegistryLifetimeHooks), @@ -50,6 +51,7 @@ var Module = fx.Module( var FrontendModule = fx.Module( "chasm.lib.nexusoperation.frontend", fx.Provide(configProvider), + fx.Provide(linkValidatorProvider), fx.Provide(nexusoperationpb.NewNexusOperationServiceLayeredClient), fx.Provide(NewFrontendHandler), fx.Provide(newComponentOnlyLibrary), diff --git a/chasm/lib/nexusoperation/handler.go b/chasm/lib/nexusoperation/handler.go index 3e005654f99..86c09ff263b 100644 --- a/chasm/lib/nexusoperation/handler.go +++ b/chasm/lib/nexusoperation/handler.go @@ -17,14 +17,16 @@ import ( type handler struct { nexusoperationpb.UnimplementedNexusOperationServiceServer - config *Config - logger log.Logger + config *Config + linkValidator *linkValidator + logger log.Logger } -func newHandler(config *Config, logger log.Logger) *handler { +func newHandler(config *Config, linkValidator *linkValidator, logger log.Logger) *handler { return &handler{ - config: config, - logger: logger, + config: config, + linkValidator: linkValidator, + logger: logger, } } @@ -36,14 +38,22 @@ func (h *handler) StartNexusOperation( defer log.CapturePanic(h.logger, &err) frontendReq := req.GetFrontendRequest() + // Read once, so it's consistent for both the creation and on-conflict paths, + // despite being in two transactions. + limits := callbackLimits{ + maxCount: h.config.MaxCallbacksPerExecution(frontendReq.GetNamespace()), + maxSourceContextSize: h.config.WorkerSourceContextAggregateMaxSize(frontendReq.GetNamespace()), + } - result, err := chasm.StartExecution[*Operation]( + result, err := chasm.StartExecution( ctx, chasm.ExecutionKey{ NamespaceID: req.GetNamespaceId(), BusinessID: frontendReq.GetOperationId(), }, - newStandaloneOperation, + func(mutableCtx chasm.MutableContext, req *nexusoperationpb.StartNexusOperationRequest) (*Operation, error) { + return newStandaloneOperation(mutableCtx, req, limits, h.linkValidator) + }, req, chasm.WithRequestID(frontendReq.GetRequestId()), chasm.WithBusinessIDPolicy( @@ -52,6 +62,8 @@ func (h *handler) StartNexusOperation( ), ) if err != nil { + // ExecutionAlreadyStarted would only be returned if the IDConflictPolicy were FAIL. + // Otherwise, we'd return the existing SANO. (And apply the OnConflictOptions next.) if alreadyStartedErr, ok := errors.AsType[*chasm.ExecutionAlreadyStartedError](err); ok { return nil, serviceerror.NewNexusOperationExecutionAlreadyStartedf( alreadyStartedErr.CurrentRequestID, @@ -64,6 +76,12 @@ func (h *handler) StartNexusOperation( return nil, err } + if !result.Created { + if err := h.applyOnConflictOptions(ctx, result.ExecutionKey, frontendReq, limits); err != nil { + return nil, err + } + } + return &nexusoperationpb.StartNexusOperationResponse{ FrontendResponse: &workflowservice.StartNexusOperationExecutionResponse{ RunId: result.ExecutionKey.RunID, @@ -72,6 +90,47 @@ func (h *handler) StartNexusOperation( }, nil } +// applyOnConflictOptions applies the request's on_conflict_options to a SANO. +func (h *handler) applyOnConflictOptions( + ctx context.Context, + key chasm.ExecutionKey, + req *workflowservice.StartNexusOperationExecutionRequest, + limits callbackLimits, +) error { + cbs := req.GetCompletionCallbacks() + links := req.GetLinks() + onConflict := req.GetOnConflictOptions() + attachCallbacks := onConflict.GetAttachCompletionCallbacks() && len(cbs) > 0 + attachLinks := onConflict.GetAttachLinks() && len(links) > 0 + if !attachCallbacks && !attachLinks { + return nil + } + + requestID := req.GetRequestId() + namespaceName := req.GetNamespace() + // TODO: Use chasm.UpdateWithStartExecution to avoid a second transaction once the engine supports + // BusinessIDConflictPolicyFail in the updateFn path. (Same for standalone Activities.) + _, _, err := chasm.UpdateComponent( + ctx, + chasm.NewComponentRef[*Operation](key), + func(o *Operation, ctx chasm.MutableContext, _ any) (any, error) { + if attachCallbacks { + if err := o.addCompletionCallbacks(ctx, requestID, cbs, limits); err != nil { + return nil, err + } + } + if attachLinks { + if err := o.attachLinks(ctx, links, requestID, h.linkValidator, namespaceName); err != nil { + return nil, err + } + } + return nil, nil + }, + nil, + ) + return err +} + // DescribeNexusOperation queries current operation state, optionally as a long-poll that waits // for any state change. // diff --git a/chasm/lib/nexusoperation/link_validator.go b/chasm/lib/nexusoperation/link_validator.go new file mode 100644 index 00000000000..06abd61f838 --- /dev/null +++ b/chasm/lib/nexusoperation/link_validator.go @@ -0,0 +1,37 @@ +package nexusoperation + +import ( + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/links" +) + +// linkValidator validates links attached to standalone Nexus operation executions. +type linkValidator struct { + // Distinct per-component type: all CHASM lib fx modules provide into the same + // container, so *commonlinks.Validator cannot be injected directly. + *links.Validator +} + +func newLinkValidator( + maxLinksPerRequest dynamicconfig.IntPropertyFnWithNamespaceFilter, + maxLinksPerComponent dynamicconfig.IntPropertyFnWithNamespaceFilter, + linkMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter, +) *linkValidator { + return &linkValidator{ + links.NewValidator( + "a nexus operation", + maxLinksPerRequest, + maxLinksPerComponent, + linkMaxSize, + ), + } +} + +// linkValidatorProvider builds the linkValidator from dynamic config. +func linkValidatorProvider(dc *dynamicconfig.Collection) *linkValidator { + return newLinkValidator( + dynamicconfig.FrontendMaxLinksPerRequest.Get(dc), + dynamicconfig.MaxLinksPerComponent.Get(dc), + dynamicconfig.FrontendLinkMaxSize.Get(dc), + ) +} diff --git a/chasm/lib/nexusoperation/links_test.go b/chasm/lib/nexusoperation/links_test.go new file mode 100644 index 00000000000..785ce3214dd --- /dev/null +++ b/chasm/lib/nexusoperation/links_test.go @@ -0,0 +1,259 @@ +package nexusoperation + +import ( + "testing" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm" + nexusoperationpb "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" + "go.temporal.io/server/common/testing/protorequire" +) + +// newTestLinkValidator returns a linkValidator with the given caps and a permissive per-link size limit. +func newTestLinkValidator(maxPerRequest, maxPerComponent int) *linkValidator { + return newLinkValidator( + func(string) int { return maxPerRequest }, + func(string) int { return maxPerComponent }, + func(string) int { return 4000 }, + ) +} + +// newLinkTestContext returns a mock context backed by stored, standing in for the framework's +// per-request link storage: writes land in ctx.LinksByRequest, while reads (Links/RequestLinks) are +// served from stored. Tests copy a write into stored to simulate it having been persisted. +func newLinkTestContext(stored map[string][]*commonpb.Link) *chasm.MockMutableContext { + ctx := newCallbackTestContext() + ctx.HandleLinks = func(chasm.Component) []*commonpb.Link { + var all []*commonpb.Link + for _, links := range stored { + all = append(all, links...) + } + return all + } + ctx.HandleRequestLinks = func(_ chasm.Component, requestID string) ([]*commonpb.Link, error) { + return stored[requestID], nil + } + return ctx +} + +func testLink(workflowID string) *commonpb.Link { + return &commonpb.Link{Variant: &commonpb.Link_WorkflowEvent_{ + WorkflowEvent: &commonpb.Link_WorkflowEvent{ + Namespace: "ns-name", + WorkflowId: workflowID, + RunId: "wf-run-id", + }, + }} +} + +func TestNewStandaloneOperationAttachesLinks(t *testing.T) { + t.Parallel() + + newStartReq := func(links ...*commonpb.Link) *nexusoperationpb.StartNexusOperationRequest { + return &nexusoperationpb.StartNexusOperationRequest{ + EndpointId: "endpoint-id", + FrontendRequest: &workflowservice.StartNexusOperationExecutionRequest{ + Namespace: "ns-name", + OperationId: "op-id", + RequestId: "req-id", + Endpoint: "test-endpoint", + Service: "test-service", + Operation: "test-operation", + Links: links, + }, + } + } + + t.Run("WithLinks", func(t *testing.T) { + ctx := newLinkTestContext(map[string][]*commonpb.Link{}) + link := testLink("wf-id") + + op, err := newStandaloneOperation(ctx, newStartReq(link), testCallbackLimits(), newTestLinkValidator(10, 10)) + require.NoError(t, err) + require.Equal(t, nexusoperationpb.OPERATION_STATUS_SCHEDULED, op.Status) + + // Links are keyed by the request that contributed them. + protorequire.ProtoSliceEqual(t, []*commonpb.Link{link}, ctx.LinksByRequest[op]["req-id"]) + }) + + t.Run("WithoutLinks", func(t *testing.T) { + ctx := newLinkTestContext(map[string][]*commonpb.Link{}) + + op, err := newStandaloneOperation(ctx, newStartReq(), testCallbackLimits(), newTestLinkValidator(10, 10)) + require.NoError(t, err) + require.Empty(t, ctx.LinksByRequest[op]) + }) + + t.Run("RejectsAnInvalidLink", func(t *testing.T) { + ctx := newLinkTestContext(map[string][]*commonpb.Link{}) + invalid := &commonpb.Link{Variant: &commonpb.Link_WorkflowEvent_{ + WorkflowEvent: &commonpb.Link_WorkflowEvent{WorkflowId: "wf-id", RunId: "wf-run-id"}, + }} + + _, err := newStandaloneOperation(ctx, newStartReq(invalid), testCallbackLimits(), newTestLinkValidator(10, 10)) + require.ErrorAs(t, err, new(*serviceerror.InvalidArgument)) + require.ErrorContains(t, err, "must not have an empty namespace") + }) + + t.Run("EnforcesThePerComponentCap", func(t *testing.T) { + ctx := newLinkTestContext(map[string][]*commonpb.Link{}) + + _, err := newStandaloneOperation( + ctx, + newStartReq(testLink("wf-1"), testLink("wf-2")), + testCallbackLimits(), + newTestLinkValidator(10, 1), + ) + require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) + require.ErrorContains(t, err, "cannot attach more than 1 links to a nexus operation") + }) +} + +func TestOperationAttachLinks(t *testing.T) { + t.Parallel() + + t.Run("EmptyListIsANoOp", func(t *testing.T) { + stored := map[string][]*commonpb.Link{} + ctx := newLinkTestContext(stored) + op := newScheduledTestOperation(t, ctx) + + require.NoError(t, op.attachLinks(ctx, nil, "req-id", newTestLinkValidator(10, 10), "ns-name")) + require.Empty(t, ctx.LinksByRequest[op]) + }) + + t.Run("SameRequestIDIsNoOp", func(t *testing.T) { + // A retried start (or a retried on_conflict_options attach) must not duplicate links. + stored := map[string][]*commonpb.Link{} + ctx := newLinkTestContext(stored) + op := newScheduledTestOperation(t, ctx) + validator := newTestLinkValidator(10, 10) + linkA, linkB := testLink("wf-a"), testLink("wf-b") + + // The first call records the request's links verbatim, without intra-batch dedup, matching + // the workflow and standalone activity start paths. + require.NoError(t, op.attachLinks(ctx, []*commonpb.Link{linkA, linkA}, "req-1", validator, "ns-name")) + stored["req-1"] = ctx.LinksByRequest[op]["req-1"] + protorequire.ProtoSliceEqual(t, []*commonpb.Link{linkA, linkA}, stored["req-1"]) + + // A retry under the same requestID is a no-op, even with different links. + require.NoError(t, op.attachLinks(ctx, []*commonpb.Link{linkB}, "req-1", validator, "ns-name")) + protorequire.ProtoSliceEqual(t, []*commonpb.Link{linkA, linkA}, ctx.LinksByRequest[op]["req-1"]) + }) + + t.Run("DistinctRequestsAccumulate", func(t *testing.T) { + stored := map[string][]*commonpb.Link{} + ctx := newLinkTestContext(stored) + op := newScheduledTestOperation(t, ctx) + validator := newTestLinkValidator(10, 10) + + require.NoError(t, op.attachLinks(ctx, []*commonpb.Link{testLink("wf-1")}, "req-1", validator, "ns-name")) + stored["req-1"] = ctx.LinksByRequest[op]["req-1"] + require.NoError(t, op.attachLinks(ctx, []*commonpb.Link{testLink("wf-2")}, "req-2", validator, "ns-name")) + + require.Len(t, ctx.LinksByRequest[op], 2) + }) + + t.Run("RejectsAClosedOperation", func(t *testing.T) { + ctx := newLinkTestContext(map[string][]*commonpb.Link{}) + op := newScheduledTestOperation(t, ctx) + require.NoError(t, TransitionSucceeded.Apply(op, ctx, EventSucceeded{})) + + err := op.attachLinks(ctx, []*commonpb.Link{testLink("wf-id")}, "req-new", newTestLinkValidator(10, 10), "ns-name") + require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) + require.ErrorContains(t, err, "cannot attach links to a closed nexus operation") + }) + + t.Run("IsIdempotentAfterClose", func(t *testing.T) { + // The original attach succeeded but the response was lost; by the time the client retries the + // operation has closed. The retry must still report success for links already persisted. + link := testLink("wf-id") + ctx := newLinkTestContext(map[string][]*commonpb.Link{"req-1": {link}}) + op := newScheduledTestOperation(t, ctx) + require.NoError(t, TransitionSucceeded.Apply(op, ctx, EventSucceeded{})) + + require.NoError(t, op.attachLinks(ctx, []*commonpb.Link{link}, "req-1", newTestLinkValidator(10, 10), "ns-name")) + }) + + t.Run("RejectsExceedingThePerRequestCap", func(t *testing.T) { + ctx := newLinkTestContext(map[string][]*commonpb.Link{}) + op := newScheduledTestOperation(t, ctx) + + err := op.attachLinks( + ctx, + []*commonpb.Link{testLink("wf-1"), testLink("wf-2")}, + "req-id", + newTestLinkValidator(1, 10), + "ns-name", + ) + require.ErrorAs(t, err, new(*serviceerror.InvalidArgument)) + require.ErrorContains(t, err, "cannot attach more than 1 links per request") + require.Empty(t, ctx.LinksByRequest[op]) + }) + + t.Run("RejectsExceedingThePerComponentCapWithAlreadyAttachedLinks", func(t *testing.T) { + ctx := newLinkTestContext(map[string][]*commonpb.Link{"req-1": {testLink("wf-existing")}}) + op := newScheduledTestOperation(t, ctx) + + err := op.attachLinks( + ctx, + []*commonpb.Link{testLink("wf-new")}, + "req-2", + newTestLinkValidator(10, 1), + "ns-name", + ) + require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) + require.ErrorContains(t, err, "1 links already attached") + }) +} + +// TestDescribeResponseIncludesLinks covers the plumbing from both link sources into the +// DescribeNexusOperationExecution response. +func TestDescribeResponseIncludesLinks(t *testing.T) { + t.Parallel() + + req := &nexusoperationpb.DescribeNexusOperationRequest{ + FrontendRequest: &workflowservice.DescribeNexusOperationExecutionRequest{}, + } + newOp := func(ctx chasm.MutableContext) *Operation { + op := newTestOperation() + op.RequestData = chasm.NewDataField(ctx, &nexusoperationpb.OperationRequestData{}) + op.Visibility = chasm.NewComponentField(ctx, chasm.NewVisibilityWithData(ctx, nil, nil)) + return op + } + + t.Run("UnionsCallerAndHandlerLinks", func(t *testing.T) { + callerLink, handlerLink := testLink("caller-wf"), testLink("handler-wf") + ctx := newLinkTestContext(map[string][]*commonpb.Link{"req-id": {callerLink}}) + op := newOp(ctx) + // Links returned by the Nexus handler on its start/completion response. + op.Links = []*commonpb.Link{handlerLink} + + resp, err := op.buildDescribeResponse(ctx, req) + require.NoError(t, err) + protorequire.ProtoElementsMatch(t, + []*commonpb.Link{callerLink, handlerLink}, + resp.GetFrontendResponse().GetInfo().GetLinks()) + }) + + t.Run("CallerLinksOnly", func(t *testing.T) { + callerLink := testLink("caller-wf") + ctx := newLinkTestContext(map[string][]*commonpb.Link{"req-id": {callerLink}}) + + resp, err := newOp(ctx).buildDescribeResponse(ctx, req) + require.NoError(t, err) + protorequire.ProtoSliceEqual(t, + []*commonpb.Link{callerLink}, + resp.GetFrontendResponse().GetInfo().GetLinks()) + }) + + t.Run("WithoutLinks", func(t *testing.T) { + ctx := newLinkTestContext(map[string][]*commonpb.Link{}) + + resp, err := newOp(ctx).buildDescribeResponse(ctx, req) + require.NoError(t, err) + require.Empty(t, resp.GetFrontendResponse().GetInfo().GetLinks()) + }) +} diff --git a/chasm/lib/nexusoperation/operation.go b/chasm/lib/nexusoperation/operation.go index 4ce985df2fb..27178bdce84 100644 --- a/chasm/lib/nexusoperation/operation.go +++ b/chasm/lib/nexusoperation/operation.go @@ -3,6 +3,7 @@ package nexusoperation import ( "fmt" "maps" + "slices" "strings" "time" @@ -12,14 +13,19 @@ import ( enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" nexuspb "go.temporal.io/api/nexus/v1" + apinexusoperationpb "go.temporal.io/api/nexusoperation/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/callback" nexusoperationpb "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/backoff" + commoncallbacks "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/metrics" commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/softassert" queueserrors "go.temporal.io/server/service/history/queues/errors" @@ -40,6 +46,7 @@ var _ chasm.RootComponent = (*Operation)(nil) var _ chasm.StateMachine[nexusoperationpb.OperationStatus] = (*Operation)(nil) var _ chasm.VisibilitySearchAttributesProvider = (*Operation)(nil) var _ chasm.NexusCompletionHandler = (*Operation)(nil) +var _ callback.CompletionSource = (*Operation)(nil) // ErrCancellationAlreadyRequested is returned when a cancellation has already been requested for an operation. var ErrCancellationAlreadyRequested = serviceerror.NewFailedPrecondition("cancellation already requested") @@ -99,6 +106,10 @@ type Operation struct { Cancellation chasm.Field[*Cancellation] Outcome chasm.Field[*nexusoperationpb.OperationOutcome] Visibility chasm.Field[*chasm.Visibility] + + // Callbacks holds completion callbacks to be invoked when this reaches a terminal state. + // Keyed by completionCallbackID(requestID, index). + Callbacks chasm.Map[string, *callback.Callback] } // NewOperation creates a new Operation component with the given persisted state. @@ -109,6 +120,8 @@ func NewOperation(state *nexusoperationpb.OperationState) *Operation { func newStandaloneOperation( ctx chasm.MutableContext, req *nexusoperationpb.StartNexusOperationRequest, + limits callbackLimits, + linkValidator *linkValidator, ) (*Operation, error) { frontendReq := req.GetFrontendRequest() op := NewOperation(&nexusoperationpb.OperationState{ @@ -133,6 +146,23 @@ func newStandaloneOperation( frontendReq.GetSearchAttributes().GetIndexedFields(), nil, )) + if err := op.addCompletionCallbacks( + ctx, + frontendReq.GetRequestId(), + frontendReq.GetCompletionCallbacks(), + limits, + ); err != nil { + return nil, err + } + if err := op.attachLinks( + ctx, + frontendReq.GetLinks(), + frontendReq.GetRequestId(), + linkValidator, + frontendReq.GetNamespace(), + ); err != nil { + return nil, err + } if err := TransitionScheduled.Apply(op, ctx, EventScheduled{}); err != nil { return nil, err } @@ -304,6 +334,12 @@ func (o *Operation) loadStartArgs( } else { // Standalone operation: there is no workflow caller, so add a nexus_operation self-link // as the caller link for the completion callback. + // + // Only the caller link goes to the handler. Links the client attached to the start request + // (and any it attached later via on_conflict_options) are recorded on the operation and + // surfaced through Describe, but are not forwarded: this matches the workflow-backed case, + // where the handler likewise receives only the workflow_event caller link and not the links + // carried on the StartWorkflowExecution request. requestData := o.RequestData.Get(ctx) invocationData = InvocationData{ Input: requestData.GetInput(), @@ -393,7 +429,7 @@ func (o *Operation) resolveUnsuccessfully(ctx chasm.MutableContext, failure *fai // NextAttemptScheduleTime is only valid in BACKING_OFF; clear on close o.NextAttemptScheduleTime = nil - return nil + return o.scheduleCompletionCallbacks(ctx) } func (o *Operation) getOrCreateOutcome(ctx chasm.MutableContext) *nexusoperationpb.OperationOutcome { @@ -405,6 +441,235 @@ func (o *Operation) getOrCreateOutcome(ctx chasm.MutableContext) *nexusoperation return outcome } +// callbackLimits bounds what an operation's completion callbacks may carry. +type callbackLimits struct { + // maxCount is the number of callbacks one operation may have attached. + maxCount int + // maxSourceContextSize is the total bytes of Worker source context they may carry between them. + maxSourceContextSize int +} + +// addCompletionCallbacks creates the child CHASM callback components. They stay in STANDBY until this +// Operation reaches a terminal state. +// +// Callbacks are keyed by request ID plus their position within the request, so re-attaching the same +// request is a no-op rather than a duplicate. The idempotency probe runs before the closed check, so a +// retry still succeeds if the operation closed after the first attach. +// +// The limits are re-checked here because callback.Validator only bounds the callbacks on the start +// request; callbacks added later via on_conflict_options bypass it. +func (o *Operation) addCompletionCallbacks( + ctx chasm.MutableContext, + requestID string, + completionCallbacks []*commonpb.Callback, + limits callbackLimits, +) error { + if len(completionCallbacks) == 0 { + return nil + } + if requestID == "" { + return serviceerror.NewInvalidArgument("cannot attach completion callbacks without a request ID") + } + // Attaching is atomic, so the presence of the first key means this request already attached all of + // its callbacks. See the note above on why this precedes the closed check. + if _, ok := o.Callbacks[completionCallbackID(requestID, 0)]; ok { + return nil + } + if o.isClosed() { + return serviceerror.NewFailedPrecondition("cannot attach callbacks to a closed nexus operation") + } + + currentCount := len(o.Callbacks) + if len(completionCallbacks)+currentCount > limits.maxCount { + return serviceerror.NewFailedPreconditionf( + "cannot attach more than %d callbacks to a nexus operation (%d callbacks already attached)", + limits.maxCount, + currentCount, + ) + } + + // Verify that adding the new callbacks won't exceed the aggregate size allowed. + existingBytes := 0 + for _, cb := range o.Callbacks { + existingBytes += cb.Get(ctx).SourceContextSize() + } + addingBytes := commoncallbacks.SourceContextSize(completionCallbacks) + if existingBytes+addingBytes > limits.maxSourceContextSize { + return serviceerror.NewFailedPreconditionf( + "cannot attach more than %d bytes of callback source_context to a nexus operation "+ + "(%d bytes already attached, %d more requested)", + limits.maxSourceContextSize, + existingBytes, + addingBytes, + ) + } + + if o.Callbacks == nil { + o.Callbacks = make(chasm.Map[string, *callback.Callback], len(completionCallbacks)) + } + + registrationTime := timestamppb.New(ctx.Now(o)) + for idx, cb := range completionCallbacks { + chasmCB, err := callback.FromAPICallback(cb) + if err != nil { + return err + } + + // Give each callback its own, unique request ID. Since using the same request ID as the + // operation which added the callbacks would be ambiguous if it added more than one callback. + cbRequestID := uuid.NewString() + callbackObj := callback.NewCallback(cbRequestID, registrationTime, chasmCB) + o.Callbacks[completionCallbackID(requestID, idx)] = chasm.NewComponentField(ctx, callbackObj) + } + return nil +} + +// attachLinks records the given links on the operation keyed by requestID. Duplicates within the same +// batch are kept as-is, matching the workflow and standalone activity start paths. If the requestID has +// already been used to attach links the call is a no-op, making retries idempotent even after the +// operation has closed. Returns an error if the operation is closed (and the requestID is new), if the +// per-component cap would be exceeded, or if the request's per-link size, per-request count, or variant +// shape is invalid. +func (o *Operation) attachLinks( + ctx chasm.MutableContext, + links []*commonpb.Link, + requestID string, + validator *linkValidator, + namespaceName string, +) error { + if len(links) == 0 { + return nil + } + // Idempotency check must run before isClosed: if a prior attach succeeded but the response was + // lost and the operation closed before the client retried, we must still return success rather + // than FailedPrecondition for work already persisted. + priorForRequest, err := ctx.RequestLinks(o, requestID) + if err != nil { + return err + } + if len(priorForRequest) > 0 { + return nil + } + if o.isClosed() { + return serviceerror.NewFailedPrecondition("cannot attach links to a closed nexus operation") + } + if err := validator.ValidateRequest(namespaceName, links); err != nil { + return err + } + // The cap bounds caller-attached links only; links returned by the Nexus handler + // (OperationState.links) arrive on a separate path and are not counted here. + if err := validator.ValidateTotal(namespaceName, len(ctx.Links(o)), len(links)); err != nil { + return err + } + return ctx.SetRequestLinks(o, requestID, links) +} + +// allLinks returns every link associated with the operation: those attached by callers on their start +// and on-conflict attach requests, plus any the Nexus handler returned on its start or completion +// response (standalone operations only; workflow-backed ones carry theirs on history events). +func (o *Operation) allLinks(ctx chasm.Context) []*commonpb.Link { + requestLinks := ctx.Links(o) + all := make([]*commonpb.Link, 0, len(requestLinks)+len(o.Links)) + all = append(all, common.CloneProtoSlice(requestLinks)...) + all = append(all, common.CloneProtoSlice(o.Links)...) + return all +} + +// completionCallbackID defines the stable key used for keeping track of attached completion callbacks. +func completionCallbackID(requestID string, idx int) string { + return fmt.Sprintf("%s-%d", requestID, idx) +} + +// scheduleCompletionCallbacks releases every STANDBY completion callback for delivery. Called from each +// terminal transition. +func (o *Operation) scheduleCompletionCallbacks(ctx chasm.MutableContext) error { + return callback.ScheduleStandbyCallbacks(ctx, o.Callbacks) +} + +// GetNexusCompletion implements callback.CompletionSource, providing the result of the Nexus operation. +func (o *Operation) GetNexusCompletion(ctx chasm.Context, _ string) (nexusrpc.CompleteOperationOptions, error) { + if !o.isClosed() { + return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInternal("nexus operation has not completed yet") + } + + key := ctx.ExecutionKey() + backLink := commonnexus.ConvertLinkNexusOperationToNexusLink(&commonpb.Link_NexusOperation{ + Namespace: ctx.NamespaceEntry().Name().String(), + OperationId: key.BusinessID, + RunId: key.RunID, + }) + + opts := nexusrpc.CompleteOperationOptions{ + StartTime: o.GetScheduledTime().AsTime(), + CloseTime: ctx.ExecutionInfo().CloseTime, + Links: []nexus.Link{backLink}, + } + + result, failure := o.outcome(ctx) + if o.Status == nexusoperationpb.OPERATION_STATUS_SUCCEEDED { + opts.Result = result + return opts, nil + } + if failure == nil { + return nexusrpc.CompleteOperationOptions{}, + serviceerror.NewInternalf("nexus operation in status %v has no outcome", o.Status) + } + + state := nexus.OperationStateFailed + message := "operation failed" + if o.Status == nexusoperationpb.OPERATION_STATUS_CANCELED { + state = nexus.OperationStateCanceled + message = "operation canceled" + } + + nf, err := commonnexus.TemporalFailureToNexusFailure(failure) + if err != nil { + return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInternalf("failed to convert failure: %v", err) + } + opErr := &nexus.OperationError{ + State: state, + Message: message, + Cause: &nexus.FailureError{Failure: nf}, + } + if err := nexusrpc.MarkAsWrapperError(nexusrpc.DefaultFailureConverter(), opErr); err != nil { + return nexusrpc.CompleteOperationOptions{}, err + } + opts.Error = opErr + return opts, nil +} + +// buildCompletionCallbackInfos projects the attached completion callbacks onto the API surface for the +// describe response. +func (o *Operation) buildCompletionCallbackInfos(ctx chasm.Context) ([]*apinexusoperationpb.CallbackInfo, error) { + if len(o.Callbacks) == 0 { + return nil, nil + } + + // All standalone Nexus operation callbacks have the same trigger. + trigger := &apinexusoperationpb.CallbackInfo_Trigger{ + Variant: &apinexusoperationpb.CallbackInfo_Trigger_OperationCompleted{ + OperationCompleted: &apinexusoperationpb.CallbackInfo_OperationCompleted{}, + }, + } + + // We make no attempt to return Callbacks in the specific order they were + // added, we just sort the Callbacks so that the returned CallbackInfo is + // stable. + sortedCallbackKeys := slices.Sorted(maps.Keys(o.Callbacks)) + infos := make([]*apinexusoperationpb.CallbackInfo, 0, len(o.Callbacks)) + for _, id := range sortedCallbackKeys { + info, err := o.Callbacks[id].Get(ctx).ToAPICallbackInfo(ctx) + if err != nil { + return nil, err + } + infos = append(infos, &apinexusoperationpb.CallbackInfo{ + Trigger: trigger, + Info: info, + }) + } + return infos, nil +} + func (o *Operation) Terminate( ctx chasm.MutableContext, req chasm.TerminateComponentRequest, @@ -416,6 +681,9 @@ func (o *Operation) Terminate( } return chasm.TerminateComponentResponse{}, nil } + if o.isClosed() { + return chasm.TerminateComponentResponse{}, ErrOperationAlreadyCompleted + } return chasm.TerminateComponentResponse{}, TransitionTerminated.Apply(o, ctx, EventTerminated{ TerminateComponentRequest: req, @@ -441,10 +709,16 @@ func (o *Operation) buildDescribeResponse( return nil, err } + callbackInfos, err := o.buildCompletionCallbackInfos(ctx) + if err != nil { + return nil, err + } + resp := &workflowservice.DescribeNexusOperationExecutionResponse{ - RunId: ctx.ExecutionKey().RunID, - Info: o.buildExecutionInfo(ctx), - LongPollToken: token, + RunId: ctx.ExecutionKey().RunID, + Info: o.buildExecutionInfo(ctx), + LongPollToken: token, + CompletionCallbacks: callbackInfos, } if req.GetFrontendRequest().GetIncludeInput() { resp.Input = o.RequestData.Get(ctx).GetInput() @@ -554,7 +828,7 @@ func (o *Operation) buildExecutionInfo(ctx chasm.Context) *nexuspb.NexusOperatio }, NexusHeader: requestData.GetNexusHeader(), UserMetadata: requestData.GetUserMetadata(), - Links: o.Links, + Links: o.allLinks(ctx), Identity: requestData.GetIdentity(), } diff --git a/chasm/lib/nexusoperation/operation_statemachine.go b/chasm/lib/nexusoperation/operation_statemachine.go index ba6f2498992..01217bd3301 100644 --- a/chasm/lib/nexusoperation/operation_statemachine.go +++ b/chasm/lib/nexusoperation/operation_statemachine.go @@ -190,8 +190,8 @@ var TransitionSucceeded = chasm.NewTransition( } o.emitOnSucceededMetrics(ctx, closeTime) - // Terminal state - no tasks to emit. - return nil + // Schedule the SANO's completion callbacks. + return o.scheduleCompletionCallbacks(ctx) }, ) @@ -259,7 +259,6 @@ var TransitionTerminated = chasm.NewTransition( nexusoperationpb.OPERATION_STATUS_SCHEDULED, nexusoperationpb.OPERATION_STATUS_STARTED, nexusoperationpb.OPERATION_STATUS_BACKING_OFF, - nexusoperationpb.OPERATION_STATUS_CANCELED, }, nexusoperationpb.OPERATION_STATUS_TERMINATED, func(o *Operation, ctx chasm.MutableContext, event EventTerminated) error { diff --git a/chasm/lib/nexusoperation/task_handler_helpers.go b/chasm/lib/nexusoperation/task_handler_helpers.go index 06a3d040193..2a6d0a03e39 100644 --- a/chasm/lib/nexusoperation/task_handler_helpers.go +++ b/chasm/lib/nexusoperation/task_handler_helpers.go @@ -270,7 +270,7 @@ func newInvocationResult( } if opErr, ok := errors.AsType[*nexus.OperationError](callErr); ok { - failure, err := operationErrorToFailure(opErr) + failure, err := commonnexus.OperationErrorToTemporalFailure(opErr) if err != nil { return nil, err } @@ -309,20 +309,6 @@ func newInvocationResult( return invocationResultRetry{failure: failure}, nil } -func operationErrorToFailure(opErr *nexus.OperationError) (*failurepb.Failure, error) { - var nf nexus.Failure - if opErr.OriginalFailure != nil { - nf = *opErr.OriginalFailure - } else { - var err error - nf, err = nexusrpc.DefaultFailureConverter().ErrorToFailure(opErr) - if err != nil { - return nil, err - } - } - return commonnexus.NexusFailureToTemporalFailure(*nexusrpc.UnwrapFailure(&nf)) -} - func buildCallbackURL( useSystemCallback bool, callbackTemplate *template.Template, diff --git a/chasm/lib/nexusoperation/validator.go b/chasm/lib/nexusoperation/validator.go index 7b0da3dcd50..2011e47eb5b 100644 --- a/chasm/lib/nexusoperation/validator.go +++ b/chasm/lib/nexusoperation/validator.go @@ -1,6 +1,7 @@ package nexusoperation import ( + "context" "slices" "strings" @@ -10,6 +11,7 @@ import ( "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/primitives/timestamp" @@ -33,10 +35,12 @@ type cancelOrTerminateRequest interface { // Requests are mutated in place: defaults are applied, values are capped to their configured // limits, and headers are lower-cased. type validator struct { - config *Config - logger log.Logger - saMapperProvider searchattribute.MapperProvider - saValidator *searchattribute.Validator + config *Config + logger log.Logger + saMapperProvider searchattribute.MapperProvider + saValidator *searchattribute.Validator + callbackValidator callbacks.Validator + linkValidator *linkValidator } func newValidator( @@ -44,16 +48,23 @@ func newValidator( logger log.Logger, saMapperProvider searchattribute.MapperProvider, saValidator *searchattribute.Validator, + callbackValidator callbacks.Validator, + linkValidator *linkValidator, ) *validator { return &validator{ - config: config, - logger: logger, - saMapperProvider: saMapperProvider, - saValidator: saValidator, + config: config, + logger: logger, + saMapperProvider: saMapperProvider, + saValidator: saValidator, + callbackValidator: callbackValidator, + linkValidator: linkValidator, } } -func (v *validator) validateAndNormalizeStartRequest(req *workflowservice.StartNexusOperationExecutionRequest) error { +func (v *validator) validateAndNormalizeStartRequest( + ctx context.Context, + req *workflowservice.StartNexusOperationExecutionRequest, +) error { ns := req.GetNamespace() if err := v.normalizeRequestID(&req.RequestId); err != nil { @@ -87,6 +98,32 @@ func (v *validator) validateAndNormalizeStartRequest(req *workflowservice.StartN if err := v.validateAndNormalizeSearchAttributes(req); err != nil { return err } + // Callbacks + cbs := req.GetCompletionCallbacks() + if len(cbs) > 0 { + enabledKinds := v.config.EnabledCallbackKinds(ns) + if len(enabledKinds) == 0 { + return serviceerror.NewInvalidArgument("completion callbacks are not enabled for this namespace") + } + opts := callbacks.ValidatorOptions{EnabledKinds: enabledKinds} + if err := v.callbackValidator.Validate(ctx, ns, cbs, opts); err != nil { + return err + } + // Validate checks callback sizes individually. Now check the aggregate size against the limit. + // Use 0 for existing size, because the frontend cannot see an already-running operation's + // callbacks. Operation.addCompletionCallbacks re-checks against those. + if err := v.callbackValidator.ValidateTotalSourceContextSize(ns, 0, callbacks.SourceContextSize(cbs)); err != nil { + return err + } + } + // Links, including those carried by the request's callbacks. + links := req.GetLinks() + if err := v.linkValidator.ValidateRequestWithCallbacks(ns, links, cbs); err != nil { + return err + } + if err := v.validateOnConflictOptions(req); err != nil { + return err + } v.normalizeIDPolicies(req) return nil @@ -313,6 +350,31 @@ func (v *validator) normalizeIDPolicies(req *workflowservice.StartNexusOperation } } +// validateOnConflictOptions validates the on_conflict_options of a start request: +// - attach_completion_callbacks requires attach_request_id, since it is embedded into the keys we use +// to persist them on the Operation's Callbacks map. +// - attach_request_id requires at least one completion callback or link, since attaching a request ID +// is only meaningful alongside something to attach. +// +// attach_links is independent and may be set on its own. +func (v *validator) validateOnConflictOptions(req *workflowservice.StartNexusOperationExecutionRequest) error { + onConflictOptions := req.GetOnConflictOptions() + if onConflictOptions == nil { + return nil + } + if onConflictOptions.GetAttachCompletionCallbacks() && !onConflictOptions.GetAttachRequestId() { + return serviceerror.NewInvalidArgument( + "on_conflict_options: attach_completion_callbacks requires attach_request_id to be set") + } + if onConflictOptions.GetAttachRequestId() && + len(req.GetCompletionCallbacks()) == 0 && + len(req.GetLinks()) == 0 { + return serviceerror.NewInvalidArgument( + "on_conflict_options: attach_request_id requires at least one completion callback or link") + } + return nil +} + // normalizeRequestID validates the request ID, or sets it to a UUID if empty. func (v *validator) normalizeRequestID(requestID *string) error { if *requestID == "" { diff --git a/chasm/lib/nexusoperation/validator_test.go b/chasm/lib/nexusoperation/validator_test.go index 6dff582da19..5895ee5f84e 100644 --- a/chasm/lib/nexusoperation/validator_test.go +++ b/chasm/lib/nexusoperation/validator_test.go @@ -1,6 +1,9 @@ package nexusoperation import ( + "context" + "fmt" + "regexp" "strings" "testing" "time" @@ -8,10 +11,12 @@ import ( "github.com/stretchr/testify/require" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" + nexusoperationpb "go.temporal.io/api/nexusoperation/v1" sdkpb "go.temporal.io/api/sdk/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" @@ -23,7 +28,72 @@ import ( ) func newTestValidator(config *Config) *validator { - return newValidator(config, log.NewNoopLogger(), nil, nil) + return newValidator( + config, + log.NewNoopLogger(), + nil, + nil, + mustNewCallbackValidator(), + newTestLinkValidator(10, 10), + ) +} + +func mustNewCallbackValidator() callbacks.Validator { + allowAllAddresses := callbacks.AddressMatchRules{ + Rules: []callbacks.AddressMatchRule{ + {Regexp: regexp.MustCompile(`.*`), AllowInsecure: true}, + }, + } + cfg := callbacks.ValidatorConfig{ + MaxCallbacksPerExecution: func(string) int { return 10 }, + MaxIDLengthLimit: func() int { return 10 }, + URLMaxLength: func(string) int { return 1000 }, + HeaderMaxSize: func(string) int { return 4096 }, + EndpointRules: func(string) callbacks.AddressMatchRules { return allowAllAddresses }, + MaxServiceNameLength: func(string) int { return 10 }, + MaxOperationNameLength: func(string) int { return 10 }, + WorkerSourceContextMaxSize: func(string) int { return 1000 }, + WorkerSourceContextAggregateMaxSize: func(string) int { return 1500 }, + } + + v, err := callbacks.NewValidator(cfg) + if err != nil { + panic("creating callback validator: " + err.Error()) + } + return v +} + +func newNexusCallback() *commonpb.Callback { + return &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: "https://nexus.ex.xxxxx.cluster.tmprl.cloud:7243/Namespaces/ex.xxxxx/nexus/callback", + Header: map[string]string{ + "Nexus-Operation-State": "succeeded", + "Content-Type": "application/json", + }, + }, + }, + } +} + +// newWorkerCallback returns a Worker-variant callback whose source context serializes to approximately sourceCtxSize bytes. +func newWorkerCallback(sourceCtxSize int) *commonpb.Callback { + sourceContext := &commonpb.Payload{Data: make([]byte, sourceCtxSize)} + return &commonpb.Callback{ + Variant: &commonpb.Callback_Worker_{Worker: &commonpb.Callback_Worker{ + TaskQueueName: "wc-queue", + Service: "Adapter", + Operation: "Deliver", + SourceContext: sourceContext, + }}, + } +} + +func enableWorkerCallbacks(c *Config) { + c.EnabledCallbackKinds = func(string) []callbacks.Kind { + return []callbacks.Kind{callbacks.KindNexus, callbacks.KindWorker} + } } func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { @@ -56,13 +126,18 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { MaxOperationHeaderSize: func(string) int { return 10 }, DisallowedOperationHeaders: func() []string { return []string{"disallowed-header"} }, MaxOperationScheduleToCloseTimeout: func(string) time.Duration { return time.Hour }, + EnabledCallbackKinds: func(string) []callbacks.Kind { + return []callbacks.Kind{callbacks.KindNexus} + }, } for _, tc := range []struct { - name string - mutate func(*workflowservice.StartNexusOperationExecutionRequest) - errMsg string - check func(*testing.T, *workflowservice.StartNexusOperationExecutionRequest) + name string + mutate func(*workflowservice.StartNexusOperationExecutionRequest) + mutateConfig func(*Config) + wantErr string + // Check the request after validation, to verify situations where it normalizes values. + postValidateCheck func(*testing.T, *workflowservice.StartNexusOperationExecutionRequest) }{ { name: "valid request", @@ -72,21 +147,21 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.OperationId = "" }, - errMsg: "operation_id is required", + wantErr: "operation_id is required", }, { name: "operation_id - exceeds length limit", mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.OperationId = strings.Repeat("x", 51) }, - errMsg: "operation_id exceeds length limit", + wantErr: "operation_id exceeds length limit", }, { name: "request_id - defaults empty to UUID", mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.RequestId = "" }, - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Len(t, r.RequestId, 36) // UUID length }, }, @@ -95,63 +170,63 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.RequestId = strings.Repeat("x", 51) }, - errMsg: "request_id exceeds length limit", + wantErr: "request_id exceeds length limit", }, { name: "identity - exceeds length limit", mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.Identity = strings.Repeat("x", 51) }, - errMsg: "identity exceeds length limit", + wantErr: "identity exceeds length limit", }, { - name: "endpoint - required", - mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.Endpoint = "" }, - errMsg: "endpoint is required", + name: "endpoint - required", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.Endpoint = "" }, + wantErr: "endpoint is required", }, { - name: "service - required", - mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.Service = "" }, - errMsg: "service is required", + name: "service - required", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.Service = "" }, + wantErr: "service is required", }, { name: "service - exceeds length limit", mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.Service = "too-long-svc" }, - errMsg: "service exceeds length limit", + wantErr: "service exceeds length limit", }, { - name: "operation - required", - mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.Operation = "" }, - errMsg: "operation is required", + name: "operation - required", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.Operation = "" }, + wantErr: "operation is required", }, { name: "operation - exceeds length limit", mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.Operation = "too-long-op!" }, - errMsg: "operation exceeds length limit", + wantErr: "operation exceeds length limit", }, { name: "schedule_to_close_timeout - invalid", mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.ScheduleToCloseTimeout = &durationpb.Duration{Seconds: -1} }, - errMsg: "schedule_to_close_timeout is invalid", + wantErr: "schedule_to_close_timeout is invalid", }, { name: "schedule_to_close_timeout - caps exceeding max", mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.ScheduleToCloseTimeout = durationpb.New(2 * time.Hour) }, - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Equal(t, time.Hour, r.ScheduleToCloseTimeout.AsDuration()) }, }, { name: "schedule_to_close_timeout - caps unset to max", - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Equal(t, time.Hour, r.ScheduleToCloseTimeout.AsDuration()) }, }, @@ -160,7 +235,7 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.ScheduleToCloseTimeout = durationpb.New(30 * time.Minute) }, - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Equal(t, 30*time.Minute, r.ScheduleToCloseTimeout.AsDuration()) }, }, @@ -169,14 +244,14 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.ScheduleToStartTimeout = &durationpb.Duration{Seconds: -1} }, - errMsg: "schedule_to_start_timeout is invalid", + wantErr: "schedule_to_start_timeout is invalid", }, { name: "schedule_to_start_timeout - caps to defaulted schedule_to_close_timeout", mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.ScheduleToStartTimeout = durationpb.New(2 * time.Hour) }, - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Equal(t, time.Hour, r.ScheduleToCloseTimeout.AsDuration()) require.Equal(t, time.Hour, r.ScheduleToStartTimeout.AsDuration()) }, @@ -187,7 +262,7 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { r.ScheduleToCloseTimeout = durationpb.New(30 * time.Minute) r.ScheduleToStartTimeout = durationpb.New(time.Hour) }, - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Equal(t, 30*time.Minute, r.ScheduleToStartTimeout.AsDuration()) }, }, @@ -197,7 +272,7 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { r.ScheduleToCloseTimeout = durationpb.New(30 * time.Minute) r.ScheduleToStartTimeout = durationpb.New(20 * time.Minute) }, - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Equal(t, 20*time.Minute, r.ScheduleToStartTimeout.AsDuration()) }, }, @@ -206,14 +281,14 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.StartToCloseTimeout = &durationpb.Duration{Seconds: -1} }, - errMsg: "start_to_close_timeout is invalid", + wantErr: "start_to_close_timeout is invalid", }, { name: "start_to_close_timeout - caps to defaulted schedule_to_close_timeout", mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.StartToCloseTimeout = durationpb.New(2 * time.Hour) }, - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Equal(t, time.Hour, r.ScheduleToCloseTimeout.AsDuration()) require.Equal(t, time.Hour, r.StartToCloseTimeout.AsDuration()) }, @@ -224,7 +299,7 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { r.ScheduleToCloseTimeout = durationpb.New(30 * time.Minute) r.StartToCloseTimeout = durationpb.New(time.Hour) }, - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Equal(t, 30*time.Minute, r.StartToCloseTimeout.AsDuration()) }, }, @@ -234,7 +309,7 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { r.ScheduleToCloseTimeout = durationpb.New(30 * time.Minute) r.StartToCloseTimeout = durationpb.New(10 * time.Minute) }, - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Equal(t, 10*time.Minute, r.StartToCloseTimeout.AsDuration()) }, }, @@ -249,7 +324,7 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.Input = &commonpb.Payload{Data: []byte("this-input-is-longer-than-twenty-characters")} }, - errMsg: "input exceeds size limit", + wantErr: "input exceeds size limit", }, { name: "user_metadata.summary - exceeds size limit", @@ -258,7 +333,7 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { Summary: &commonpb.Payload{Data: []byte("too-long-summary")}, } }, - errMsg: "user_metadata.summary exceeds size limit", + wantErr: "user_metadata.summary exceeds size limit", }, { name: "user_metadata.details - exceeds size limit", @@ -267,25 +342,25 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { Details: &commonpb.Payload{Data: []byte("this-details-payload-is-too-long")}, } }, - errMsg: "user_metadata.details exceeds size limit", + wantErr: "user_metadata.details exceeds size limit", }, { name: "nexus_header - disallowed key", mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.NexusHeader = map[string]string{"Disallowed-Header": "value"} }, - errMsg: "nexus_header contains a disallowed key", + wantErr: "nexus_header contains a disallowed key", }, { name: "nexus_header - exceeds size limit", mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { r.NexusHeader = map[string]string{"key": "too-long-val"} }, - errMsg: "nexus_header exceeds size limit", + wantErr: "nexus_header exceeds size limit", }, { name: "id_policies - defaults unspecified", - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Equal(t, enumspb.NEXUS_OPERATION_ID_REUSE_POLICY_ALLOW_DUPLICATE, r.IdReusePolicy) require.Equal(t, enumspb.NEXUS_OPERATION_ID_CONFLICT_POLICY_FAIL, r.IdConflictPolicy) }, @@ -296,7 +371,7 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { r.IdReusePolicy = enumspb.NEXUS_OPERATION_ID_REUSE_POLICY_REJECT_DUPLICATE r.IdConflictPolicy = enumspb.NEXUS_OPERATION_ID_CONFLICT_POLICY_USE_EXISTING }, - check: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { + postValidateCheck: func(t *testing.T, r *workflowservice.StartNexusOperationExecutionRequest) { require.Equal(t, enumspb.NEXUS_OPERATION_ID_REUSE_POLICY_REJECT_DUPLICATE, r.IdReusePolicy) require.Equal(t, enumspb.NEXUS_OPERATION_ID_CONFLICT_POLICY_USE_EXISTING, r.IdConflictPolicy) }, @@ -312,7 +387,7 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { }, } }, - errMsg: "number of search attributes", + wantErr: "number of search attributes", }, { name: "search_attributes - value exceeds size limit", @@ -326,7 +401,139 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { }, } }, - errMsg: "exceeds size limit", + wantErr: "exceeds size limit", + }, + { + name: "completion_callbacks - accepts the nexus variant", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + r.CompletionCallbacks = []*commonpb.Callback{newNexusCallback()} + }, + }, + { + // The default for enabledCallbackKinds is empty, i.e. the feature is off. + name: "completion_callbacks - rejected when no kinds are enabled", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + r.CompletionCallbacks = []*commonpb.Callback{newNexusCallback()} + }, + mutateConfig: func(c *Config) { + c.EnabledCallbackKinds = func(string) []callbacks.Kind { return nil } + }, + wantErr: "completion callbacks are not enabled for this namespace", + }, + { + name: "completion_callbacks - rejects a kind that is not enabled", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + r.CompletionCallbacks = []*commonpb.Callback{newWorkerCallback(0)} + }, + wantErr: "worker callbacks are not enabled for this execution type", + }, + { + name: "source_context - rejects a single callback over the per-callback limit", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + r.CompletionCallbacks = []*commonpb.Callback{newWorkerCallback(1001)} + }, + mutateConfig: enableWorkerCallbacks, + wantErr: "source_context exceeds size limit", + }, + { + name: "links - accepts a valid link", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + r.Links = []*commonpb.Link{testLink("wf-id")} + }, + }, + { + name: "links - rejects an incomplete variant", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + r.Links = []*commonpb.Link{{Variant: &commonpb.Link_WorkflowEvent_{ + WorkflowEvent: &commonpb.Link_WorkflowEvent{WorkflowId: "wf-id", RunId: "wf-run-id"}, + }}} + }, + wantErr: "must not have an empty namespace", + }, + { + name: "links - rejects more than the per-request limit", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + // newTestLinkValidator below allows 10 links per request. + for i := range 11 { + r.Links = append(r.Links, testLink(fmt.Sprintf("wf-%d", i))) + } + }, + wantErr: "cannot attach more than 10 links per request", + }, + { + // Links ride along on callbacks as well as on the request, and are validated whether or + // not the request brought any of its own. + name: "links - rejects an incomplete variant carried by a callback", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + cb := newNexusCallback() + cb.Links = []*commonpb.Link{{Variant: &commonpb.Link_WorkflowEvent_{ + WorkflowEvent: &commonpb.Link_WorkflowEvent{WorkflowId: "wf-id", RunId: "wf-run-id"}, + }}} + r.CompletionCallbacks = []*commonpb.Callback{cb} + }, + wantErr: "must not have an empty namespace", + }, + { + name: "links - rejects an oversized link carried by a callback", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + // newTestLinkValidator below allows 4000 bytes per link. + cb := newNexusCallback() + cb.Links = []*commonpb.Link{testLink(strings.Repeat("x", 4001))} + r.CompletionCallbacks = []*commonpb.Callback{cb} + }, + wantErr: "link exceeds allowed size of 4000", + }, + { + // A callback's links count toward the same per-request limit as the request's own, so + // neither side can smuggle links past it by splitting them across the two. + name: "links - counts callback links toward the per-request limit", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + cb := newNexusCallback() + for i := range 6 { + r.Links = append(r.Links, testLink(fmt.Sprintf("req-wf-%d", i))) + cb.Links = append(cb.Links, testLink(fmt.Sprintf("cb-wf-%d", i))) + } + r.CompletionCallbacks = []*commonpb.Callback{cb} + }, + wantErr: "cannot attach more than 10 links per request", + }, + { + name: "on_conflict_options - attach_completion_callbacks requires attach_request_id", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + r.CompletionCallbacks = []*commonpb.Callback{newNexusCallback()} + r.OnConflictOptions = &nexusoperationpb.OnConflictOptions{ + AttachCompletionCallbacks: true, + } + }, + wantErr: "attach_completion_callbacks requires attach_request_id", + }, + { + name: "on_conflict_options - attach_request_id requires a completion callback or a link", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + r.OnConflictOptions = &nexusoperationpb.OnConflictOptions{ + AttachRequestId: true, + } + }, + wantErr: "attach_request_id requires at least one completion callback or link", + }, + { + name: "on_conflict_options - attach_request_id is satisfied by links alone", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + r.Links = []*commonpb.Link{testLink("wf-id")} + r.OnConflictOptions = &nexusoperationpb.OnConflictOptions{ + AttachRequestId: true, + AttachLinks: true, + } + }, + }, + { + name: "on_conflict_options - attach_links may be set on its own", + mutate: func(r *workflowservice.StartNexusOperationExecutionRequest) { + r.Links = []*commonpb.Link{testLink("wf-id")} + r.OnConflictOptions = &nexusoperationpb.OnConflictOptions{ + AttachLinks: true, + } + }, }, } { t.Run(tc.name, func(t *testing.T) { @@ -346,22 +553,87 @@ func TestValidateStartNexusOperationExecutionRequest(t *testing.T) { if tc.mutate != nil { tc.mutate(req) } - err := newValidator(config, log.NewNoopLogger(), nil, saValidator). - validateAndNormalizeStartRequest(req) - if tc.errMsg != "" { + caseConfig := *config + if tc.mutateConfig != nil { + tc.mutateConfig(&caseConfig) + } + + cbValidator := mustNewCallbackValidator() + logger := log.NewNoopLogger() + v := newValidator(&caseConfig, logger, nil, saValidator, cbValidator, newTestLinkValidator(10, 10)) + + err := v.validateAndNormalizeStartRequest(context.Background(), req) + if tc.wantErr != "" { var invalidArgErr *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgErr) - require.Contains(t, err.Error(), tc.errMsg) + require.Contains(t, err.Error(), tc.wantErr) } else { require.NoError(t, err) } - if tc.check != nil { - tc.check(t, req) + if tc.postValidateCheck != nil { + tc.postValidateCheck(t, req) } }) } } +// Tests for the the aggregate cap on worker-callback source context payloads. +func TestValidateStartRequestSourceContextAggregate(t *testing.T) { + t.Parallel() + ctx := context.Background() + + newRequest := func(cbs ...*commonpb.Callback) *workflowservice.StartNexusOperationExecutionRequest { + return &workflowservice.StartNexusOperationExecutionRequest{ + Namespace: "ns-name", + OperationId: "op-id", + RequestId: "req-id", + Endpoint: "endpoint", + Service: "service", + Operation: "operation", + CompletionCallbacks: cbs, + } + } + + config := &Config{ + MaxIDLengthLimit: func() int { return 50 }, + MaxServiceNameLength: func(string) int { return 10 }, + MaxOperationNameLength: func(string) int { return 10 }, + PayloadSizeLimit: func(string) int { return 20 }, + PayloadSizeLimitWarn: func(string) int { return 10 }, + MaxUserMetadataSummarySize: func(string) int { return 10 }, + MaxUserMetadataDetailsSize: func(string) int { return 20 }, + MaxOperationHeaderSize: func(string) int { return 10 }, + DisallowedOperationHeaders: func() []string { return nil }, + MaxOperationScheduleToCloseTimeout: func(string) time.Duration { return time.Hour }, + } + enableWorkerCallbacks(config) + v := newTestValidator(config) + + t.Run("AcceptsCallbacksWithinTheAggregateLimit", func(t *testing.T) { + err := v.validateAndNormalizeStartRequest(ctx, newRequest(newWorkerCallback(900))) + require.NoError(t, err) + }) + + t.Run("RejectsCallbacksOverTheAggregateLimitTogether", func(t *testing.T) { + err := v.validateAndNormalizeStartRequest( + ctx, + newRequest(newWorkerCallback(900), newWorkerCallback(900))) + + var failedPreconditionErr *serviceerror.FailedPrecondition + require.ErrorAs(t, err, &failedPreconditionErr) + require.ErrorContains(t, err, "cannot attach more than 1500 bytes of callback source_context") + }) + + t.Run("NexusCallbacksCarryNoSourceContext", func(t *testing.T) { + cbs := []*commonpb.Callback{newWorkerCallback(900)} + for range 5 { + cbs = append(cbs, newNexusCallback()) + } + err := v.validateAndNormalizeStartRequest(ctx, newRequest(cbs...)) + require.NoError(t, err) + }) +} + func TestValidateDescribeNexusOperationExecutionRequest(t *testing.T) { config := &Config{ MaxIDLengthLimit: func() int { return 20 }, @@ -788,3 +1060,99 @@ func TestValidatePollNexusOperationExecutionRequest(t *testing.T) { }) } } + +func TestValidateOnConflictOptions(t *testing.T) { + t.Parallel() + + // on_conflict_options validation is config-independent. + v := newTestValidator(&Config{}) + cb := newNexusCallback() + + t.Run("Unset", func(t *testing.T) { + require.NoError(t, v.validateOnConflictOptions(&workflowservice.StartNexusOperationExecutionRequest{})) + }) + + t.Run("Empty", func(t *testing.T) { + require.NoError(t, v.validateOnConflictOptions(&workflowservice.StartNexusOperationExecutionRequest{ + OnConflictOptions: &nexusoperationpb.OnConflictOptions{}, + })) + }) + + t.Run("AttachRequestIdAndCallbacksWithCallback", func(t *testing.T) { + require.NoError(t, v.validateOnConflictOptions(&workflowservice.StartNexusOperationExecutionRequest{ + CompletionCallbacks: []*commonpb.Callback{cb}, + OnConflictOptions: &nexusoperationpb.OnConflictOptions{ + AttachRequestId: true, + AttachCompletionCallbacks: true, + }, + })) + }) + + t.Run("AttachRequestIdOnlyWithCallback", func(t *testing.T) { + require.NoError(t, v.validateOnConflictOptions(&workflowservice.StartNexusOperationExecutionRequest{ + CompletionCallbacks: []*commonpb.Callback{cb}, + OnConflictOptions: &nexusoperationpb.OnConflictOptions{AttachRequestId: true}, + })) + }) + + t.Run("AttachCallbacksWithoutAttachRequestId", func(t *testing.T) { + err := v.validateOnConflictOptions(&workflowservice.StartNexusOperationExecutionRequest{ + CompletionCallbacks: []*commonpb.Callback{cb}, + OnConflictOptions: &nexusoperationpb.OnConflictOptions{ + AttachCompletionCallbacks: true, + }, + }) + var invalidArgErr *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgErr) + require.Contains(t, err.Error(), "attach_completion_callbacks requires attach_request_id") + }) + + t.Run("AttachRequestIdOnlyWithLink", func(t *testing.T) { + require.NoError(t, v.validateOnConflictOptions(&workflowservice.StartNexusOperationExecutionRequest{ + Links: []*commonpb.Link{testLink("wf-id")}, + OnConflictOptions: &nexusoperationpb.OnConflictOptions{AttachRequestId: true}, + })) + }) + + t.Run("AttachLinksOnly", func(t *testing.T) { + // attach_links stands on its own: links are keyed by the request ID the caller always supplies. + require.NoError(t, v.validateOnConflictOptions(&workflowservice.StartNexusOperationExecutionRequest{ + Links: []*commonpb.Link{testLink("wf-id")}, + OnConflictOptions: &nexusoperationpb.OnConflictOptions{AttachLinks: true}, + })) + }) + + t.Run("AttachLinksAndCallbacks", func(t *testing.T) { + require.NoError(t, v.validateOnConflictOptions(&workflowservice.StartNexusOperationExecutionRequest{ + CompletionCallbacks: []*commonpb.Callback{cb}, + Links: []*commonpb.Link{testLink("wf-id")}, + OnConflictOptions: &nexusoperationpb.OnConflictOptions{ + AttachRequestId: true, + AttachCompletionCallbacks: true, + AttachLinks: true, + }, + })) + }) + + t.Run("AttachRequestIdWithoutCallbackOrLink", func(t *testing.T) { + err := v.validateOnConflictOptions(&workflowservice.StartNexusOperationExecutionRequest{ + OnConflictOptions: &nexusoperationpb.OnConflictOptions{AttachRequestId: true}, + }) + var invalidArgErr *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgErr) + require.Contains(t, err.Error(), "attach_request_id requires at least one completion callback or link") + }) + + t.Run("AttachRequestIdAndCallbacksWithoutCallbackProvided", func(t *testing.T) { + err := v.validateOnConflictOptions(&workflowservice.StartNexusOperationExecutionRequest{ + OnConflictOptions: &nexusoperationpb.OnConflictOptions{ + AttachRequestId: true, + AttachCompletionCallbacks: true, + }, + }) + var invalidArgErr *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgErr) + require.Contains(t, err.Error(), "attach_request_id requires at least one completion callback") + }) + +} diff --git a/chasm/lib/workflow/config.go b/chasm/lib/workflow/config.go index 7c91c1430d3..95989ec4087 100644 --- a/chasm/lib/workflow/config.go +++ b/chasm/lib/workflow/config.go @@ -1,10 +1,18 @@ package workflow import ( + commoncallbacks "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/retrypolicy" ) +var EnabledCallbackKinds = dynamicconfig.NewNamespaceTypedSettingWithConverter( + "workflow.enabledCallbackKinds", + commoncallbacks.ConvertEnabledKinds, + []commoncallbacks.Kind{commoncallbacks.KindNexus}, + `The list of completion callback kinds that may be attached to a workflow execution.`, +) + type Config struct { maxIDLengthLimit dynamicconfig.IntPropertyFn defaultWorkflowRetrySettings dynamicconfig.TypedPropertyFnWithNamespaceFilter[retrypolicy.DefaultRetrySettings] diff --git a/chasm/lib/workflow/workflow.go b/chasm/lib/workflow/workflow.go index 2b5409a4147..a633f0c6017 100644 --- a/chasm/lib/workflow/workflow.go +++ b/chasm/lib/workflow/workflow.go @@ -157,17 +157,9 @@ func addCallbacksToMap( ) error { chasmCBs := make([]*callbackspb.Callback, len(completionCallbacks)) for i, cb := range completionCallbacks { - chasmCB := &callbackspb.Callback{Links: cb.GetLinks()} - switch variant := cb.Variant.(type) { - case *commonpb.Callback_Nexus_: - chasmCB.Variant = &callbackspb.Callback_Nexus_{ - Nexus: &callbackspb.Callback_Nexus{ - Url: variant.Nexus.GetUrl(), - Header: variant.Nexus.GetHeader(), - }, - } - default: - return serviceerror.NewInvalidArgumentf("unsupported callback variant: %T", variant) + chasmCB, err := callback.FromAPICallback(cb) + if err != nil { + return err } chasmCBs[i] = chasmCB } @@ -181,7 +173,7 @@ func addCallbacksToMap( // Already registered, skip to avoid overwriting. continue } - callbackObj := callback.NewCallback(requestID, eventTime, &callbackspb.CallbackState{}, chasmCB) + callbackObj := callback.NewCallback(requestID, eventTime, chasmCB) target[id] = chasm.NewComponentField(ctx, callbackObj) } return nil diff --git a/cmd/tools/getproto/files.go b/cmd/tools/getproto/files.go index 38b6ef67622..4006de0ad41 100644 --- a/cmd/tools/getproto/files.go +++ b/cmd/tools/getproto/files.go @@ -21,6 +21,7 @@ import ( history "go.temporal.io/api/history/v1" namespace "go.temporal.io/api/namespace/v1" nexus "go.temporal.io/api/nexus/v1" + nexusoperation "go.temporal.io/api/nexusoperation/v1" protocol "go.temporal.io/api/protocol/v1" query "go.temporal.io/api/query/v1" replication "go.temporal.io/api/replication/v1" @@ -82,6 +83,7 @@ func init() { importMap["temporal/api/history/v1/message.proto"] = history.File_temporal_api_history_v1_message_proto importMap["temporal/api/namespace/v1/message.proto"] = namespace.File_temporal_api_namespace_v1_message_proto importMap["temporal/api/nexus/v1/message.proto"] = nexus.File_temporal_api_nexus_v1_message_proto + importMap["temporal/api/nexusoperation/v1/message.proto"] = nexusoperation.File_temporal_api_nexusoperation_v1_message_proto importMap["temporal/api/protocol/v1/message.proto"] = protocol.File_temporal_api_protocol_v1_message_proto importMap["temporal/api/query/v1/message.proto"] = query.File_temporal_api_query_v1_message_proto importMap["temporal/api/replication/v1/message.proto"] = replication.File_temporal_api_replication_v1_message_proto diff --git a/common/callbacks/kind.go b/common/callbacks/kind.go new file mode 100644 index 00000000000..af8a5aa88fd --- /dev/null +++ b/common/callbacks/kind.go @@ -0,0 +1,77 @@ +package callbacks + +import ( + "fmt" + "slices" + + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/server/common/dynamicconfig" +) + +// Kind identifies a callback variant. +type Kind string + +const ( + // KindUnknown is the kind of a callback with an unset or unrecognized variant. + KindUnknown Kind = "unknown" + KindNexus Kind = "nexus" + KindWorker Kind = "worker" +) + +func (k Kind) String() string { + switch k { + case KindUnknown, KindNexus, KindWorker: + return string(k) + default: + return string(KindUnknown) + } +} + +// KindOf reports which [Kind] the given callback is. +func KindOf(cb *commonpb.Callback) Kind { + switch cb.GetVariant().(type) { + case *commonpb.Callback_Nexus_: + return KindNexus + case *commonpb.Callback_Worker_: + return KindWorker + case *commonpb.Callback_Internal_: + // Internal-variant callbacks are not used and should be removed entirely. + return KindUnknown + default: + return KindUnknown + } +} + +// ConvertEnabledKinds converts a dynamic config value, a list of kind names into a []Kind. +// +// Returns an error and use the default config value if any callback kinds are unrecognized. +// An empty list not specifying any callback kinds is allowed. +func ConvertEnabledKinds(val any) ([]Kind, error) { + names, err := dynamicconfig.ConvertStructure[[]string](nil)(val) + if err != nil { + return nil, err + } + + enabledKinds := make([]Kind, 0, 2) + configurableKinds := map[string]Kind{ + KindNexus.String(): KindNexus, + KindWorker.String(): KindWorker, + } + var unknownNames []string + for _, name := range names { + kind, ok := configurableKinds[name] + if !ok { + unknownNames = append(unknownNames, name) + continue + } + if !slices.Contains(enabledKinds, kind) { + enabledKinds = append(enabledKinds, kind) + } + } + if len(unknownNames) > 0 { + return nil, fmt.Errorf( + "%v does not match a known callback kind [nexus, worker]", + unknownNames) + } + return enabledKinds, nil +} diff --git a/common/callbacks/kind_test.go b/common/callbacks/kind_test.go new file mode 100644 index 00000000000..48a3065f6b5 --- /dev/null +++ b/common/callbacks/kind_test.go @@ -0,0 +1,130 @@ +package callbacks + +import ( + "testing" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" +) + +func TestKindOf(t *testing.T) { + for _, tc := range []struct { + callback *commonpb.Callback + want Kind + wantName string + }{ + { + callback: newNexusCallback(), + want: KindNexus, + wantName: "nexus", + }, + { + callback: newWorkerCallback(), + want: KindWorker, + wantName: "worker", + }, + { + // Internal-variant callbacks should be removed; treated as Unknown. + callback: &commonpb.Callback{ + Variant: &commonpb.Callback_Internal_{ + Internal: &commonpb.Callback_Internal{}, + }, + }, + want: KindUnknown, + wantName: "unknown", + }, + { + callback: &commonpb.Callback{}, + want: KindUnknown, + wantName: "unknown", + }, + { + callback: nil, + want: KindUnknown, + wantName: "unknown", + }, + } { + t.Run(tc.wantName, func(t *testing.T) { + require.Equal(t, tc.want, KindOf(tc.callback)) + require.Equal(t, tc.wantName, tc.want.String()) + }) + } +} + +func TestConvertEnabledKinds(t *testing.T) { + for _, tc := range []struct { + name string + val any + want []Kind + }{ + // Empty lists are supported. It means no callbacks are allowed on the execution. + {name: "Empty", val: []string{}, want: []Kind{}}, + {name: "Nil", val: nil, want: []Kind{}}, + + {name: "NexusOnly", val: []string{"nexus"}, want: []Kind{KindNexus}}, + {name: "WorkerOnly", val: []string{"worker"}, want: []Kind{KindWorker}}, + { + name: "Both", + val: []string{"nexus", "worker"}, + want: []Kind{KindNexus, KindWorker}, + }, + { + name: "OrderPreserved", + val: []string{"worker", "nexus"}, + want: []Kind{KindWorker, KindNexus}, + }, + { + name: "DuplicatesDropped", + val: []string{"nexus", "nexus"}, + want: []Kind{KindNexus}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := ConvertEnabledKinds(tc.val) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } + + // Return an error for any unrecognized name, rejecting the whole value. So dynamic config + // parsing will log and fall back to the setting's default value. + for _, tc := range []struct { + name string + val any + errMsg string + }{ + {name: "NotAList", val: 42, errMsg: "source data must be an array or slice"}, + { + name: "OnlyUnknownNames", + val: []string{"carrier-pigeon"}, + errMsg: `[carrier-pigeon] does not match a known callback kind`, + }, + { + name: "UnknownNameBesideKnownName", + val: []string{"nexsus", "worker"}, + errMsg: `[nexsus] does not match a known callback kind`, + }, + { + // Internal callbacks are server-generated and cannot be enabled by an operator. + name: "Internal", + val: []string{"nexus", "internal"}, + errMsg: `[internal] does not match a known callback kind`, + }, + { + name: "NotNormalizedOrTrimmed", + val: []any{" Nexus", "WORKER", " nexus", "worker "}, + errMsg: "[ Nexus WORKER nexus worker ] does not match a known callback kind", + }, + { + name: "AllUnknownNamesReported", + val: []string{"nexsus", "wroker"}, + errMsg: `[nexsus wroker] does not match a known callback kind`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := ConvertEnabledKinds(tc.val) + require.ErrorContains(t, err, tc.errMsg) + require.Nil(t, got) + }) + } +} diff --git a/common/callbacks/validator.go b/common/callbacks/validator.go index e55c9c0c0fd..3c8c8f00df0 100644 --- a/common/callbacks/validator.go +++ b/common/callbacks/validator.go @@ -3,27 +3,60 @@ package callbacks import ( "context" "fmt" + "slices" "strings" commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/tqid" "google.golang.org/grpc/status" ) +type ValidatorOptions struct { + // EnabledKinds are the callback kinds that may be attached to the execution being validated. + // A client-supplied callback of any other kind is rejected with an InvalidArgument error. + EnabledKinds []Kind +} + // Validator validates completion callbacks attached to executions (e.g. workflows and standalone activities). type Validator interface { - Validate(ctx context.Context, namespaceName string, cbs []*commonpb.Callback) error + // Validate rejects callbacks that are not enabled for the execution, or are malformed. + // Will mutate the supplied Callbacks to normalize. e.g. converting Nexus headers to lower-case. + Validate(ctx context.Context, namespaceName string, cbs []*commonpb.Callback, opts ValidatorOptions) error + + // ValidateTotalSourceContextSize checks that adding addingBytes of Worker source context to an + // execution already carrying existingBytes will not exceed the per-execution limit. + ValidateTotalSourceContextSize(namespaceName string, existingBytes, addingBytes int) error +} + +// SourceContextSize returns the total size in bytes of the Worker source context payloads carried +// by cbs. Callbacks of any other kind contribute nothing. +func SourceContextSize(cbs []*commonpb.Callback) int { + total := 0 + for _, cb := range cbs { + if sc := cb.GetWorker().GetSourceContext(); sc != nil { + total += sc.Size() + } + } + return total } // ValidatorConfig holds the limits a [Validator] enforces. type ValidatorConfig struct { MaxCallbacksPerExecution dynamicconfig.IntPropertyFnWithNamespaceFilter + MaxIDLengthLimit dynamicconfig.IntPropertyFn // All ID types use the same global setting. // Nexus-variant limits. URLMaxLength dynamicconfig.IntPropertyFnWithNamespaceFilter HeaderMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter EndpointRules dynamicconfig.TypedPropertyFnWithNamespaceFilter[AddressMatchRules] + + // Worker-variant limits. + MaxServiceNameLength dynamicconfig.IntPropertyFnWithNamespaceFilter + MaxOperationNameLength dynamicconfig.IntPropertyFnWithNamespaceFilter + WorkerSourceContextMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter + WorkerSourceContextAggregateMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter } func (vc *ValidatorConfig) Validate() error { @@ -31,6 +64,9 @@ func (vc *ValidatorConfig) Validate() error { if vc.MaxCallbacksPerExecution == nil { missingFields = append(missingFields, "MaxCallbacksPerExecution") } + if vc.MaxIDLengthLimit == nil { + missingFields = append(missingFields, "MaxIDLengthLimit") + } if vc.URLMaxLength == nil { missingFields = append(missingFields, "URLMaxLength") } @@ -40,6 +76,18 @@ func (vc *ValidatorConfig) Validate() error { if vc.EndpointRules == nil { missingFields = append(missingFields, "EndpointRules") } + if vc.MaxServiceNameLength == nil { + missingFields = append(missingFields, "MaxServiceNameLength") + } + if vc.MaxOperationNameLength == nil { + missingFields = append(missingFields, "MaxOperationNameLength") + } + if vc.WorkerSourceContextMaxSize == nil { + missingFields = append(missingFields, "WorkerSourceContextMaxSize") + } + if vc.WorkerSourceContextAggregateMaxSize == nil { + missingFields = append(missingFields, "WorkerSourceContextAggregateMaxSize") + } if len(missingFields) != 0 { return fmt.Errorf("missing required fields: %v", missingFields) @@ -59,48 +107,124 @@ func NewValidator(config ValidatorConfig) (Validator, error) { return &validator{config: config}, nil } -// Validate validates completion callbacks: count, URL length, endpoint allowlist, header size, and normalizes header -// keys to lowercase. -func (v *validator) Validate(_ context.Context, namespaceName string, cbs []*commonpb.Callback) error { +// Validate validates completion callbacks: their kind, their count, and the fields of each variant. +// Nexus header keys are normalized to lowercase in place. +func (v *validator) Validate( + _ context.Context, + namespaceName string, + cbs []*commonpb.Callback, + opts ValidatorOptions, +) error { if len(cbs) > v.config.MaxCallbacksPerExecution(namespaceName) { return serviceerror.NewInvalidArgumentf( "cannot attach more than %d callbacks to an execution", v.config.MaxCallbacksPerExecution(namespaceName), ) } - for _, cb := range cbs { - switch variant := cb.GetVariant().(type) { - case *commonpb.Callback_Nexus_: - rawURL := variant.Nexus.GetUrl() - if len(rawURL) > v.config.URLMaxLength(namespaceName) { - return serviceerror.NewInvalidArgumentf( - "invalid url: url length longer than max length allowed of %d", v.config.URLMaxLength(namespaceName), - ) - } - if err := v.config.EndpointRules(namespaceName).Validate(rawURL); err != nil { - if s, ok := status.FromError(err); ok { - return serviceerror.NewInvalidArgument(s.Message()) - } - return serviceerror.NewInvalidArgument(err.Error()) - } - - headerSize := 0 - lowerCaseHeaders := make(map[string]string, len(variant.Nexus.GetHeader())) - for k, val := range variant.Nexus.GetHeader() { - headerSize += len(k) + len(val) - lowerCaseHeaders[strings.ToLower(k)] = val - } - if headerSize > v.config.HeaderMaxSize(namespaceName) { - return serviceerror.NewInvalidArgumentf( - "invalid header: header size longer than max allowed size of %d", v.config.HeaderMaxSize(namespaceName), - ) - } - variant.Nexus.Header = lowerCaseHeaders - case *commonpb.Callback_Internal_: - continue - default: - return serviceerror.NewUnimplemented(fmt.Sprintf("unknown callback variant: %T", variant)) + for i, cb := range cbs { + if err := v.validateCallback(cb, namespaceName, opts); err != nil { + return fmt.Errorf("completion_callbacks[%d]: %w", i, err) + } + } + return nil +} + +func (v *validator) validateCallback(cb *commonpb.Callback, namespaceName string, opts ValidatorOptions) error { + kind := KindOf(cb) + + // For unknown callbacks, prefer the "unknown callback variant" error below. + if kind != KindUnknown && !slices.Contains(opts.EnabledKinds, kind) { + return serviceerror.NewInvalidArgumentf("%s callbacks are not enabled for this execution type", kind) + } + + switch kind { + case KindNexus: + return v.validateNexus(namespaceName, cb.GetNexus()) + case KindWorker: + return v.validateWorker(namespaceName, cb.GetWorker()) + case KindUnknown: + fallthrough + default: + return serviceerror.NewUnimplementedf("unknown callback variant: %T", cb.GetVariant()) + } +} + +func (v *validator) validateNexus(namespaceName string, cb *commonpb.Callback_Nexus) error { + rawURL := cb.GetUrl() + if len(rawURL) > v.config.URLMaxLength(namespaceName) { + return serviceerror.NewInvalidArgumentf( + "invalid url: url length longer than max length allowed of %d", + v.config.URLMaxLength(namespaceName), + ) + } + if err := v.config.EndpointRules(namespaceName).Validate(rawURL); err != nil { + msg := err.Error() + if s, ok := status.FromError(err); ok { + msg = s.Message() } + return serviceerror.NewInvalidArgument(msg) + } + + // Validate total size of all headers, as well as normalize to lowercase. + headerSize := 0 + lowerCaseHeaders := make(map[string]string, len(cb.GetHeader())) + for k, val := range cb.GetHeader() { + headerSize += len(k) + len(val) + lowerCaseHeaders[strings.ToLower(k)] = val + } + if headerSize > v.config.HeaderMaxSize(namespaceName) { + return serviceerror.NewInvalidArgumentf( + "invalid header: header size longer than max allowed size of %d", + v.config.HeaderMaxSize(namespaceName), + ) + } + cb.Header = lowerCaseHeaders + return nil +} + +func (v *validator) validateWorker(namespaceName string, cb *commonpb.Callback_Worker) error { + // Task Queue + if err := tqid.Validate(cb.GetTaskQueueName(), v.config.MaxIDLengthLimit()); err != nil { + return err + } + + // Nexus handler + for _, field := range []struct { + name string + value string + maxLength int + }{ + {"service", cb.GetService(), v.config.MaxServiceNameLength(namespaceName)}, + {"operation", cb.GetOperation(), v.config.MaxOperationNameLength(namespaceName)}, + } { + if field.value == "" { + return serviceerror.NewInvalidArgumentf("%s is required", field.name) + } + if len(field.value) > field.maxLength { + return serviceerror.NewInvalidArgumentf( + "%s exceeds length limit. Length=%d Limit=%d", + field.name, len(field.value), field.maxLength) + } + } + + // Source Context blob + maxSize := v.config.WorkerSourceContextMaxSize(namespaceName) + if size := cb.GetSourceContext().Size(); size > maxSize { + return serviceerror.NewInvalidArgumentf( + "source_context exceeds size limit. Length=%d Limit=%d", + size, v.config.WorkerSourceContextMaxSize(namespaceName)) + } + + return nil +} + +func (v *validator) ValidateTotalSourceContextSize(namespaceName string, existingBytes, addingBytes int) error { + maxSize := v.config.WorkerSourceContextAggregateMaxSize(namespaceName) + if existingBytes+addingBytes > maxSize { + return serviceerror.NewFailedPreconditionf( + "cannot attach more than %d bytes of callback source_context to an execution "+ + "(%d bytes already attached, %d more requested)", + maxSize, existingBytes, addingBytes) } return nil } diff --git a/common/callbacks/validator_test.go b/common/callbacks/validator_test.go index a27f19830f4..ab9fd31989c 100644 --- a/common/callbacks/validator_test.go +++ b/common/callbacks/validator_test.go @@ -2,7 +2,9 @@ package callbacks import ( "context" + "reflect" "regexp" + "strings" "testing" "github.com/stretchr/testify/require" @@ -10,6 +12,52 @@ import ( "go.temporal.io/api/serviceerror" ) +func newNexusCallback() *commonpb.Callback { + return &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: "https://nexus.ex.xxxxx.cluster.tmprl.cloud:7243/Namespaces/ex.xxxxx/nexus/callback", + Header: map[string]string{ + "Nexus-Operation-State": "succeeded", + "Content-Type": "application/json", + }, + }, + }, + } +} + +func newWorkerCallback() *commonpb.Callback { + return &commonpb.Callback{ + Variant: &commonpb.Callback_Worker_{ + Worker: &commonpb.Callback_Worker{ + TaskQueueName: "wc-queue", + Service: "CompletionService", + Operation: "DeliverAsWebhook", + SourceContext: &commonpb.Payload{Data: []byte("data")}, + }, + }, + } +} + +func newValidatorConfig() ValidatorConfig { + allowAllAddresses := AddressMatchRules{ + Rules: []AddressMatchRule{ + {Regexp: regexp.MustCompile(`.*`), AllowInsecure: true}, + }, + } + return ValidatorConfig{ + MaxCallbacksPerExecution: func(string) int { return 10 }, + MaxIDLengthLimit: func() int { return 10 }, + URLMaxLength: func(string) int { return 1000 }, + HeaderMaxSize: func(string) int { return 4096 }, + EndpointRules: func(string) AddressMatchRules { return allowAllAddresses }, + MaxServiceNameLength: func(string) int { return 40 }, + MaxOperationNameLength: func(string) int { return 40 }, + WorkerSourceContextMaxSize: func(string) int { return 1000 }, + WorkerSourceContextAggregateMaxSize: func(string) int { return 4000 }, + } +} + func mustNewValidator(t *testing.T, cfg ValidatorConfig) Validator { t.Helper() v, err := NewValidator(cfg) @@ -18,111 +66,130 @@ func mustNewValidator(t *testing.T, cfg ValidatorConfig) Validator { } func TestValidatorConfigValidate(t *testing.T) { - cfg := ValidatorConfig{ - MaxCallbacksPerExecution: func(string) int { return 10 }, - HeaderMaxSize: func(string) int { return 4096 }, - } + cfg := newValidatorConfig() + cfg.URLMaxLength = nil + cfg.EndpointRules = nil _, err := NewValidator(cfg) require.EqualError(t, err, "missing required fields: [URLMaxLength EndpointRules]") } +// Catch when a new field is added to ValidatorConfig but not checked in Validate(). +func TestValidatorConfigValidateNamesEveryField(t *testing.T) { + _, err := NewValidator(ValidatorConfig{}) + require.Error(t, err) + + for field := range reflect.TypeFor[ValidatorConfig]().Fields() { + require.Containsf(t, err.Error(), field.Name, + "ValidatorConfig.%s is not checked by Validate", field.Name) + } +} + func TestValidateCallbacks(t *testing.T) { ctx := context.Background() - allowAllAddresses := AddressMatchRules{ - Rules: []AddressMatchRule{ - {Regexp: regexp.MustCompile(`.*`), AllowInsecure: true}, - }, + opts := ValidatorOptions{ + EnabledKinds: []Kind{KindNexus, KindWorker}, } - getStandardConfig := func() ValidatorConfig { - return ValidatorConfig{ - MaxCallbacksPerExecution: func(string) int { return 10 }, - URLMaxLength: func(string) int { return 1000 }, - HeaderMaxSize: func(string) int { return 4096 }, - EndpointRules: func(string) AddressMatchRules { return allowAllAddresses }, - } - } - v := mustNewValidator(t, getStandardConfig()) + v := mustNewValidator(t, newValidatorConfig()) + + t.Run("EmptyCallbacksNoError", func(t *testing.T) { + err := v.Validate(ctx, "ns", nil, opts) + require.NoError(t, err) + }) t.Run("ValidNexusCallback", func(t *testing.T) { cbs := []*commonpb.Callback{ - {Variant: &commonpb.Callback_Nexus_{ - Nexus: &commonpb.Callback_Nexus{ - Url: "http://localhost:8080/callback", - Header: map[string]string{"Content-Type": "application/json"}, - }, - }}, + newNexusCallback(), } - err := v.Validate(ctx, "ns", cbs) + err := v.Validate(ctx, "ns", cbs, opts) require.NoError(t, err) }) + t.Run("ValidWorkerCallback", func(t *testing.T) { + cbs := []*commonpb.Callback{ + newWorkerCallback(), + } + require.NoError(t, v.Validate(ctx, "ns", cbs, opts)) + }) + + t.Run("InternalCallbacksFail", func(t *testing.T) { + cbs := []*commonpb.Callback{ + { + Variant: &commonpb.Callback_Internal_{ + Internal: &commonpb.Callback_Internal{}, + }, + }, + } + + err := v.Validate(ctx, "ns", cbs, opts) + require.Error(t, err) + require.ErrorContains(t, err, "unknown callback variant") + }) + t.Run("TooManyCallbacks", func(t *testing.T) { cbs := []*commonpb.Callback{ - {Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{Url: "http://localhost/cb1"}}}, - {Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{Url: "http://localhost/cb2"}}}, + newNexusCallback(), + newNexusCallback(), } - cfg := getStandardConfig() + cfg := newValidatorConfig() cfg.MaxCallbacksPerExecution = func(string) int { return 1 } v := mustNewValidator(t, cfg) - err := v.Validate(ctx, "ns", cbs) + err := v.Validate(ctx, "ns", cbs, opts) var invalidArgErr *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgErr) - require.Contains(t, err.Error(), "cannot attach more than 1 callbacks") + require.ErrorContains(t, err, "cannot attach more than 1 callbacks") }) t.Run("URLTooLong", func(t *testing.T) { + nexusCb := newNexusCallback() + nexusCb.GetNexus().Url = "http://localhost/" + string(make([]byte, 51)) cbs := []*commonpb.Callback{ - {Variant: &commonpb.Callback_Nexus_{ - Nexus: &commonpb.Callback_Nexus{ - Url: "http://localhost/" + string(make([]byte, 51)), - }, - }}, + newNexusCallback(), + nexusCb, } - cfg := getStandardConfig() + cfg := newValidatorConfig() cfg.URLMaxLength = func(string) int { return 50 } v := mustNewValidator(t, cfg) - err := v.Validate(ctx, "ns", cbs) + err := v.Validate(ctx, "ns", cbs, opts) var invalidArgErr *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgErr) - require.Contains(t, err.Error(), "url length longer than max length allowed") + require.Error(t, err, "invalid url: url length longer than max length allowed of 50") }) t.Run("HeaderTooLarge", func(t *testing.T) { + nexusCb := newNexusCallback() + nexusCb.GetNexus().Header = map[string]string{"X-Large": string(make([]byte, 5000))} cbs := []*commonpb.Callback{ - {Variant: &commonpb.Callback_Nexus_{ - Nexus: &commonpb.Callback_Nexus{ - Url: "http://localhost:8080/callback", - Header: map[string]string{"X-Large": string(make([]byte, 5000))}, - }, - }}, + nexusCb, } - err := v.Validate(ctx, "ns", cbs) + err := v.Validate(ctx, "ns", cbs, opts) var invalidArgErr *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgErr) - require.Contains(t, err.Error(), "header size longer than max allowed size") + require.ErrorContains(t, err, "invalid header: header size longer than max allowed size of 4096") }) t.Run("HeaderKeysNormalizedToLowercase", func(t *testing.T) { + nexusCb := newNexusCallback() + nexusCb.GetNexus().Header = map[string]string{ + "Content-Type": "application/json", + "X-Custom": "value", + } cbs := []*commonpb.Callback{ - {Variant: &commonpb.Callback_Nexus_{ - Nexus: &commonpb.Callback_Nexus{ - Url: "http://localhost:8080/callback", - Header: map[string]string{"Content-Type": "application/json", "X-Custom": "value"}, - }, - }}, + nexusCb, } - err := v.Validate(ctx, "ns", cbs) + err := v.Validate(ctx, "ns", cbs, opts) require.NoError(t, err) - nexus := cbs[0].GetNexus() + + // Mutation is in-place. + nexus := nexusCb.GetNexus() require.Equal(t, "application/json", nexus.Header["content-type"]) require.Equal(t, "value", nexus.Header["x-custom"]) _, hasMixed := nexus.Header["Content-Type"] @@ -131,47 +198,158 @@ func TestValidateCallbacks(t *testing.T) { t.Run("URLNotInAllowlist", func(t *testing.T) { cbs := []*commonpb.Callback{ - {Variant: &commonpb.Callback_Nexus_{ - Nexus: &commonpb.Callback_Nexus{ - Url: "http://localhost:8080/callback", - }, - }}, + newNexusCallback(), } - cfg := getStandardConfig() - cfg.EndpointRules = func(string) AddressMatchRules { return AddressMatchRules{} } + cfg := newValidatorConfig() + cfg.EndpointRules = func(string) AddressMatchRules { + // No rules in the allow list. + return AddressMatchRules{} + } v := mustNewValidator(t, cfg) - err := v.Validate(ctx, "ns", cbs) + err := v.Validate(ctx, "ns", cbs, opts) var invalidArgErr *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgErr) - require.Contains(t, err.Error(), "does not match any configured callback address") + require.ErrorContains(t, err, "does not match any configured callback address") }) t.Run("UnsupportedVariant", func(t *testing.T) { cbs := []*commonpb.Callback{ - {Variant: nil}, + { + Variant: nil, + }, } - err := v.Validate(ctx, "ns", cbs) + err := v.Validate(ctx, "ns", cbs, opts) var unimplementedErr *serviceerror.Unimplemented require.ErrorAs(t, err, &unimplementedErr) require.Contains(t, err.Error(), "unknown callback variant") }) +} - t.Run("EmptyCallbacksNoError", func(t *testing.T) { - err := v.Validate(ctx, "ns", nil) - require.NoError(t, err) +func TestValidateWorkerCallback(t *testing.T) { + ctx := context.Background() + + cfg := newValidatorConfig() + v := mustNewValidator(t, cfg) + opts := ValidatorOptions{ + EnabledKinds: []Kind{KindNexus, KindWorker}, + } + + for _, tc := range []struct { + name string + mutate func(*commonpb.Callback_Worker) + errMsg string + }{ + { + name: "task_queue is not set", + mutate: func(w *commonpb.Callback_Worker) { w.TaskQueueName = "" }, + errMsg: "taskQueue is not set", + }, + { + name: "task_queue length exceeds limit", + mutate: func(w *commonpb.Callback_Worker) { w.TaskQueueName = strings.Repeat("x", 11) }, + errMsg: "taskQueue length exceeds limit", + }, + { + name: "task_queue uses reserved prefix", + mutate: func(w *commonpb.Callback_Worker) { w.TaskQueueName = "/_sys/tq" }, + errMsg: "task queue name cannot start with reserved prefix /_sys/", + }, + { + name: "service is required", + mutate: func(w *commonpb.Callback_Worker) { w.Service = "" }, + errMsg: "service is required", + }, + { + name: "service length", + mutate: func(w *commonpb.Callback_Worker) { w.Service = strings.Repeat("x", 41) }, + errMsg: "service exceeds length limit", + }, + { + name: "operation is required", + mutate: func(w *commonpb.Callback_Worker) { w.Operation = "" }, + errMsg: "operation is required", + }, + { + name: "operation length", + mutate: func(w *commonpb.Callback_Worker) { w.Operation = strings.Repeat("x", 41) }, + errMsg: "operation exceeds length limit", + }, + { + name: "source_context size", + mutate: func(w *commonpb.Callback_Worker) { + w.SourceContext = &commonpb.Payload{Data: []byte(strings.Repeat("x", 1001))} + }, + errMsg: "source_context exceeds size limit", + }, + } { + t.Run(tc.name, func(t *testing.T) { + cb := newWorkerCallback() + tc.mutate(cb.GetWorker()) + + cbs := []*commonpb.Callback{ + cb, + } + + err := v.Validate(ctx, "ns", cbs, opts) + var invalidArgErr *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgErr) + require.ErrorContains(t, err, tc.errMsg) + }) + } +} + +func TestValidateEnabledKinds(t *testing.T) { + ctx := context.Background() + v := mustNewValidator(t, newValidatorConfig()) + nexusCb := newNexusCallback() + workerCb := newWorkerCallback() + + allowAllKindsOpts := ValidatorOptions{ + EnabledKinds: []Kind{KindNexus, KindWorker}, + } + nexusOnlyOpts := ValidatorOptions{ + EnabledKinds: []Kind{KindNexus}, + } + + t.Run("NoCallbacks", func(t *testing.T) { + require.NoError(t, v.Validate(ctx, "ns", nil, nexusOnlyOpts)) }) - t.Run("InternalCallbackSkipped", func(t *testing.T) { - cbs := []*commonpb.Callback{ - {Variant: &commonpb.Callback_Internal_{ - Internal: &commonpb.Callback_Internal{}, - }}, - } + t.Run("AllSupported", func(t *testing.T) { + require.NoError(t, v.Validate(ctx, "ns", + []*commonpb.Callback{nexusCb, workerCb}, + allowAllKindsOpts, + )) + }) - err := v.Validate(ctx, "ns", cbs) - require.NoError(t, err) + t.Run("NoKindsEnabled", func(t *testing.T) { + // The zero value supports no client-supplied kinds at all. + err := v.Validate(ctx, "ns", []*commonpb.Callback{nexusCb}, ValidatorOptions{}) + var invalidArgErr *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgErr) + require.ErrorContains(t, err, "nexus callbacks are not enabled for this execution type") + }) + + t.Run("DisabledKind", func(t *testing.T) { + err := v.Validate(ctx, "ns", + []*commonpb.Callback{nexusCb, workerCb}, + ValidatorOptions{EnabledKinds: []Kind{KindNexus}}, + ) + var invalidArgErr *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgErr) + require.ErrorContains(t, err, "worker callbacks are not enabled for this execution type") + }) + + t.Run("CheckEnabledBeforeValidation", func(t *testing.T) { + invalidWorkerCb := newWorkerCallback() + invalidWorkerCb.GetWorker().TaskQueueName = "" + err := v.Validate(ctx, "ns", []*commonpb.Callback{invalidWorkerCb}, nexusOnlyOpts) + + var invalidArgErr *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgErr) + require.ErrorContains(t, err, "completion_callbacks[0]: worker callbacks are not enabled for this execution type") }) } diff --git a/common/links/validator.go b/common/links/validator.go index b6499102c66..219d60cf591 100644 --- a/common/links/validator.go +++ b/common/links/validator.go @@ -62,6 +62,26 @@ func validateFields(l *commonpb.Link) error { if t.BatchJob.GetJobId() == "" { return serviceerror.NewInvalidArgument("batch job link must not have an empty job ID") } + case *commonpb.Link_Callback_: + if t.Callback.GetExecution() == nil { + return serviceerror.NewInvalidArgument("callback link must have an execution") + } + exType := t.Callback.GetExecution().GetType() + if exType == enumspb.EXECUTION_TYPE_UNSPECIFIED { + return serviceerror.NewInvalidArgument("callback link execution must have a type") + } + if _, ok := enumspb.ExecutionType_name[int32(exType)]; !ok { + return serviceerror.NewInvalidArgument("callback link execution type is unknown") + } + if t.Callback.GetExecution().GetBusinessId() == "" { + return serviceerror.NewInvalidArgument("callback link execution must have a business ID") + } + if t.Callback.GetExecution().GetRunId() == "" { + return serviceerror.NewInvalidArgument("callback link execution must have a run ID") + } + if t.Callback.GetRequestId() == "" { + return serviceerror.NewInvalidArgument("callback link must have a request ID") + } case *commonpb.Link_NexusOperation_: if t.NexusOperation.GetNamespace() == "" { return serviceerror.NewInvalidArgument("nexus operation link must not have an empty namespace field") diff --git a/common/links/validator_test.go b/common/links/validator_test.go index ef64cae5eac..dbac60bf6f0 100644 --- a/common/links/validator_test.go +++ b/common/links/validator_test.go @@ -35,6 +35,18 @@ func TestValidate(t *testing.T) { BatchJob: &commonpb.Link_BatchJob{JobId: "job"}, }, } + validCallback := &commonpb.Link{ + Variant: &commonpb.Link_Callback_{ + Callback: &commonpb.Link_Callback{ + Execution: &commonpb.Execution{ + Type: enumspb.EXECUTION_TYPE_NEXUS_OPERATION, + BusinessId: "op", + RunId: "run", + }, + RequestId: "req", + }, + }, + } validNexusOperation := &commonpb.Link{ Variant: &commonpb.Link_NexusOperation_{ NexusOperation: &commonpb.Link_NexusOperation{ @@ -67,10 +79,11 @@ func TestValidate(t *testing.T) { err := links.Validate([]*commonpb.Link{ validWorkflowEvent, validBatchJob, + validCallback, validNexusOperation, validActivity, validWorkflow, - }, maxLinks+2, maxSize) + }, maxLinks+3, maxSize) require.NoError(t, err) }) @@ -120,6 +133,20 @@ func TestValidate(t *testing.T) { require.ErrorContains(t, err, "batch job link must not have an empty job ID") }) + t.Run("Callback/EmptyExecution", func(t *testing.T) { + l := proto.Clone(validCallback).(*commonpb.Link) + l.GetCallback().Execution = nil + err := links.Validate([]*commonpb.Link{l}, maxLinks, maxSize) + require.ErrorContains(t, err, "callback link must have an execution") + }) + + t.Run("Callback/RequestID", func(t *testing.T) { + l := proto.Clone(validCallback).(*commonpb.Link) + l.GetCallback().RequestId = "" + err := links.Validate([]*commonpb.Link{l}, maxLinks, maxSize) + require.ErrorContains(t, err, "callback link must have a request ID") + }) + t.Run("NexusOperation/EmptyNamespace", func(t *testing.T) { l := proto.Clone(validNexusOperation).(*commonpb.Link) l.GetNexusOperation().Namespace = "" diff --git a/common/nexus/dispatch_response.go b/common/nexus/dispatch_response.go index f5218f89c08..c04af18b4a1 100644 --- a/common/nexus/dispatch_response.go +++ b/common/nexus/dispatch_response.go @@ -2,6 +2,7 @@ package nexus import ( "github.com/nexus-rpc/sdk-go/nexus" + enumspb "go.temporal.io/api/enums/v1" nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/sdk/temporal" "go.temporal.io/server/api/matchingservice/v1" @@ -19,6 +20,9 @@ func MatchingDispatchResponseToError(resp *matchingservice.DispatchNexusTaskResp case *matchingservice.DispatchNexusTaskResponse_Failure: // Worker received the task and explicitly failed it (via RespondNexusTaskFailed). return temporal.GetDefaultFailureConverter().FailureToError(t.Failure) + case *matchingservice.DispatchNexusTaskResponse_HandlerError: + //nolint:staticcheck // Deprecated, still sent by older workers. + return protoHandlerErrorToError(t.HandlerError) case *matchingservice.DispatchNexusTaskResponse_RequestTimeout: return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUpstreamTimeout, "upstream timeout") case *matchingservice.DispatchNexusTaskResponse_Response: @@ -39,7 +43,41 @@ func StartOperationResponseToError(resp *nexuspb.StartOperationResponse) error { case *nexuspb.StartOperationResponse_Failure: // Operation processed but failed — the worker returned an explicit failure. return temporal.GetDefaultFailureConverter().FailureToError(t.Failure) + case *nexuspb.StartOperationResponse_OperationError: //nolint:staticcheck // Deprecated, still sent by older workers. + //nolint:staticcheck // Deprecated fields on a deprecated variant. + cause := ProtoFailureToNexusFailure(t.OperationError.GetFailure()) + return &nexus.OperationError{ + //nolint:staticcheck // Deprecated fields on a deprecated variant. + State: nexus.OperationState(t.OperationError.GetOperationState()), + // OperationError.Error() does not include the cause, so carry the worker's message here + // too. Otherwise callers that record err.Error() lose it. + Message: cause.Message, + Cause: &nexus.FailureError{Failure: cause}, + } default: return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "empty or unknown start operation response variant") } } + +// protoHandlerErrorToError converts the deprecated HandlerError outcome into a nexus.HandlerError, so that +// the worker's error type and retry behavior are preserved rather than collapsed into an internal error. +func protoHandlerErrorToError(handlerErr *nexuspb.HandlerError) error { + var retryBehavior nexus.HandlerErrorRetryBehavior + //nolint:exhaustive // Unspecified defers to the error type's default. + switch handlerErr.GetRetryBehavior() { + case enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_RETRYABLE: + retryBehavior = nexus.HandlerErrorRetryBehaviorRetryable + case enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_NON_RETRYABLE: + retryBehavior = nexus.HandlerErrorRetryBehaviorNonRetryable + } + //nolint:staticcheck // Deprecated function still in use for backward compatibility. + cause := ProtoFailureToNexusFailure(handlerErr.GetFailure()) + return &nexus.HandlerError{ + Type: nexus.HandlerErrorType(handlerErr.GetErrorType()), + RetryBehavior: retryBehavior, + // HandlerError.Error() does not include the cause, so carry the worker's message here too. + // Otherwise callers that record err.Error() lose it. + Message: cause.Message, + Cause: &nexus.FailureError{Failure: cause}, + } +} diff --git a/common/nexus/dispatch_response_test.go b/common/nexus/dispatch_response_test.go index d73daa14118..29e0c72406c 100644 --- a/common/nexus/dispatch_response_test.go +++ b/common/nexus/dispatch_response_test.go @@ -5,6 +5,7 @@ import ( "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/sdk/temporal" @@ -139,6 +140,83 @@ func TestMatchingDispatchResponseToError_OperationFailure_CanceledError(t *testi require.ErrorAs(t, err, &cancelErr) } +// Older workers report a handler error with the deprecated HandlerError outcome. Its type and retry +// behavior decide whether the caller retries, so both have to survive the conversion. +func TestMatchingDispatchResponseToError_DeprecatedHandlerError(t *testing.T) { + for _, tc := range []struct { + name string + retryBehavior enumspb.NexusHandlerErrorRetryBehavior + errorType string + wantRetryable bool + }{ + { + name: "retryable by type", + errorType: string(nexus.HandlerErrorTypeInternal), + wantRetryable: true, + }, + { + name: "non-retryable by type", + errorType: string(nexus.HandlerErrorTypeBadRequest), + wantRetryable: false, + }, + { + // An explicit retry behavior wins over the type's default. + name: "non-retryable by behavior", + errorType: string(nexus.HandlerErrorTypeInternal), + retryBehavior: enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_NON_RETRYABLE, + wantRetryable: false, + }, + { + name: "retryable by behavior", + errorType: string(nexus.HandlerErrorTypeBadRequest), + retryBehavior: enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_RETRYABLE, + wantRetryable: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + resp := &matchingservice.DispatchNexusTaskResponse{ + //nolint:staticcheck // Deprecated, still sent by older workers. + Outcome: &matchingservice.DispatchNexusTaskResponse_HandlerError{ + HandlerError: &nexuspb.HandlerError{ + ErrorType: tc.errorType, + RetryBehavior: tc.retryBehavior, + Failure: &nexuspb.Failure{Message: "worker said no"}, + }, + }, + } + err := MatchingDispatchResponseToError(resp) + + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorType(tc.errorType), handlerErr.Type) + require.Equal(t, tc.wantRetryable, handlerErr.Retryable()) + // HandlerError.Error() renders the type and message but not the cause, so the worker's + // message has to be on the error itself to reach anything that records err.Error(). + require.Contains(t, err.Error(), "worker said no") + }) + } +} + +// Older workers report a failed operation with the deprecated OperationError variant. +func TestStartOperationResponseToError_DeprecatedOperationError(t *testing.T) { + resp := &nexuspb.StartOperationResponse{ + //nolint:staticcheck // Deprecated, still sent by older workers. + Variant: &nexuspb.StartOperationResponse_OperationError{ + OperationError: &nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateCanceled), + Failure: &nexuspb.Failure{Message: "operation was canceled"}, + }, + }, + } + err := StartOperationResponseToError(resp) + + var opErr *nexus.OperationError + require.ErrorAs(t, err, &opErr) + require.Equal(t, nexus.OperationStateCanceled, opErr.State) + // OperationError.Error() does not include the cause either. + require.Contains(t, err.Error(), "operation was canceled") +} + func TestMatchingDispatchResponseToError_EmptyOutcome(t *testing.T) { resp := &matchingservice.DispatchNexusTaskResponse{} err := MatchingDispatchResponseToError(resp) diff --git a/common/nexus/failure.go b/common/nexus/failure.go index 0b5dfb7a427..44e7f0972e2 100644 --- a/common/nexus/failure.go +++ b/common/nexus/failure.go @@ -14,8 +14,9 @@ import ( failurepb "go.temporal.io/api/failure/v1" nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/api/serviceerror" + "go.temporal.io/server/common" + "go.temporal.io/server/common/nexus/nexusrpc" "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" ) @@ -323,16 +324,10 @@ func nexusFailureMetadataToApplicationFailureInfo(failure nexus.Failure) (*failu // and // https://github.com/grpc-ecosystem/grpc-gateway/blob/a7cf811e6ffabeaddcfb4ff65602c12671ff326e/runtime/errors.go#L56. func ConvertGRPCError(err error, exposeDetails bool) error { - var st *status.Status - stGetter, ok := err.(interface{ Status() *status.Status }) - if ok { - st = stGetter.Status() - } else { - st, ok = status.FromError(err) - if !ok { - // The Nexus SDK will translate this into an internal server error and will not expose the error details. - return err - } + st, ok := common.GetRPCStatus(err) + if !ok { + // The Nexus SDK will translate this into an internal server error and will not expose the error details. + return err } errMessage := err.Error() @@ -439,3 +434,21 @@ func AdaptAuthorizeError(permissionDeniedError *serviceerror.PermissionDenied) e } return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnauthorized, "permission denied") } + +func OperationErrorToTemporalFailure(opErr *nexus.OperationError) (*failurepb.Failure, error) { + var nf nexus.Failure + if opErr.OriginalFailure != nil { + nf = *opErr.OriginalFailure + } else { + var err error + nf, err = nexusrpc.DefaultFailureConverter().ErrorToFailure(opErr) + if err != nil { + return nil, err + } + } + + // The Nexus failure may contain a metadata key requesting that the unwrapped + // (Cause) of the failure is sent, to avoid an unnecessary layer of indirection. + unwrappedFailure := nexusrpc.UnwrapFailure(&nf) + return NexusFailureToTemporalFailure(*unwrappedFailure) +} diff --git a/common/util.go b/common/util.go index 750dabac745..ff02414660b 100644 --- a/common/util.go +++ b/common/util.go @@ -703,6 +703,19 @@ func CloneProtoMap[K comparable, T proto.Message](src map[K]T) map[K]T { return result } +// CloneProtoSlice returns a new slice containing a clone of each individual proto. +func CloneProtoSlice[T proto.Message](src []T) []T { + if src == nil { + return nil + } + + result := make([]T, len(src)) + for i, v := range src { + result[i] = CloneProto(v) + } + return result +} + // DiscardUnknownProto discards unknown fields in a proto message. func DiscardUnknownProto(m proto.Message) error { return protorange.Range(m.ProtoReflect(), func(values protopath.Values) error { @@ -762,18 +775,22 @@ func getFieldNameFromStruct(structPtr any, fieldPtr any) (string, error) { return "", serviceerror.NewInternal("field not found in the struct") } -// IsRetryableRPCError checks if the error is a retryable gRPC error. -func IsRetryableRPCError(err error) bool { - var st *status.Status +// GetRPCStatus attempts to get the gRPC status from the error if possible. +// Returns nil, false if it was not a gRPC-induced error. +func GetRPCStatus(err error) (*status.Status, bool) { stGetter, ok := err.(interface{ Status() *status.Status }) if ok { - st = stGetter.Status() - } else { - st, ok = status.FromError(err) - if !ok { - // Not a gRPC induced error - return false - } + return stGetter.Status(), true + } + return status.FromError(err) +} + +// IsRetryableRPCError checks if the error is a retryable gRPC error. +func IsRetryableRPCError(err error) bool { + st, ok := GetRPCStatus(err) + if !ok { + // Not a gRPC error. + return false } // nolint:exhaustive switch st.Code() { diff --git a/components/callbacks/chasm_invocation.go b/components/callbacks/chasm_invocation.go index 568ac78a4af..1cbb6ec04e2 100644 --- a/components/callbacks/chasm_invocation.go +++ b/components/callbacks/chasm_invocation.go @@ -11,13 +11,12 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/common" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -71,7 +70,7 @@ func (c chasmInvocation) Invoke(ctx context.Context, ns *namespace.Namespace, e _, err = e.HistoryClient.CompleteNexusOperationChasm(ctx, request) if err != nil { redactedErr := logInternalError(e.Logger, "failed to complete Nexus operation: %v", err) - if isRetryableRPCResponse(err) { + if common.IsRetryableRPCError(err) { return invocationResultRetry{redactedErr} } return invocationResultFail{redactedErr} @@ -133,30 +132,3 @@ func (c chasmInvocation) getHistoryRequest( return req, nil } - -func isRetryableRPCResponse(err error) bool { - var st *status.Status - stGetter, ok := err.(interface{ Status() *status.Status }) - if ok { - st = stGetter.Status() - } else { - st, ok = status.FromError(err) - if !ok { - // Not a gRPC induced error - return false - } - } - // nolint:exhaustive - switch st.Code() { - case codes.Canceled, - codes.Unknown, - codes.Unavailable, - codes.DeadlineExceeded, - codes.ResourceExhausted, - codes.Aborted, - codes.Internal: - return true - default: - return false - } -} diff --git a/go.mod b/go.mod index 7a9d0965bba..9c23853b382 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.44.0 - go.temporal.io/api v1.63.5 + go.temporal.io/api v1.63.6-0.20260819173644-14a6a42d0634 // DO NOT SUBMIT, points to feature/worker-callbacks branch go.temporal.io/auto-scaled-workers v0.0.0-20260811170210-91f6fe1d10ab go.temporal.io/sdk v1.44.0 go.uber.org/fx v1.24.0 diff --git a/go.sum b/go.sum index 95797622c25..5e2ef30e97e 100644 --- a/go.sum +++ b/go.sum @@ -479,8 +479,8 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= -go.temporal.io/api v1.63.5 h1:c11+kPYHkXXL3UiShPdbMD+xtvqGsbTibUA9ypmiCa4= -go.temporal.io/api v1.63.5/go.mod h1:SrlW2JMwVlDP4nRWSNznUFqnSHd+YeMDS1BkYo63HCQ= +go.temporal.io/api v1.63.6-0.20260819173644-14a6a42d0634 h1:unIqutnzqqpkJke6+WFQTHgDpzAHJ5AWmiBuB73hkRU= +go.temporal.io/api v1.63.6-0.20260819173644-14a6a42d0634/go.mod h1:SrlW2JMwVlDP4nRWSNznUFqnSHd+YeMDS1BkYo63HCQ= go.temporal.io/auto-scaled-workers v0.0.0-20260811170210-91f6fe1d10ab h1:99wXW0317BBi49d6xgMdA0EZtvA+xbBUWV4HsTEGEcg= go.temporal.io/auto-scaled-workers v0.0.0-20260811170210-91f6fe1d10ab/go.mod h1:hhHijO9XRPIkAflLJJHix61M9FzbRPqk8fSydkcLkqw= go.temporal.io/sdk v1.44.0 h1:suitPDukX74rW3/N1FqvEbZTZVJJsxMKhv0KMa/j7pU= diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 15b12f1575a..f839d38ab52 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -905,12 +905,18 @@ func OperatorHandlerProvider( // callbackValidatorProvider creates a callback Validator using the production dynamic config keys // so that existing operator configurations (callback.allowedAddresses) are honored. func callbackValidatorProvider(dc *dynamicconfig.Collection) (callbacks.Validator, error) { - return callbacks.NewValidator(callbacks.ValidatorConfig{ - MaxCallbacksPerExecution: chasmcallback.MaxPerExecution.Get(dc), - URLMaxLength: dynamicconfig.FrontendCallbackURLMaxLength.Get(dc), - HeaderMaxSize: dynamicconfig.FrontendCallbackHeaderMaxSize.Get(dc), - EndpointRules: chasmcallback.AllowedAddresses.Get(dc), - }) + cfg := callbacks.ValidatorConfig{ + MaxCallbacksPerExecution: chasmcallback.MaxPerExecution.Get(dc), + MaxIDLengthLimit: dynamicconfig.MaxIDLengthLimit.Get(dc), + URLMaxLength: dynamicconfig.FrontendCallbackURLMaxLength.Get(dc), + HeaderMaxSize: dynamicconfig.FrontendCallbackHeaderMaxSize.Get(dc), + EndpointRules: chasmcallback.AllowedAddresses.Get(dc), + MaxServiceNameLength: chasmnexus.MaxServiceNameLength.Get(dc), + MaxOperationNameLength: chasmnexus.MaxOperationNameLength.Get(dc), + WorkerSourceContextMaxSize: chasmcallback.WorkerSourceContextMaxSize.Get(dc), + WorkerSourceContextAggregateMaxSize: chasmcallback.WorkerSourceContextAggregateMaxSize.Get(dc), + } + return callbacks.NewValidator(cfg) } func HandlerProvider( diff --git a/service/frontend/service.go b/service/frontend/service.go index f48d86c33bc..13b92c01e41 100644 --- a/service/frontend/service.go +++ b/service/frontend/service.go @@ -13,6 +13,7 @@ import ( "go.temporal.io/server/chasm/lib/activity" chasmcallback "go.temporal.io/server/chasm/lib/callback" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" @@ -217,6 +218,9 @@ type Config struct { MaxCallbacksPerWorkflow dynamicconfig.IntPropertyFnWithNamespaceFilter CallbackEndpointConfigs dynamicconfig.TypedPropertyFnWithNamespaceFilter[callbacks.AddressMatchRules] + // The callback kinds a client may attach to a workflow execution. + WorkflowEnabledCallbackKinds dynamicconfig.TypedPropertyFnWithNamespaceFilter[[]callbacks.Kind] + MaxNexusOperationTokenLength dynamicconfig.IntPropertyFnWithNamespaceFilter NexusRequestHeadersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp] NexusForwardRequestUseEndpoint dynamicconfig.BoolPropertyFn @@ -419,7 +423,9 @@ func NewConfig( LinkMaxSize: dynamicconfig.FrontendLinkMaxSize.Get(dc), MaxLinksPerRequest: dynamicconfig.FrontendMaxLinksPerRequest.Get(dc), - CallbackEndpointConfigs: chasmcallback.AllowedAddresses.Get(dc), + CallbackEndpointConfigs: chasmcallback.AllowedAddresses.Get(dc), + WorkflowEnabledCallbackKinds: chasmworkflow.EnabledCallbackKinds.Get(dc), + AdminEnableListHistoryTasks: dynamicconfig.AdminEnableListHistoryTasks.Get(dc), MaskInternalErrorDetails: dynamicconfig.FrontendMaskInternalErrorDetails.Get(dc), diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index 9bed3410555..206bc2d5d6b 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -688,7 +688,10 @@ func (wh *WorkflowHandler) prepareStartWorkflowRequest( } if cbs := request.GetCompletionCallbacks(); len(cbs) > 0 { - if err := wh.callbackValidator.Validate(ctx, namespaceName.String(), cbs); err != nil { + opts := callbacks.ValidatorOptions{ + EnabledKinds: wh.config.WorkflowEnabledCallbackKinds(namespaceName.String()), + } + if err := wh.callbackValidator.Validate(ctx, namespaceName.String(), cbs, opts); err != nil { return nil, err } } @@ -5500,7 +5503,10 @@ func (wh *WorkflowHandler) prepareUpdateWorkflowRequest( } if cbs := request.GetRequest().GetCompletionCallbacks(); len(cbs) > 0 { - if err := wh.callbackValidator.Validate(ctx, namespaceName.String(), cbs); err != nil { + opts := callbacks.ValidatorOptions{ + EnabledKinds: wh.config.WorkflowEnabledCallbackKinds(namespaceName.String()), + } + if err := wh.callbackValidator.Validate(ctx, namespaceName.String(), cbs, opts); err != nil { return err } } diff --git a/service/frontend/workflow_handler_test.go b/service/frontend/workflow_handler_test.go index b58abcc4a81..de3417095a6 100644 --- a/service/frontend/workflow_handler_test.go +++ b/service/frontend/workflow_handler_test.go @@ -177,6 +177,7 @@ func (s *WorkflowHandlerSuite) getWorkflowHandler(config *Config) *WorkflowHandl healthInterceptor.SetHealthy(true) cbValidator, err := callbacks.NewValidator(callbacks.ValidatorConfig{ + MaxIDLengthLimit: func() int { return 100 }, MaxCallbacksPerExecution: func(string) int { return 2000 }, URLMaxLength: config.CallbackURLMaxLength, HeaderMaxSize: config.CallbackHeaderMaxSize, @@ -187,6 +188,10 @@ func (s *WorkflowHandlerSuite) getWorkflowHandler(config *Config) *WorkflowHandl }, } }, + MaxServiceNameLength: func(string) int { return 100 }, + MaxOperationNameLength: func(string) int { return 100 }, + WorkerSourceContextMaxSize: func(string) int { return 4096 }, + WorkerSourceContextAggregateMaxSize: func(string) int { return 2 * 1024 * 1024 }, }) s.NoError(err) @@ -243,6 +248,8 @@ func (s *WorkflowHandlerSuite) getWorkflowHandler(config *Config) *WorkflowHandl nil, s.mockResource.GetSearchAttributesMapperProvider(), nil, + nil, + nil, ), nil, // Not testing CHASM registry here quotas.NoopRequestRateLimiter, @@ -1128,7 +1135,11 @@ func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Failed_InvalidLinks() s.ErrorContains(err, "nexus operation link must not have an empty run ID field") } -func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Failed_InvalidCallbackLinks() { +// Creates a valid StartWorkflowExecution request, but overwrites the CompletionCallbacks field with +// the supplied data. Returns the result. +func (s *WorkflowHandlerSuite) startWorkflowWithCallbacks( + cbs []*commonpb.Callback, +) (*workflowservice.StartWorkflowExecutionResponse, error) { s.mockSearchAttributesMapperProvider.EXPECT().GetMapper(gomock.Any()).AnyTimes().Return(nil, nil) config := s.newConfig() wh := s.getWorkflowHandler(config) @@ -1142,15 +1153,21 @@ func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Failed_InvalidCallback TaskQueue: &taskqueuepb.TaskQueue{ Name: "task-queue", }, - RequestId: uuid.NewString(), - CompletionCallbacks: []*commonpb.Callback{ - { - Variant: nexusCallbackVariant(), - Links: []*commonpb.Link{ - { - Variant: &commonpb.Link_WorkflowEvent_{ - WorkflowEvent: &commonpb.Link_WorkflowEvent{}, - }, + RequestId: uuid.NewString(), + CompletionCallbacks: cbs, + } + + return wh.StartWorkflowExecution(context.Background(), req) +} + +func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Failed_InvalidCallbackLinks() { + cbs := []*commonpb.Callback{ + { + Variant: nexusCallbackVariant(), + Links: []*commonpb.Link{ + { + Variant: &commonpb.Link_WorkflowEvent_{ + WorkflowEvent: &commonpb.Link_WorkflowEvent{}, }, }, }, @@ -1158,11 +1175,49 @@ func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Failed_InvalidCallback } var invalidArgument *serviceerror.InvalidArgument - _, err := wh.StartWorkflowExecution(context.Background(), req) + _, err := s.startWorkflowWithCallbacks(cbs) s.ErrorAs(err, &invalidArgument) s.ErrorContains(err, "workflow event link must not have an empty namespace field") } +// Assert that Workflows can only accept Nexus-variant callbacks. (Or Internal.) +func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Failed_NonNexusCallback() { + testCases := []struct { + Name string + Callback *commonpb.Callback + // ErrTarget is a pointer to a serviceerror pointer, as expected by ErrorAs. + ErrMsg string + }{ + { + Name: "worker", + Callback: &commonpb.Callback{ + Variant: &commonpb.Callback_Worker_{ + Worker: &commonpb.Callback_Worker{ + TaskQueueName: "completions-task-queue", + Service: "HTTPAdapter", + Operation: "DeliverAsWebhook", + }, + }, + }, + // The validator rejects the Worker variant explicitly, before it reaches + // the unknown-variant fallback. + ErrMsg: "worker callbacks are not enabled for this execution type", + }, + { + Name: "nil variant", + Callback: &commonpb.Callback{}, + ErrMsg: "unknown callback variant", + }, + } + + for _, tc := range testCases { + s.Run(tc.Name, func() { + _, err := s.startWorkflowWithCallbacks([]*commonpb.Callback{tc.Callback}) + s.Require().ErrorContains(err, tc.ErrMsg) + }) + } +} + func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Failed_InvalidAggregatedLinks() { s.mockSearchAttributesMapperProvider.EXPECT().GetMapper(gomock.Any()).AnyTimes().Return(nil, nil) config := s.newConfig() diff --git a/service/history/api/describeworkflow/api.go b/service/history/api/describeworkflow/api.go index 94cfaf60535..8961a5e4e49 100644 --- a/service/history/api/describeworkflow/api.go +++ b/service/history/api/describeworkflow/api.go @@ -19,10 +19,10 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" chasmcallback "go.temporal.io/server/chasm/lib/callback" - callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/chasm/lib/nexusoperation" chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common" + commoncallbacks "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/locks" "go.temporal.io/server/common/log" @@ -278,13 +278,11 @@ func Invoke( ) } chasmCallbackInfos, err := buildCallbackInfosFromChasm( - ctx, namespaceID, wf, chasmCtx, executionInfo, executionState, - outboundQueueCBPool, shard.GetLogger(), ) if err != nil { @@ -502,13 +500,11 @@ func buildCallbackInfosFromHSM( // buildCallbackInfosFromChasm reads callbacks from the CHASM workflow component and converts them to API format. // TODO(long-nt-tran): move this to chasm/lib/workflow/workflow.go to be within the CHASM workflow context. func buildCallbackInfosFromChasm( - ctx context.Context, namespaceID namespace.ID, wf *chasmworkflow.Workflow, chasmCtx chasm.Context, executionInfo *persistencespb.WorkflowExecutionInfo, executionState *persistencespb.WorkflowExecutionState, - outboundQueueCBPool *circuitbreakerpool.OutboundQueueCircuitBreakerPool, logger log.Logger, ) ([]*workflowpb.CallbackInfo, error) { result := make([]*workflowpb.CallbackInfo, 0, len(wf.Callbacks)) @@ -519,7 +515,7 @@ func buildCallbackInfosFromChasm( Variant: &workflowpb.CallbackInfo_Trigger_WorkflowClosed{}, } - callbackInfo, err := buildCallbackInfoFromChasm(ctx, namespaceID, callback, trigger, outboundQueueCBPool) + callbackInfo, err := buildChasmCallbackInfo(chasmCtx, callback, trigger) if err != nil { logger.Error( "failed to build callback info from CHASM callback", @@ -550,7 +546,7 @@ func buildCallbackInfosFromChasm( }, } - callbackInfo, err := buildCallbackInfoFromChasm(ctx, namespaceID, callback, trigger, outboundQueueCBPool) + callbackInfo, err := buildChasmCallbackInfo(chasmCtx, callback, trigger) if err != nil { logger.Error( "failed to build callback info from CHASM update callback", @@ -571,82 +567,41 @@ func buildCallbackInfosFromChasm( return result, nil } -// buildCallbackInfoFromChasm converts a single CHASM callback to API format. -func buildCallbackInfoFromChasm( - ctx context.Context, - namespaceID namespace.ID, - callback *chasmcallback.Callback, - trigger *workflowpb.CallbackInfo_Trigger, - outboundQueueCBPool *circuitbreakerpool.OutboundQueueCircuitBreakerPool, -) (*workflowpb.CallbackInfo, error) { - // Create a circuit breaker state checker function - circuitBreakerState := func(destination string) bool { - cb := outboundQueueCBPool.Get(tasks.TaskGroupNamespaceIDAndDestination{ - TaskGroup: callbacks.TaskTypeInvocation, - NamespaceID: namespaceID.String(), - Destination: destination, - }) - return cb.State() != gobreaker.StateClosed - } - - return buildChasmCallbackInfo(ctx, namespaceID.String(), callback, trigger, circuitBreakerState) -} - // buildChasmCallbackInfo converts a single CHASM callback to API CallbackInfo format. // Returns nil if the callback should not be included in the response. +// +//nolint:revive // context.Context is an input parameter for chasm component methods, not a function parameter func buildChasmCallbackInfo( - ctx context.Context, - namespaceID string, + ctx chasm.Context, cb *chasmcallback.Callback, trigger *workflowpb.CallbackInfo_Trigger, - circuitBreakerState func(destination string) bool, ) (*workflowpb.CallbackInfo, error) { - nexusVariant := cb.GetCallback().GetNexus() - if nexusVariant == nil { - // Only Nexus callbacks are supported - return nil, nil - } - - cbSpec, err := cb.ToAPICallback() + apiCb, err := cb.ToAPICallback() if err != nil { return nil, err } - var state enumspb.CallbackState - switch cb.Status { - case callbackspb.CALLBACK_STATUS_UNSPECIFIED: - return nil, serviceerror.NewInternal("callback with UNSPECIFIED state") - case callbackspb.CALLBACK_STATUS_STANDBY: - state = enumspb.CALLBACK_STATE_STANDBY - case callbackspb.CALLBACK_STATUS_SCHEDULED: - state = enumspb.CALLBACK_STATE_SCHEDULED - case callbackspb.CALLBACK_STATUS_BACKING_OFF: - state = enumspb.CALLBACK_STATE_BACKING_OFF - case callbackspb.CALLBACK_STATUS_FAILED: - state = enumspb.CALLBACK_STATE_FAILED - case callbackspb.CALLBACK_STATUS_SUCCEEDED: - state = enumspb.CALLBACK_STATE_SUCCEEDED - default: - return nil, serviceerror.NewInternalf("unknown callback state: %v", cb.Status) + cbKind := commoncallbacks.KindOf(apiCb) + if cbKind == commoncallbacks.KindUnknown { + // A variant this server does not know how to describe, e.g. one written by a newer server. + // Omit it rather than failing the whole response. + return nil, nil } - blockedReason := "" - if state == enumspb.CALLBACK_STATE_SCHEDULED { - if circuitBreakerState(cbSpec.GetNexus().GetUrl()) { - state = enumspb.CALLBACK_STATE_BLOCKED - blockedReason = "The circuit breaker is open." - } + state, blockedReason, err := cb.APIState(ctx) + if err != nil { + return nil, err } return &workflowpb.CallbackInfo{ - Callback: cbSpec, + Callback: apiCb, Trigger: trigger, - RegistrationTime: cb.RegistrationTime, + RegistrationTime: common.CloneProto(cb.RegistrationTime), State: state, Attempt: cb.Attempt, - LastAttemptCompleteTime: cb.LastAttemptCompleteTime, - LastAttemptFailure: cb.LastAttemptFailure, - NextAttemptScheduleTime: cb.NextAttemptScheduleTime, + LastAttemptCompleteTime: common.CloneProto(cb.LastAttemptCompleteTime), + LastAttemptFailure: common.CloneProto(cb.LastAttemptFailure), + NextAttemptScheduleTime: common.CloneProto(cb.NextAttemptScheduleTime), BlockedReason: blockedReason, }, nil } diff --git a/service/history/fx.go b/service/history/fx.go index ce20b90be3a..10a208c9ae1 100644 --- a/service/history/fx.go +++ b/service/history/fx.go @@ -5,6 +5,7 @@ import ( "time" "github.com/nexus-rpc/sdk-go/nexus" + "github.com/sony/gobreaker" "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity" @@ -43,12 +44,14 @@ import ( "go.temporal.io/server/service" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/archival" + "go.temporal.io/server/service/history/circuitbreakerpool" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" "go.temporal.io/server/service/history/replication" "go.temporal.io/server/service/history/shard" + "go.temporal.io/server/service/history/tasks" "go.temporal.io/server/service/history/workflow" "go.temporal.io/server/service/history/workflow/cache" "go.temporal.io/server/service/worker/workerdeployment" @@ -123,11 +126,28 @@ var Module = fx.Options( activity.HistoryModule, scheduler.Module, callback.Module, + fx.Provide(CallbackDestinationBlockedProvider), chasmnexus.Module, chasmworkflow.Module, chasmworkflow.HistoryHandlerModule, ) +// CallbackDestinationBlockedProvider lets the callback library report a callback as BLOCKED while +// the outbound queue's circuit breaker for its destination is open. Only the history service runs +// that queue, so only it can answer. +func CallbackDestinationBlockedProvider( + outboundQueueCBPool *circuitbreakerpool.OutboundQueueCircuitBreakerPool, +) callback.DestinationBlockedFn { + return func(namespaceID string, destination string) bool { + cb := outboundQueueCBPool.Get(tasks.TaskGroupNamespaceIDAndDestination{ + TaskGroup: callback.InvocationTaskGroup, + NamespaceID: namespaceID, + Destination: destination, + }) + return cb.State() != gobreaker.StateClosed + } +} + func ServerProvider(grpcServerOptions []grpc.ServerOption) *grpc.Server { return grpc.NewServer(grpcServerOptions...) } diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index 9407c3db49c..c9e3192678a 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -10,6 +10,7 @@ import ( "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" activitypb "go.temporal.io/api/activity/v1" + callbackpb "go.temporal.io/api/callback/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" @@ -10066,6 +10067,31 @@ func (env *standaloneActivityEnv) startActivityWithType(ctx context.Context, act }) } +// awaitCallbackInfo polls DescribeActivityExecution until the activity's single completion +// callback reaches wantState, and returns that CallbackInfo. +func (env *standaloneActivityEnv) awaitCallbackInfo( + ctx context.Context, + t *testing.T, + activityID string, + wantState enumspb.CallbackState, +) *callbackpb.CallbackInfo { + t.Helper() + var cbInfo *callbackpb.CallbackInfo + await.Require(ctx, t, func(c *await.T) { + descResp, err := env.FrontendClient().DescribeActivityExecution(c.Context(), &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + }) + require.NoError(c, err) + require.Len(c, descResp.GetCallbacks(), 1) + cbInfo = descResp.GetCallbacks()[0].GetInfo() + require.NotNil(c, cbInfo) + require.Equal(c, wantState, cbInfo.GetState()) + }, 10*time.Second, 100*time.Millisecond) + return cbInfo +} + +// Tests verifying that completion callbacks attached to standalone Activities get triggered. func (s *standaloneActivityTestSuite) TestCallbacks() { env := s.newTestEnv() t := s.T() @@ -10075,6 +10101,59 @@ func (s *standaloneActivityTestSuite) TestCallbacks() { []any{map[string]any{"Pattern": "*", "AllowInsecure": true}}, ) + // Confirm that SAA fails with unsupported callback variants. + t.Run("RejectNonNexusCallbacks", func(t *testing.T) { + activityID := testcore.RandomizeStr(t.Name()) + taskQueue := testcore.RandomizeStr(t.Name()) + + tests := []struct { + Name string + Callback *commonpb.Callback + ErrMsg string + }{ + { + Name: "worker", + Callback: &commonpb.Callback{ + Variant: &commonpb.Callback_Worker_{ + Worker: &commonpb.Callback_Worker{ + TaskQueueName: "completions-task-queue", + Service: "HTTPAdapter", + Operation: "DeliverAsWebhook", + }, + }, + }, + // The validator rejects the Worker variant explicitly, before it reaches + // the unknown-variant fallback. + ErrMsg: "worker callbacks are not enabled for this execution type", + }, + { + Name: "nil", + Callback: &commonpb.Callback{}, + ErrMsg: "unknown callback variant", + }, + } + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + resp, err := env.FrontendClient().StartActivityExecution(s.Context(), &workflowservice.StartActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + ActivityType: env.Tv().ActivityType(), + Identity: env.Tv().WorkerIdentity(), + Input: defaultInput, + TaskQueue: &taskqueuepb.TaskQueue{ + Name: taskQueue, + }, + StartToCloseTimeout: durationpb.New(defaultStartToCloseTimeout), + RequestId: env.Tv().Any().String(), + CompletionCallbacks: []*commonpb.Callback{test.Callback}, + }) + + require.Nil(t, resp) + require.ErrorContains(t, err, test.ErrMsg) + }) + } + }) + t.Run("AcceptedOnStart", func(t *testing.T) { activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) @@ -10163,6 +10242,8 @@ func (s *standaloneActivityTestSuite) TestCallbacks() { require.Equal(t, callbackURL, cbInfo.GetInfo().GetCallback().GetNexus().GetUrl()) require.Equal(t, enumspb.CALLBACK_STATE_STANDBY, cbInfo.GetInfo().GetState()) require.NotNil(t, cbInfo.GetInfo().GetRegistrationTime()) + // Confirm there is no result, because the callback hasn't been triggered. + require.Nil(t, cbInfo.GetInfo().GetResult()) }) t.Run("ExceedsMaxCallbacksLimit", func(t *testing.T) { @@ -10257,6 +10338,10 @@ func (s *standaloneActivityTestSuite) TestCallbacks() { }) require.NoError(t, err) require.Equal(t, enumspb.ACTIVITY_EXECUTION_STATUS_COMPLETED, descResp.GetInfo().GetStatus()) + + // Wait for the callback to complete and confirm it has a Success result. + cbInfo := env.awaitCallbackInfo(s.Context(), t, activityID, enumspb.CALLBACK_STATE_SUCCEEDED) + require.NotNil(t, cbInfo.GetSuccess()) }) t.Run("FailsWithCallbacks", func(t *testing.T) { @@ -10323,6 +10408,9 @@ func (s *standaloneActivityTestSuite) TestCallbacks() { }) require.NoError(t, err) require.Equal(t, enumspb.ACTIVITY_EXECUTION_STATUS_FAILED, descResp.GetInfo().GetStatus()) + + // The Activity may have failed, but the callback reporting the failure should be successful. + env.awaitCallbackInfo(s.Context(), t, activityID, enumspb.CALLBACK_STATE_SUCCEEDED) }) t.Run("TerminatedWithCallbacks", func(t *testing.T) { @@ -10390,6 +10478,9 @@ func (s *standaloneActivityTestSuite) TestCallbacks() { }) require.NoError(t, err) require.Equal(t, enumspb.ACTIVITY_EXECUTION_STATUS_TERMINATED, descResp.GetInfo().GetStatus()) + + // The callback reporting the termination should be delivered successfully. + env.awaitCallbackInfo(s.Context(), t, activityID, enumspb.CALLBACK_STATE_SUCCEEDED) }) t.Run("CanceledWithCallbacks", func(t *testing.T) { @@ -10462,6 +10553,9 @@ func (s *standaloneActivityTestSuite) TestCallbacks() { }) require.NoError(t, err) require.Equal(t, enumspb.ACTIVITY_EXECUTION_STATUS_CANCELED, descResp.GetInfo().GetStatus()) + + // The callback reporting the cancellation should be delivered successfully. + env.awaitCallbackInfo(s.Context(), t, activityID, enumspb.CALLBACK_STATE_SUCCEEDED) }) // This test covers the timeout callback path using schedule-to-start, but the callback behavior @@ -10517,6 +10611,104 @@ func (s *standaloneActivityTestSuite) TestCallbacks() { }) require.NoError(t, err) require.Equal(t, enumspb.ACTIVITY_EXECUTION_STATUS_TIMED_OUT, descResp.GetInfo().GetStatus()) + + // The callback delivering the timeout failure should itself succeed. + env.awaitCallbackInfo(s.Context(), t, activityID, enumspb.CALLBACK_STATE_SUCCEEDED) + }) + + // Verify that if the callback fails to be delivered for some reason, that the failure is + // persisted correctly and available from the Describe operation. + t.Run("CallbackDeliveryFailure", func(t *testing.T) { + activityID := testcore.RandomizeStr(t.Name()) + taskQueue := testcore.RandomizeStr(t.Name()) + + ch, callbackAddress := newNexusCompletionHandler(t) + + // Start and successfully complete a standalone Activity. + _, err := env.FrontendClient().StartActivityExecution(s.Context(), &workflowservice.StartActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + ActivityType: env.Tv().ActivityType(), + Identity: env.Tv().WorkerIdentity(), + Input: defaultInput, + TaskQueue: &taskqueuepb.TaskQueue{ + Name: taskQueue, + }, + StartToCloseTimeout: durationpb.New(defaultStartToCloseTimeout), + RequestId: env.Tv().Any().String(), + CompletionCallbacks: []*commonpb.Callback{{ + Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{Url: callbackAddress}}, + }}, + }) + require.NoError(t, err) + + pollResp, err := env.FrontendClient().PollActivityTaskQueue(s.Context(), &workflowservice.PollActivityTaskQueueRequest{ + Namespace: env.Namespace().String(), + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, + Identity: env.Tv().WorkerIdentity(), + }) + require.NoError(t, err) + + _, err = env.FrontendClient().RespondActivityTaskCompleted(s.Context(), &workflowservice.RespondActivityTaskCompletedRequest{ + Namespace: env.Namespace().String(), + TaskToken: pollResp.TaskToken, + Result: defaultResult, + Identity: defaultIdentity, + }) + require.NoError(t, err) + + // Simulate the completion handler returning a retryable error followed by + // an unretryable error. Confirm the SAA's CallbackInfo includes the terminal + // failure. + for deliveryAttempt := 1; deliveryAttempt <= 2; deliveryAttempt++ { + select { + case completion := <-ch.requestCh: + // Pull the completion request from the channel. + require.Equal(t, nexus.OperationStateSucceeded, completion.State) + if deliveryAttempt == 1 { + // The first attempt to deliver the Activity's completion callback reports a retryable error. + // Call Describe and confirm the Callback has just been scheduled. + cbInfo := env.awaitCallbackInfo(s.Context(), t, activityID, enumspb.CALLBACK_STATE_SCHEDULED) + require.EqualValues(t, 0, cbInfo.GetAttempt()) // zero attempts so far. + require.Nil(t, cbInfo.GetLastAttemptFailure()) + require.Nil(t, cbInfo.GetResult()) + + // Retryable error. + ch.requestCompleteCh <- nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "delivery #1") + } else { + // The second attempt to deliver the Activity's completion callback should report a + // non-retryable error. + // Call Describe and confirm the CallbackInfo describes the previous delivery attempt. + cbInfo := env.awaitCallbackInfo(s.Context(), t, activityID, enumspb.CALLBACK_STATE_SCHEDULED) + require.EqualValues(t, 1, cbInfo.GetAttempt()) // 1 attempt so far, the 2nd is in-progress. + require.NotNil(t, cbInfo.GetLastAttemptFailure()) + require.Contains(t, cbInfo.GetLastAttemptFailure().GetMessage(), "delivery #1") + require.Nil(t, cbInfo.GetResult()) + + // Unretryable error. + ch.requestCompleteCh <- nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "delivery #2") + } + + case <-s.Context().Done(): + require.Fail(t, "timed out waiting for completion callback") + } + } + + // Verify the Activity is in completed state. + descResp, err := env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + }) + require.NoError(t, err) + require.Equal(t, enumspb.ACTIVITY_EXECUTION_STATUS_COMPLETED, descResp.GetInfo().GetStatus()) + + // Verify the completion callback delivery has failed. + cbInfo := env.awaitCallbackInfo(s.Context(), t, activityID, enumspb.CALLBACK_STATE_FAILED) + // Both the last delivery failure and the terminal failure come from delivery #2. + const lastDeliveryFailureMessage = "handler error (BAD_REQUEST): delivery #2" + require.NotNil(t, cbInfo.GetFailure()) + require.Equal(t, lastDeliveryFailureMessage, cbInfo.GetFailure().GetMessage()) + require.Equal(t, lastDeliveryFailureMessage, cbInfo.GetLastAttemptFailure().GetMessage()) }) } diff --git a/tests/callbacks_circuitbreaker_test.go b/tests/callbacks_circuitbreaker_test.go new file mode 100644 index 00000000000..54881ef6841 --- /dev/null +++ b/tests/callbacks_circuitbreaker_test.go @@ -0,0 +1,247 @@ +package tests + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + notificationpb "go.temporal.io/api/notificationservice/v1" + "go.temporal.io/api/workflowservice/v1" + sdkworker "go.temporal.io/sdk/worker" + "go.temporal.io/server/chasm/lib/callback" + "go.temporal.io/server/chasm/lib/nexusoperation" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/common/testing/parallelsuite" + "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/tests/testcore" +) + +// WorkerCallbacksCircuitBreakerSuite covers the outbound queue's circuit breaker as it applies to +// Worker-variant callback deliveries: which failures count against it, that it is keyed per task +// queue, that Describe reports a held-back callback as BLOCKED, and that BLOCKED is not terminal. +// +// Standalone Nexus operations stand in for every execution type here, since the delivery path a +// callback takes is the same whatever it hangs off; see [TestWorkerCallbacks]. +type WorkerCallbacksCircuitBreakerSuite struct { + parallelsuite.Suite[*WorkerCallbacksCircuitBreakerSuite] +} + +func TestWorkerCallbacksCircuitBreakerSuite(t *testing.T) { + parallelsuite.Run(t, &WorkerCallbacksCircuitBreakerSuite{}) +} + +// gobreaker's default ReadyToTrip, which the outbound queue's pool leaves in place, opens the +// breaker once consecutive failures exceed five. +const circuitBreakerFailureThreshold = 5 + +// newCircuitBreakerEnv builds an env with the callback retry policy dialed down so that failures +// accumulate within a test's lifetime. The retry policy is a global setting, hence the dedicated +// cluster. +func (s *WorkerCallbacksCircuitBreakerSuite) newCircuitBreakerEnv(extra ...testcore.TestOption) *NexusTestEnv { + opts := []testcore.TestOption{ + testcore.WithDedicatedCluster(), + testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true), + testcore.WithDynamicConfig(dynamicconfig.EnableCHASMCallbacks, true), + testcore.WithDynamicConfig(nexusoperation.Enabled, true), + testcore.WithDynamicConfig(nexusoperation.EnabledCallbackKinds, []string{"worker"}), + testcore.WithDynamicConfig(callback.RetryPolicyInitialInterval, 10*time.Millisecond), + testcore.WithDynamicConfig(callback.RetryPolicyMaximumInterval, 10*time.Millisecond), + } + return newNexusTestEnv(s.T(), true, append(opts, extra...)...) +} + +// fastUpstreamTimeoutOpts make a delivery to a task queue with no poller time out promptly instead +// of after the default ten seconds. Matching buffers the dispatch deadline by MinDispatchTaskTimeout, +// so that has to come down too, and the request timeout has to stay above it — a request timeout +// below the buffer leaves zero time and would fail a delivery to a live worker as well. +func fastUpstreamTimeoutOpts() []testcore.TestOption { + return []testcore.TestOption{ + testcore.WithDynamicConfig(nexusoperations.MinDispatchTaskTimeout, 10*time.Millisecond), + testcore.WithDynamicConfig(callback.RequestTimeout, 500*time.Millisecond), + } +} + +func workerCallbackTo(taskQueue string) *commonpb.Callback { + return &commonpb.Callback{ + Variant: &commonpb.Callback_Worker_{ + Worker: &commonpb.Callback_Worker{ + TaskQueueName: taskQueue, + Service: "completion-service", + Operation: "on-complete", + }, + }, + } +} + +// startHandler starts a worker whose completion handler fails its first initialFailures deliveries +// with a retryable error and succeeds after that, and returns a Worker-variant callback addressed to +// it. A randomized task queue keeps each caller from tripping any other's breaker, since the task +// queue is the destination the breaker is keyed by. +func (s *WorkerCallbacksCircuitBreakerSuite) startHandler( + env *NexusTestEnv, + t *testing.T, + taskQueue string, + initialFailures int32, +) { + t.Helper() + + var delivered atomic.Int32 + service := nexus.NewService("completion-service") + operation := nexus.NewSyncOperation( + "on-complete", + func(_ context.Context, _ *notificationpb.OnCompleteRequest, _ nexus.StartOperationOptions) (*notificationpb.OnCompleteResponse, error) { + if delivered.Add(1) <= initialFailures { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "intentional failure") + } + return ¬ificationpb.OnCompleteResponse{}, nil + }, + ) + require.NoError(t, service.Register(operation)) + + worker := sdkworker.New(env.SdkClient(), taskQueue, sdkworker.Options{}) + worker.RegisterNexusService(service) + require.NoError(t, worker.Start()) + t.Cleanup(worker.Stop) +} + +// startSANOWithCallbacks starts an operation against endpointName, which completes it immediately, +// so its callbacks are scheduled for delivery right away. +func (s *WorkerCallbacksCircuitBreakerSuite) startSANOWithCallbacks( + env *NexusTestEnv, + t *testing.T, + endpointName string, + cbs ...*commonpb.Callback, +) string { + t.Helper() + + operationID := testcore.RandomizeStr(t.Name()) + startResp, err := env.startNexusOperation(s.Context(), &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + CompletionCallbacks: cbs, + }) + require.NoError(t, err) + require.True(t, startResp.GetStarted()) + return operationID +} + +// awaitCallbackState waits for the callback at index to reach wantState, failing if it is reported +// as any of forbidden along the way. +func (s *WorkerCallbacksCircuitBreakerSuite) awaitCallbackState( + env *NexusTestEnv, + t *testing.T, + operationID string, + index, wantCount int, + wantState enumspb.CallbackState, + forbidden ...enumspb.CallbackState, +) { + t.Helper() + + await.Require(s.Context(), t, func(c *await.T) { + cbs := env.describeNexusOperation(c.Context(), c, operationID).GetCompletionCallbacks() + require.Len(c, cbs, wantCount) + got := cbs[index].GetInfo().GetState() + // Asserted on the enclosing t: a forbidden state is a failure, not something to retry until + // it goes away. + require.NotContains(t, forbidden, got) + require.Equal(c, wantState, got) + }, 30*time.Second, 200*time.Millisecond) +} + +// TestBlockedWhenCircuitBreakerOpens covers deliveries that keep failing against the task queue +// itself: the breaker for that task queue opens and Describe reports the callback as BLOCKED. +func (s *WorkerCallbacksCircuitBreakerSuite) TestBlockedWhenCircuitBreakerOpens() { + env := s.newCircuitBreakerEnv(fastUpstreamTimeoutOpts()...) + t := s.T() + + // Nothing polls this task queue, so every delivery ends in matching's upstream timeout. That is + // a property of the destination rather than of any one callback, so it counts against the + // breaker — unlike a handler error, which is the answer of a worker that did receive the + // delivery. See TestHandlerErrorsDoNotOpenTheBreaker. + unservedTaskQueue := testcore.RandomizeStr(t.Name() + "-unserved") + endpointName := env.createSyncSuccessEndpoint(s.Context(), t, "operation-result") + operationID := s.startSANOWithCallbacks(env, t, endpointName, workerCallbackTo(unservedTaskQueue)) + + // Deliveries are retried until enough have failed to open the breaker, so the callback passes + // through SCHEDULED and BACKING_OFF on the way to BLOCKED. + s.awaitCallbackState(env, t, operationID, 0, 1, enumspb.CALLBACK_STATE_BLOCKED) + + cbs := env.describeNexusOperation(s.Context(), t, operationID).GetCompletionCallbacks() + require.Equal(t, "The circuit breaker is open.", cbs[0].GetInfo().GetBlockedReason()) + require.Greater(t, cbs[0].GetInfo().GetAttempt(), int32(circuitBreakerFailureThreshold), + "the breaker should not open before the failure threshold is exceeded") +} + +// TestHandlerErrorsDoNotOpenTheBreaker covers a handler that is up and returning retryable errors. +// That is the registering caller's problem, not the task queue's, so it must not trip the breaker +// for a task queue every other callback may also be delivering to. The handler fails more times than +// the breaker's threshold, so counting those failures — as the delivery path used to — would open it. +func (s *WorkerCallbacksCircuitBreakerSuite) TestHandlerErrorsDoNotOpenTheBreaker() { + env := s.newCircuitBreakerEnv() + t := s.T() + + taskQueue := testcore.RandomizeStr(t.Name() + "-handler") + s.startHandler(env, t, taskQueue, circuitBreakerFailureThreshold+3) + endpointName := env.createSyncSuccessEndpoint(s.Context(), t, "operation-result") + operationID := s.startSANOWithCallbacks(env, t, endpointName, workerCallbackTo(taskQueue)) + + // The delivery that follows the failures succeeds, and the callback is never held back on the + // way there. + s.awaitCallbackState(env, t, operationID, 0, 1, + enumspb.CALLBACK_STATE_SUCCEEDED, enumspb.CALLBACK_STATE_BLOCKED) +} + +// TestBreakerIsPerTaskQueue covers the isolation the per-task-queue destination buys: one dead task +// queue must not hold back callbacks delivering to a healthy one. +// +// The two operations are deliberately sequential. Attaching both callbacks to one operation does not +// test anything: the healthy delivery succeeds in milliseconds, while the dead one needs several +// hundred milliseconds per attempt to accumulate enough failures, so the healthy callback is long +// finished before there is an open breaker for it to be affected by. +func (s *WorkerCallbacksCircuitBreakerSuite) TestBreakerIsPerTaskQueue() { + env := s.newCircuitBreakerEnv(fastUpstreamTimeoutOpts()...) + t := s.T() + endpointName := env.createSyncSuccessEndpoint(s.Context(), t, "operation-result") + + unservedTaskQueue := testcore.RandomizeStr(t.Name() + "-unserved") + blockedOp := s.startSANOWithCallbacks(env, t, endpointName, workerCallbackTo(unservedTaskQueue)) + s.awaitCallbackState(env, t, blockedOp, 0, 1, enumspb.CALLBACK_STATE_BLOCKED) + + // With the breaker for the dead task queue now open, a delivery to a healthy one still goes + // through — it is keyed by its own task queue. + servedTaskQueue := testcore.RandomizeStr(t.Name() + "-served") + s.startHandler(env, t, servedTaskQueue, 0) + servedOp := s.startSANOWithCallbacks(env, t, endpointName, workerCallbackTo(servedTaskQueue)) + s.awaitCallbackState(env, t, servedOp, 0, 1, + enumspb.CALLBACK_STATE_SUCCEEDED, enumspb.CALLBACK_STATE_BLOCKED) +} + +// TestRecoversFromBlocked covers BLOCKED not being terminal: once the breaker's open period elapses +// it half-opens, and a worker that has since shown up gets the delivery. +func (s *WorkerCallbacksCircuitBreakerSuite) TestRecoversFromBlocked() { + env := s.newCircuitBreakerEnv(append( + fastUpstreamTimeoutOpts(), + // Shorten the open period so the breaker half-opens within the test rather than after the + // default minute. + testcore.WithDynamicConfig(dynamicconfig.OutboundQueueCircuitBreakerSettings, + dynamicconfig.CircuitBreakerSettings{Timeout: time.Second}), + )...) + t := s.T() + + taskQueue := testcore.RandomizeStr(t.Name() + "-late") + endpointName := env.createSyncSuccessEndpoint(s.Context(), t, "operation-result") + operationID := s.startSANOWithCallbacks(env, t, endpointName, workerCallbackTo(taskQueue)) + + // No poller yet, so deliveries fail until the breaker opens. + s.awaitCallbackState(env, t, operationID, 0, 1, enumspb.CALLBACK_STATE_BLOCKED) + + // The worker arrives late. The breaker half-opens, lets a delivery through, and it succeeds. + s.startHandler(env, t, taskQueue, 0) + s.awaitCallbackState(env, t, operationID, 0, 1, enumspb.CALLBACK_STATE_SUCCEEDED) +} diff --git a/tests/callbacks_test.go b/tests/callbacks_test.go index bd9ca60d7c9..35c60260593 100644 --- a/tests/callbacks_test.go +++ b/tests/callbacks_test.go @@ -242,7 +242,7 @@ func (s *CallbacksSuite) TestWorkflowCallbacks_InvalidArgument(opts []testcore.T _, err := env.FrontendClient().StartWorkflowExecution(s.Context(), request) var invalidArgument *serviceerror.InvalidArgument s.ErrorAs(err, &invalidArgument) - s.Equal(tc.message, err.Error()) + s.ErrorContains(err, tc.message) }) } } diff --git a/tests/callbacks_worker_test.go b/tests/callbacks_worker_test.go new file mode 100644 index 00000000000..580681c4b2a --- /dev/null +++ b/tests/callbacks_worker_test.go @@ -0,0 +1,721 @@ +package tests + +import ( + "context" + "slices" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + notificationpb "go.temporal.io/api/notificationservice/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + updatepb "go.temporal.io/api/update/v1" + workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/api/workflowservice/v1" + sdkclient "go.temporal.io/sdk/client" + sdkworker "go.temporal.io/sdk/worker" + "go.temporal.io/sdk/workflow" + chasmactivity "go.temporal.io/server/chasm/lib/activity" + "go.temporal.io/server/chasm/lib/nexusoperation" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/payload" + "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/common/testing/protorequire" + "go.temporal.io/server/common/testing/testcontext" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Worker-variant completion callbacks deliver an execution's outcome to a Nexus service on a worker +// polling within the same namespace, rather than round tripping through the frontend's Nexus HTTP +// endpoint. They are gated per execution type by an "enabledCallbackKinds" dynamic config setting, +// which never enables the Worker kind by default. +// +// These tests exercise both sides of that gate, and then the delivery itself, for every execution +// type that accepts completion callbacks: +// +// - without "worker" in the setting, attaching a Worker callback is rejected up front; +// - with "worker" added to the setting, the callback is accepted and registered on the execution, +// and is delivered to the handler its task queue, service, and operation name. +// +// Every case attaches two callbacks routed to two different handlers, so that the outcomes of +// concurrent deliveries off the same execution stay independent: one handler takes the completion +// on the first try, the other rejects it once retryably and then for good. + +const ( + workerCallbackNotEnabledErr = "worker callbacks are not enabled for this execution type" + + // The messages the retry-then-fail handler answers its first and second deliveries with. + firstDeliveryFailure = "delivery #1" + secondDeliveryFailure = "delivery #2" + + // The message the retried callback comes to rest on: the Nexus SDK's rendering of the + // non-retryable handler error the second delivery is answered with. + terminalDeliveryFailureMessage = "handler error (BAD_REQUEST): " + secondDeliveryFailure +) + +// observedCallback normalizes the callback info reported by DescribeWorkflowExecution, +// DescribeActivityExecution, and DescribeNexusOperationExecution, which use different (though +// near-identical) protos. +type observedCallback struct { + callback *commonpb.Callback + state enumspb.CallbackState + // trigger is only reported for workflow executions. + trigger *workflowpb.CallbackInfo_Trigger + attempt int32 + lastAttemptFailure *failurepb.Failure + lastAttemptCompleteTime *timestamppb.Timestamp + nextAttemptScheduleTime *timestamppb.Timestamp +} + +// describeCallbacksFn reads the callbacks currently attached to an execution. +type describeCallbacksFn func() ([]observedCallback, error) + +// workerCallbackHandler is a Nexus service on a worker in the test's namespace that receives +// Worker-variant completion callbacks. Every delivery is recorded, and answered by respond, so a +// test can drive the delivery outcome the server observes. +type workerCallbackHandler struct { + taskQueue string + service string + operation string + // sourceContext is the opaque payload the callback is registered with, which the server carries + // to the handler untouched. + sourceContext *commonpb.Payload + + // respond decides what the handler answers the nth (1-based) delivery with. A nil error reports + // a successful delivery. + respond func(delivery int) error + + mu sync.Mutex + received []*notificationpb.OnCompleteRequest +} + +// newWorkerCallbackHandler starts a worker polling its own task queue, so a delivery has to be +// routed by the callback rather than by the task queue the source execution used. The worker stops +// when t cleans up. +func newWorkerCallbackHandler( + t *testing.T, + client sdkclient.Client, + name string, + respond func(delivery int) error, +) *workerCallbackHandler { + t.Helper() + + h := &workerCallbackHandler{ + taskQueue: testcore.RandomizeStr(t.Name() + "-" + name), + service: "completion-service", + operation: "on-complete", + sourceContext: payload.EncodeString("source-context-" + name), + respond: respond, + } + + service := nexus.NewService(h.service) + require.NoError(t, service.Register(nexus.NewSyncOperation(h.operation, h.handle))) + + worker := sdkworker.New(client, h.taskQueue, sdkworker.Options{}) + worker.RegisterNexusService(service) + require.NoError(t, worker.Start()) + t.Cleanup(worker.Stop) + return h +} + +// newSucceedingWorkerCallbackHandler returns a handler that accepts every delivery. +func newSucceedingWorkerCallbackHandler(t *testing.T, client sdkclient.Client, name string) *workerCallbackHandler { + return newWorkerCallbackHandler(t, client, name, func(int) error { return nil }) +} + +// newRetryThenFailWorkerCallbackHandler returns a handler that answers its first delivery with a +// retryable handler error and every delivery after that with a non-retryable one, so the callback +// is retried exactly once and then fails for good. +// +// UNAVAILABLE rather than UPSTREAM_TIMEOUT for the retryable answer on purpose: only the latter +// counts against the outbound queue's circuit breaker for this task queue. See +// [WorkerCallbacksCircuitBreakerSuite]. +func newRetryThenFailWorkerCallbackHandler(t *testing.T, client sdkclient.Client, name string) *workerCallbackHandler { + return newWorkerCallbackHandler(t, client, name, func(delivery int) error { + if delivery == 1 { + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, firstDeliveryFailure) + } + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, secondDeliveryFailure) + }) +} + +func (h *workerCallbackHandler) handle( + _ context.Context, + req *notificationpb.OnCompleteRequest, + _ nexus.StartOperationOptions, +) (*notificationpb.OnCompleteResponse, error) { + h.mu.Lock() + h.received = append(h.received, req) + delivery := len(h.received) + h.mu.Unlock() + + if err := h.respond(delivery); err != nil { + return nil, err + } + return ¬ificationpb.OnCompleteResponse{}, nil +} + +// callback returns a Worker-variant callback addressed to this handler. +func (h *workerCallbackHandler) callback() *commonpb.Callback { + return &commonpb.Callback{ + Variant: &commonpb.Callback_Worker_{ + Worker: &commonpb.Callback_Worker{ + TaskQueueName: h.taskQueue, + Service: h.service, + Operation: h.operation, + SourceContext: h.sourceContext, + }, + }, + } +} + +// deliveries returns the completions the handler has received so far. +func (h *workerCallbackHandler) deliveries() []*notificationpb.OnCompleteRequest { + h.mu.Lock() + defer h.mu.Unlock() + return slices.Clone(h.received) +} + +// workerCallbackHandlers is the pair of handlers every execution type attaches a callback to: one +// that takes the completion on the first try, and one that has to be retried before it rejects it +// for good. +type workerCallbackHandlers struct { + succeeding *workerCallbackHandler + failing *workerCallbackHandler +} + +func newWorkerCallbackHandlers(t *testing.T, client sdkclient.Client) workerCallbackHandlers { + t.Helper() + return workerCallbackHandlers{ + succeeding: newSucceedingWorkerCallbackHandler(t, client, "succeeding"), + failing: newRetryThenFailWorkerCallbackHandler(t, client, "failing"), + } +} + +// callbacks returns the two callbacks to attach to an execution, in the order they are asserted on. +func (hs workerCallbackHandlers) callbacks() []*commonpb.Callback { + return []*commonpb.Callback{hs.succeeding.callback(), hs.failing.callback()} +} + +// requireRegistered asserts that the execution carries exactly the two Worker callbacks that were +// attached to it, keyed by the task queue each is addressed to. +func (hs workerCallbackHandlers) requireRegistered( + t require.TestingT, + cbs []observedCallback, +) map[string]observedCallback { + require.Len(t, cbs, 2) + + byTaskQueue := make(map[string]observedCallback, len(cbs)) + for _, cb := range cbs { + worker := cb.callback.GetWorker() + require.NotNil(t, worker, "callback should round-trip as the Worker variant") + byTaskQueue[worker.GetTaskQueueName()] = cb + } + require.Contains(t, byTaskQueue, hs.succeeding.taskQueue) + require.Contains(t, byTaskQueue, hs.failing.taskQueue) + return byTaskQueue +} + +// requireStandby asserts that both callbacks are registered on an execution that has not closed +// yet, so neither has been triggered. +func (hs workerCallbackHandlers) requireStandby(t *testing.T, describe describeCallbacksFn) { + t.Helper() + await.Require(testcontext.For(t), t, func(c *await.T) { + cbs, err := describe() + require.NoError(c, err) + for _, cb := range hs.requireRegistered(c, cbs) { + require.Equal(c, enumspb.CALLBACK_STATE_STANDBY, cb.state) + } + }, 15*time.Second, 200*time.Millisecond) +} + +// requireExecuted waits for both callbacks to be delivered and reach a terminal state, then asserts +// the outcome each handler drove: the succeeding one is done after a single attempt, and the +// retry-then-fail one is failed after exactly one retry, carrying the handler's own message. +// +// It returns the observed callbacks keyed by task queue, for assertions specific to an execution +// type. +func (hs workerCallbackHandlers) requireExecuted( + t *testing.T, + describe describeCallbacksFn, +) map[string]observedCallback { + t.Helper() + + var byTaskQueue map[string]observedCallback + await.Require(testcontext.For(t), t, func(c *await.T) { + cbs, err := describe() + require.NoError(c, err) + byTaskQueue = hs.requireRegistered(c, cbs) + require.Equal(c, enumspb.CALLBACK_STATE_SUCCEEDED, byTaskQueue[hs.succeeding.taskQueue].state) + require.Equal(c, enumspb.CALLBACK_STATE_FAILED, byTaskQueue[hs.failing.taskQueue].state) + }, 30*time.Second, 200*time.Millisecond) + + // The first handler took the completion on the first try, so there was nothing to retry. + succeeded := byTaskQueue[hs.succeeding.taskQueue] + require.EqualValues(t, 1, succeeded.attempt) + require.Nil(t, succeeded.lastAttemptFailure) + require.NotNil(t, succeeded.lastAttemptCompleteTime) + require.Len(t, hs.succeeding.deliveries(), 1) + + // The second handler rejected the first delivery retryably and the second permanently, so the + // callback was retried exactly once and then came to rest on the second answer. + failed := byTaskQueue[hs.failing.taskQueue] + require.EqualValues(t, 2, failed.attempt, "the callback should be retried exactly once") + require.Len(t, hs.failing.deliveries(), 2) + require.Equal(t, terminalDeliveryFailureMessage, failed.lastAttemptFailure.GetMessage()) + require.True(t, failed.lastAttemptFailure.GetApplicationFailureInfo().GetNonRetryable()) + require.NotNil(t, failed.lastAttemptCompleteTime) + require.Nil(t, failed.nextAttemptScheduleTime, "a failed callback is not scheduled for another attempt") + + // Every delivery carried the execution's outcome and the context its own callback was + // registered with. + for _, h := range []*workerCallbackHandler{hs.succeeding, hs.failing} { + for i, delivered := range h.deliveries() { + require.NotNil(t, delivered.GetSuccess(), "delivery %d to %s", i+1, h.taskQueue) + require.Nil(t, delivered.GetFailure(), "delivery %d to %s", i+1, h.taskQueue) + protorequire.ProtoEqual(t, h.sourceContext, delivered.GetSourceContext()) + } + } + + return byTaskQueue +} + +func TestWorkerCallbacks(t *testing.T) { + t.Parallel() + + t.Run("Workflow", testWorkerCallbackOnWorkflow) + t.Run("WorkflowUpdate", testWorkerCallbackOnWorkflowUpdate) + t.Run("StandaloneActivity", testWorkerCallbackOnStandaloneActivity) + t.Run("StandaloneNexusOperation", testWorkerCallbackOnStandaloneNexusOperation) +} + +// testWorkerCallbackOnWorkflow attaches Worker callbacks to a workflow execution via +// StartWorkflowExecution. +func testWorkerCallbackOnWorkflow(t *testing.T) { + t.Parallel() + + env := testcore.NewEnv(t, + testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true), + testcore.WithDynamicConfig(dynamicconfig.EnableCHASMCallbacks, true), + ) + ctx := testcontext.For(t) + + workflowType := "worker-callback-workflow" + env.SdkWorker().RegisterWorkflowWithOptions(func(ctx workflow.Context) error { + workflow.GetSignalChannel(ctx, "continue").Receive(ctx, nil) + return nil + }, workflow.RegisterOptions{Name: workflowType}) + + handlers := newWorkerCallbackHandlers(t, env.SdkClient()) + newStartRequest := func() *workflowservice.StartWorkflowExecutionRequest { + return &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: env.Namespace().String(), + WorkflowId: testcore.RandomizeStr("worker-callback-workflow"), + WorkflowType: &commonpb.WorkflowType{Name: workflowType}, + TaskQueue: &taskqueuepb.TaskQueue{Name: env.WorkerTaskQueue(), Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + Identity: t.Name(), + CompletionCallbacks: handlers.callbacks(), + } + } + + // With the setting at its Nexus-only default, the callbacks are rejected before the workflow is + // created. + _, err := env.FrontendClient().StartWorkflowExecution(ctx, newStartRequest()) + require.ErrorContains(t, err, workerCallbackNotEnabledErr) + + env.OverrideDynamicConfig(chasmworkflow.EnabledCallbackKinds, []string{"nexus", "worker"}) + + req := newStartRequest() + _, err = env.FrontendClient().StartWorkflowExecution(ctx, req) + require.NoError(t, err) + + describe := describeWorkflowCallbacks(ctx, env, req.WorkflowId, "") + + // Both callbacks are registered on the running workflow, and neither has been triggered yet. + handlers.requireStandby(t, describe) + + // Close the workflow, which triggers the callbacks. + require.NoError(t, env.SdkClient().SignalWorkflow(ctx, req.WorkflowId, "", "continue", nil)) + require.NoError(t, env.SdkClient().GetWorkflow(ctx, req.WorkflowId, "").Get(ctx, nil)) + + for _, cb := range handlers.requireExecuted(t, describe) { + require.NotNil(t, cb.trigger.GetWorkflowClosed(), + "callback should be triggered by the workflow closing") + } +} + +// testWorkerCallbackOnWorkflowUpdate attaches Worker callbacks to a workflow update via +// UpdateWorkflowExecution. +func testWorkerCallbackOnWorkflowUpdate(t *testing.T) { + t.Parallel() + + env := testcore.NewEnv(t, + testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true), + testcore.WithDynamicConfig(dynamicconfig.EnableCHASMCallbacks, true), + testcore.WithDynamicConfig(dynamicconfig.EnableWorkflowUpdateCallbacks, true), + ) + ctx := testcontext.For(t) + + const updateName = "update" + const workflowType = "worker-callback-update-workflow" + env.SdkWorker().RegisterWorkflowWithOptions(func(ctx workflow.Context) error { + if err := workflow.SetUpdateHandler(ctx, updateName, func(ctx workflow.Context) (string, error) { + return "updated", nil + }); err != nil { + return err + } + workflow.GetSignalChannel(ctx, "stop").Receive(ctx, nil) + return nil + }, workflow.RegisterOptions{Name: workflowType}) + + run, err := env.SdkClient().ExecuteWorkflow(ctx, sdkclient.StartWorkflowOptions{ + TaskQueue: env.WorkerTaskQueue(), + }, workflowType) + require.NoError(t, err) + + handlers := newWorkerCallbackHandlers(t, env.SdkClient()) + newUpdateRequest := func() *workflowservice.UpdateWorkflowExecutionRequest { + return &workflowservice.UpdateWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: run.GetID(), + RunId: run.GetRunID(), + }, + WaitPolicy: &updatepb.WaitPolicy{ + LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, + }, + Request: &updatepb.Request{ + Meta: &updatepb.Meta{UpdateId: uuid.NewString()}, + Input: &updatepb.Input{Name: updateName}, + RequestId: uuid.NewString(), + CompletionCallbacks: handlers.callbacks(), + }, + } + } + + // With the setting at its Nexus-only default, the callbacks are rejected before the update is + // admitted. + _, err = env.FrontendClient().UpdateWorkflowExecution(ctx, newUpdateRequest()) + require.ErrorContains(t, err, workerCallbackNotEnabledErr) + + env.OverrideDynamicConfig(chasmworkflow.EnabledCallbackKinds, []string{"nexus", "worker"}) + + // The update runs to completion, which triggers the callbacks. + updateResp, err := env.FrontendClient().UpdateWorkflowExecution(ctx, newUpdateRequest()) + require.NoError(t, err) + require.Equal(t, + enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, + updateResp.GetStage()) + + describe := describeWorkflowCallbacks(ctx, env, run.GetID(), run.GetRunID()) + + for _, cb := range handlers.requireExecuted(t, describe) { + require.NotNil(t, cb.trigger.GetUpdateWorkflowExecutionCompleted(), + "callback should be triggered by the update completing") + } + + require.NoError(t, env.SdkClient().SignalWorkflow(ctx, run.GetID(), run.GetRunID(), "stop", nil)) +} + +// testWorkerCallbackOnStandaloneActivity attaches Worker callbacks to a standalone activity via +// StartActivityExecution. +func testWorkerCallbackOnStandaloneActivity(t *testing.T) { + t.Parallel() + + env := testcore.NewEnv(t, + testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true), + testcore.WithDynamicConfig(chasmactivity.Enabled, true), + testcore.WithDynamicConfig(chasmactivity.EnableCallbacks, true), + ) + ctx := testcontext.For(t) + + activityID := testcore.RandomizeStr("worker-callback-activity") + taskQueue := testcore.RandomizeStr("worker-callback-activity-tq") + + handlers := newWorkerCallbackHandlers(t, env.SdkClient()) + newStartRequest := func() *workflowservice.StartActivityExecutionRequest { + return &workflowservice.StartActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + ActivityType: env.Tv().ActivityType(), + Identity: env.Tv().WorkerIdentity(), + Input: defaultInput, + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue}, + StartToCloseTimeout: durationpb.New(defaultStartToCloseTimeout), + RequestId: uuid.NewString(), + CompletionCallbacks: handlers.callbacks(), + } + } + + // With the setting at its Nexus-only default, the callbacks are rejected before the activity is + // created. + _, err := env.FrontendClient().StartActivityExecution(ctx, newStartRequest()) + require.ErrorContains(t, err, workerCallbackNotEnabledErr) + + env.OverrideDynamicConfig(chasmactivity.EnabledCallbackKinds, []string{"nexus", "worker"}) + + startResp, err := env.FrontendClient().StartActivityExecution(ctx, newStartRequest()) + require.NoError(t, err) + require.True(t, startResp.GetStarted()) + + describe := describeActivityCallbacks(ctx, env, activityID, startResp.GetRunId()) + + handlers.requireStandby(t, describe) + + // Close the activity, which triggers the callbacks. + pollResp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ + Namespace: env.Namespace().String(), + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue}, + Identity: defaultIdentity, + }) + require.NoError(t, err) + require.NotEmpty(t, pollResp.GetTaskToken()) + + _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + Namespace: env.Namespace().String(), + TaskToken: pollResp.GetTaskToken(), + Result: defaultResult, + Identity: defaultIdentity, + }) + require.NoError(t, err) + + handlers.requireExecuted(t, describe) +} + +// testWorkerCallbackOnStandaloneNexusOperation attaches Worker callbacks to a standalone Nexus +// operation via StartNexusOperationExecution. +func testWorkerCallbackOnStandaloneNexusOperation(t *testing.T) { + t.Parallel() + + // Unlike the other execution types, standalone Nexus operations accept no callback kinds by + // default, so the Nexus-only baseline the rejection below exercises has to be set explicitly. + env := newNexusTestEnv(t, true, + testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true), + testcore.WithDynamicConfig(dynamicconfig.EnableCHASMCallbacks, true), + testcore.WithDynamicConfig(nexusoperation.Enabled, true), + testcore.WithDynamicConfig(nexusoperation.EnabledCallbackKinds, []string{"nexus"}), + ) + ctx := testcontext.For(t) + + // The endpoint the operation itself runs against. Its result is what the callbacks carry to the + // handlers, and it completes the operation immediately, so the callbacks are triggered as soon + // as the operation is started. + endpointName := env.createSyncSuccessEndpoint(ctx, t, "operation-result") + + operationID := testcore.RandomizeStr("worker-callback-nexus-operation") + handlers := newWorkerCallbackHandlers(t, env.SdkClient()) + newStartRequest := func() *workflowservice.StartNexusOperationExecutionRequest { + return &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + RequestId: uuid.NewString(), + CompletionCallbacks: handlers.callbacks(), + } + } + + // With only the Nexus kind enabled, the callbacks are rejected before the operation is created. + _, err := env.startNexusOperation(ctx, newStartRequest()) + require.ErrorContains(t, err, workerCallbackNotEnabledErr) + + env.OverrideDynamicConfig(nexusoperation.EnabledCallbackKinds, []string{"nexus", "worker"}) + + startResp, err := env.startNexusOperation(ctx, newStartRequest()) + require.NoError(t, err) + require.True(t, startResp.GetStarted()) + + describe := describeNexusOperationCallbacks(ctx, env, operationID, startResp.GetRunId()) + + handlers.requireExecuted(t, describe) + + // Every callback on a standalone operation is triggered by the operation completing. The trigger + // is reported by a Nexus-operation-specific proto, so it is read here rather than through + // observedCallback. + cbInfos := env.describeNexusOperation(ctx, t, operationID).GetCompletionCallbacks() + require.Len(t, cbInfos, 2) + for _, cbInfo := range cbInfos { + require.NotNil(t, cbInfo.GetTrigger().GetOperationCompleted()) + } +} + +// TestWorkerCallbackDeliversFailedOutcome covers a failed execution: the failure reaches the handler +// in place of a result, and the callback carrying it still succeeds, since reporting a failure is a +// successful delivery. +func TestWorkerCallbackDeliversFailedOutcome(t *testing.T) { + t.Parallel() + + env := newNexusTestEnv(t, true, + testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true), + testcore.WithDynamicConfig(dynamicconfig.EnableCHASMCallbacks, true), + testcore.WithDynamicConfig(nexusoperation.Enabled, true), + testcore.WithDynamicConfig(nexusoperation.EnabledCallbackKinds, []string{"worker"}), + ) + ctx := testcontext.For(t) + + const operationFailure = "deliberate failure" + endpointName := env.createSyncFailureEndpoint(ctx, t, operationFailure) + + handler := newSucceedingWorkerCallbackHandler(t, env.SdkClient(), "succeeding") + operationID := testcore.RandomizeStr(t.Name()) + _, err := env.startNexusOperation(ctx, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + CompletionCallbacks: []*commonpb.Callback{handler.callback()}, + }) + require.NoError(t, err) + + // A failed operation does not make for a failed callback. + cbInfo := env.awaitCallbackInfo(ctx, t, operationID, enumspb.CALLBACK_STATE_SUCCEEDED) + require.NotNil(t, cbInfo.GetSuccess()) + + deliveries := handler.deliveries() + require.Len(t, deliveries, 1) + require.Nil(t, deliveries[0].GetSuccess()) + // The OperationError the server wraps the outcome in for transport is unwrapped before the + // handler sees it, so the endpoint's own failure is what arrives. + require.Equal(t, operationFailure, deliveries[0].GetFailure().GetCause().GetMessage()) + + // The operation itself is failed, and its outcome is the same failure the callback carried. + descResp := env.describeNexusOperation(ctx, t, operationID) + require.Equal(t, enumspb.NEXUS_OPERATION_EXECUTION_STATUS_FAILED, descResp.GetInfo().GetStatus()) + require.Equal(t, operationFailure, descResp.GetFailure().GetCause().GetMessage()) +} + +// TestOversizedWorkerCallbackFailsPermanently drives a completion past the 4 MiB gRPC servers accept +// by default, so matching rejects the dispatch with ResourceExhausted on receive. Those bytes are +// fixed — every retry sends the same ones — so the callback must fail rather than retry until it is +// abandoned, holding the task queue's circuit breaker open on the way. +func TestOversizedWorkerCallbackFailsPermanently(t *testing.T) { + t.Parallel() + + env := newNexusTestEnv(t, true, + testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true), + testcore.WithDynamicConfig(dynamicconfig.EnableCHASMCallbacks, true), + testcore.WithDynamicConfig(nexusoperation.Enabled, true), + testcore.WithDynamicConfig(nexusoperation.EnabledCallbackKinds, []string{"worker"}), + // Raise the source context caps so this test reaches the transport limit rather than the + // validator, whose aggregate cap of 2 MiB would otherwise reject the request first. + testcore.WithDynamicConfig(chasmcallback.WorkerSourceContextMaxSize, 8*1024*1024), + testcore.WithDynamicConfig(chasmcallback.WorkerSourceContextAggregateMaxSize, 8*1024*1024), + ) + ctx := testcontext.For(t) + + endpointName := env.createSyncSuccessEndpoint(ctx, t, "operation-result") + + handler := newSucceedingWorkerCallbackHandler(t, env.SdkClient(), "oversized") + // The dispatch carries the source context as json/protobuf, which base64s the bytes and so + // inflates them by about a third: 3.5 MiB here encodes to roughly 4.8 MiB, over the limit, while + // the start request carrying it stays under. + handler.sourceContext = &commonpb.Payload{Data: make([]byte, 3500*1024)} + + operationID := testcore.RandomizeStr(t.Name()) + _, err := env.startNexusOperation(ctx, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + CompletionCallbacks: []*commonpb.Callback{handler.callback()}, + }) + require.NoError(t, err) + + // The operation itself is unaffected by the callback it cannot deliver. + await.Require(ctx, t, func(c *await.T) { + status := env.describeNexusOperation(c.Context(), c, operationID).GetInfo().GetStatus() + require.Equal(c, enumspb.NEXUS_OPERATION_EXECUTION_STATUS_COMPLETED, status) + }, 20*time.Second, 100*time.Millisecond) + + // FAILED rather than BACKING_OFF is the whole point: the delivery is not retried. + cbInfo := env.awaitCallbackInfo(ctx, t, operationID, enumspb.CALLBACK_STATE_FAILED) + require.NotNil(t, cbInfo.GetFailure()) + // The size rejection describes the caller's own payload, so it is surfaced rather than blinded + // behind a reference ID. + require.Contains(t, cbInfo.GetFailure().GetMessage(), "larger than max") + require.NotContains(t, cbInfo.GetFailure().GetMessage(), "reference-id") + + // Matching never got the request, so the handler never ran. + require.Empty(t, handler.deliveries()) +} + +func describeWorkflowCallbacks(ctx context.Context, env *testcore.TestEnv, workflowID, runID string) describeCallbacksFn { + return func() ([]observedCallback, error) { + resp, err := env.SdkClient().DescribeWorkflowExecution(ctx, workflowID, runID) + if err != nil { + return nil, err + } + cbs := make([]observedCallback, 0, len(resp.GetCallbacks())) + for _, cb := range resp.GetCallbacks() { + cbs = append(cbs, observedCallback{ + callback: cb.GetCallback(), + state: cb.GetState(), + trigger: cb.GetTrigger(), + attempt: cb.GetAttempt(), + lastAttemptFailure: cb.GetLastAttemptFailure(), + lastAttemptCompleteTime: cb.GetLastAttemptCompleteTime(), + nextAttemptScheduleTime: cb.GetNextAttemptScheduleTime(), + }) + } + return cbs, nil + } +} + +func describeActivityCallbacks(ctx context.Context, env *testcore.TestEnv, activityID, runID string) describeCallbacksFn { + return func() ([]observedCallback, error) { + resp, err := env.FrontendClient().DescribeActivityExecution( + ctx, + &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + RunId: runID, + }) + if err != nil { + return nil, err + } + cbs := make([]observedCallback, 0, len(resp.GetCallbacks())) + for _, cb := range resp.GetCallbacks() { + cbs = append(cbs, observedCallback{ + callback: cb.GetInfo().GetCallback(), + state: cb.GetInfo().GetState(), + attempt: cb.GetInfo().GetAttempt(), + lastAttemptFailure: cb.GetInfo().GetLastAttemptFailure(), + lastAttemptCompleteTime: cb.GetInfo().GetLastAttemptCompleteTime(), + nextAttemptScheduleTime: cb.GetInfo().GetNextAttemptScheduleTime(), + }) + } + return cbs, nil + } +} + +func describeNexusOperationCallbacks(ctx context.Context, env *NexusTestEnv, operationID, runID string) describeCallbacksFn { + return func() ([]observedCallback, error) { + resp, err := env.FrontendClient().DescribeNexusOperationExecution( + ctx, + &workflowservice.DescribeNexusOperationExecutionRequest{ + Namespace: env.Namespace().String(), + OperationId: operationID, + RunId: runID, + }) + if err != nil { + return nil, err + } + cbs := make([]observedCallback, 0, len(resp.GetCompletionCallbacks())) + for _, cb := range resp.GetCompletionCallbacks() { + cbs = append(cbs, observedCallback{ + callback: cb.GetInfo().GetCallback(), + state: cb.GetInfo().GetState(), + attempt: cb.GetInfo().GetAttempt(), + lastAttemptFailure: cb.GetInfo().GetLastAttemptFailure(), + lastAttemptCompleteTime: cb.GetInfo().GetLastAttemptCompleteTime(), + nextAttemptScheduleTime: cb.GetInfo().GetNextAttemptScheduleTime(), + }) + } + return cbs, nil + } +} diff --git a/tests/nexus_standalone_callbacks_test.go b/tests/nexus_standalone_callbacks_test.go new file mode 100644 index 00000000000..179f09d0f03 --- /dev/null +++ b/tests/nexus_standalone_callbacks_test.go @@ -0,0 +1,455 @@ +package tests + +import ( + "context" + "io" + "testing" + + "github.com/nexus-rpc/sdk-go/nexus" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + nexusoperationpb "go.temporal.io/api/nexusoperation/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm/lib/callback" + "go.temporal.io/server/chasm/lib/nexusoperation" + "go.temporal.io/server/common/callbacks" + "go.temporal.io/server/common/dynamicconfig" + commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexustest" + "go.temporal.io/server/common/testing/parallelsuite" + "go.temporal.io/server/common/testing/protorequire" + "go.temporal.io/server/common/testing/testvars" + "go.temporal.io/server/tests/testcore" +) + +// NexusStandaloneCallbacksTestSuite covers standalone Nexus operation +// behavior with regard to completion callbacks. +type NexusStandaloneCallbacksTestSuite struct { + parallelsuite.Suite[*NexusStandaloneCallbacksTestSuite] +} + +func TestNexusStandaloneCallbacksTestSuite(t *testing.T) { + parallelsuite.Run(t, &NexusStandaloneCallbacksTestSuite{}) +} + +func (s *NexusStandaloneCallbacksTestSuite) newTestEnv(enableCallbacks bool) *NexusTestEnv { + var kinds []callbacks.Kind + if enableCallbacks { + kinds = append(kinds, callbacks.KindNexus) + } + + env := newNexusTestEnv(s.T(), true, + testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true), + testcore.WithDynamicConfig(nexusoperation.Enabled, true), + testcore.WithDynamicConfig(nexusoperation.EnabledCallbackKinds, kinds), + ) + if enableCallbacks { + env.OverrideDynamicConfig( + callback.AllowedAddresses, + []any{map[string]any{"Pattern": "*", "AllowInsecure": true}}, + ) + } + return env +} + +func (s *NexusStandaloneCallbacksTestSuite) TestCompletionCallbacks() { + env := s.newTestEnv(true) + alwaysSuccessEndpointName := env.createSyncSuccessEndpoint(s.Context(), s.T(), "operation-result") + + s.Run("DeliveredOnSuccess", func(s *NexusStandaloneCallbacksTestSuite) { + ctx := s.Context() + ch, callbackAddress := newNexusCompletionHandler(s.T()) + + operationID := testvars.New(s.T()).Any().String() + startResp, err := env.startNexusOperation(ctx, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: alwaysSuccessEndpointName, + CompletionCallbacks: []*commonpb.Callback{nexusCompletionCallback(callbackAddress)}, + }) + s.NoError(err) + s.True(startResp.GetStarted()) + + // Verify the callback was actually delivered with the operation's result. + var completionBody []byte + select { + case completion := <-ch.requestCh: + s.Equal(nexus.OperationStateSucceeded, completion.State) + s.Nil(completion.Error) + s.False(completion.StartTime.IsZero()) + s.False(completion.CloseTime.IsZero()) + body, readErr := io.ReadAll(completion.HTTPRequest.Body) + _ = completion.HTTPRequest.Body.Close() + s.NoError(readErr) + s.JSONEq(`"operation-result"`, string(body)) + completionBody = body + // The completion carries a back-link to the operation that produced it. + wantLink := commonnexus.ConvertLinkNexusOperationToNexusLink(&commonpb.Link_NexusOperation{ + Namespace: env.Namespace().String(), + OperationId: operationID, + RunId: startResp.GetRunId(), + }) + s.Require().Len(completion.Links, 1) + s.Equal(wantLink.URL.String(), completion.Links[0].URL.String()) + s.Equal(wantLink.Type, completion.Links[0].Type) + // Unblock CompleteOperation so it returns 200 OK to the callback library. + ch.requestCompleteCh <- nil + case <-ctx.Done(): + s.FailNow("timed out waiting for the completion callback") + } + + // Verify the operation is in completed state, and that the callback delivered the + // same result the operation. + descResp := env.describeNexusOperation(ctx, s.T(), operationID) + s.Equal(enumspb.NEXUS_OPERATION_EXECUTION_STATUS_COMPLETED, descResp.GetInfo().GetStatus()) + s.Equal(string(descResp.GetResult().GetData()), string(completionBody)) + + // Wait for the callback to complete and confirm it has a Success result. + cbInfo := env.awaitCallbackInfo(s.Context(), s.T(), operationID, enumspb.CALLBACK_STATE_SUCCEEDED) + s.NotNil(cbInfo.GetSuccess()) + s.Equal(callbackAddress, cbInfo.GetCallback().GetNexus().GetUrl()) + }) + + s.Run("DeliveredOnFailure", func(s *NexusStandaloneCallbacksTestSuite) { + ctx := s.Context() + alwaysFailingEndpointName := env.createRandomExternalNexusServer(ctx, s.T(), nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + return nil, &nexus.OperationError{ + State: nexus.OperationStateFailed, + Cause: &nexus.FailureError{Failure: nexus.Failure{Message: "deliberate failure"}}, + } + }, + }) + ch, callbackAddress := newNexusCompletionHandler(s.T()) + + operationID := testvars.New(s.T()).Any().String() + _, err := env.startNexusOperation(ctx, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: alwaysFailingEndpointName, + CompletionCallbacks: []*commonpb.Callback{nexusCompletionCallback(callbackAddress)}, + }) + s.NoError(err) + + // Verify the callback was actually delivered with failure state. + select { + case completion := <-ch.requestCh: + s.Equal(nexus.OperationStateFailed, completion.State) + s.False(completion.StartTime.IsZero()) + s.False(completion.CloseTime.IsZero()) + s.Require().NotNil(completion.Error) + var failureErr *nexus.FailureError + s.Require().ErrorAs(completion.Error.Cause, &failureErr) + // The handler's error is wrapped as an OperationError whose cause carries the original + // message, which is how a Nexus operation failure round-trips through a completion. + tFailure, convErr := commonnexus.NexusFailureToTemporalFailure(failureErr.Failure) + s.NoError(convErr) + s.Equal("OperationError", tFailure.GetApplicationFailureInfo().GetType()) + s.Equal("deliberate failure", tFailure.GetCause().GetMessage()) + ch.requestCompleteCh <- nil + case <-ctx.Done(): + s.FailNow("timed out waiting for the completion callback") + } + + // Verify the operation is in failed state. + descResp := env.describeNexusOperation(ctx, s.T(), operationID) + s.Equal(enumspb.NEXUS_OPERATION_EXECUTION_STATUS_FAILED, descResp.GetInfo().GetStatus()) + + // The operation may have failed, but the callback reporting that failure was successful. + cbInfo := env.awaitCallbackInfo(s.Context(), s.T(), operationID, enumspb.CALLBACK_STATE_SUCCEEDED) + s.NotNil(cbInfo.GetSuccess()) + }) + + // Verify that if the callback fails to be delivered for some reason, that the failure is + // persisted correctly and available from the Describe operation. + s.Run("CallbackDeliveryFailure", func(s *NexusStandaloneCallbacksTestSuite) { + ctx := s.Context() + ch, callbackAddress := newNexusCompletionHandler(s.T()) + + operationID := testvars.New(s.T()).Any().String() + _, err := env.startNexusOperation(ctx, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: alwaysSuccessEndpointName, + CompletionCallbacks: []*commonpb.Callback{nexusCompletionCallback(callbackAddress)}, + }) + s.NoError(err) + + // Simulate the completion handler returning a retryable error followed by an unretryable + // error. Confirm the SANO's CallbackInfo includes the terminal failure. + for deliveryAttempt := 1; deliveryAttempt <= 2; deliveryAttempt++ { + select { + case completion := <-ch.requestCh: + s.Equal(nexus.OperationStateSucceeded, completion.State) + if deliveryAttempt == 1 { + // While the handler is blocked here the callback is in flight: Describe reports + // it as scheduled, with no attempts recorded yet. + cbInfo := env.awaitCallbackInfo(s.Context(), s.T(), operationID, enumspb.CALLBACK_STATE_SCHEDULED) + s.EqualValues(0, cbInfo.GetAttempt()) + s.Nil(cbInfo.GetLastAttemptFailure()) + s.Nil(cbInfo.GetResult()) + + // Retryable error. + ch.requestCompleteCh <- nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "delivery #1") + } else { + // The second delivery attempt: Describe now describes the previous attempt. + cbInfo := env.awaitCallbackInfo(s.Context(), s.T(), operationID, enumspb.CALLBACK_STATE_SCHEDULED) + s.EqualValues(1, cbInfo.GetAttempt()) // 1 attempt so far, the 2nd is in-progress. + s.NotNil(cbInfo.GetLastAttemptFailure()) + s.Contains(cbInfo.GetLastAttemptFailure().GetMessage(), "delivery #1") + s.Nil(cbInfo.GetResult()) + + // Unretryable error. + ch.requestCompleteCh <- nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "delivery #2") + } + case <-ctx.Done(): + s.FailNow("timed out waiting for the completion callback") + } + } + + // A failed callback delivery does not affect the operation itself. + gotStatus := env.describeNexusOperation(ctx, s.T(), operationID).GetInfo().GetStatus() + s.Equal(enumspb.NEXUS_OPERATION_EXECUTION_STATUS_COMPLETED, gotStatus) + + // Verify the completion callback delivery has failed. + cbInfo := env.awaitCallbackInfo(s.Context(), s.T(), operationID, enumspb.CALLBACK_STATE_FAILED) + // Both the last delivery failure and the terminal failure come from delivery #2. + const lastDeliveryFailureMessage = "handler error (BAD_REQUEST): delivery #2" + s.NotNil(cbInfo.GetFailure()) + s.Equal(lastDeliveryFailureMessage, cbInfo.GetFailure().GetMessage()) + s.Equal(lastDeliveryFailureMessage, cbInfo.GetLastAttemptFailure().GetMessage()) + }) + + s.Run("DescribeReportsStandbyBeforeClose", func(s *NexusStandaloneCallbacksTestSuite) { + ctx := s.Context() + endpointName := env.createAsyncEndpoint(ctx, s.T()) + + operationID := testvars.New(s.T()).Any().String() + const startRequestID = "start-request-id" + _, err := env.startNexusOperation(ctx, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + RequestId: startRequestID, + CompletionCallbacks: []*commonpb.Callback{ + nexusCompletionCallback("http://localhost/cb1"), + nexusCompletionCallback("http://localhost/cb2"), + }, + }) + s.NoError(err) + + // The operation stays STARTED, so its callbacks stay in STANDBY with no result. + infos := env.describeNexusOperation(ctx, s.T(), operationID).GetCompletionCallbacks() + s.Len(infos, 2) + requestIDs := make(map[string]struct{}, len(infos)) + for _, info := range infos { + // Every callback on a standalone operation is triggered by the operation completing. + s.NotNil(info.GetTrigger().GetOperationCompleted()) + s.Equal(enumspb.CALLBACK_STATE_STANDBY, info.GetInfo().GetState()) + s.NotNil(info.GetInfo().GetRegistrationTime()) + s.Nil(info.GetInfo().GetResult()) + + // The idempotency token each delivery will carry. Server-generated per callback, so it + // is neither empty nor the request ID that attached them. + requestID := info.GetInfo().GetRequestId() + s.NotEmpty(requestID) + s.NotEqual(startRequestID, requestID) + requestIDs[requestID] = struct{}{} + } + s.Len(requestIDs, 2, "each callback must get its own request ID") + }) + + s.Run("AttachOnConflict", func(s *NexusStandaloneCallbacksTestSuite) { + ctx := s.Context() + endpointName := env.createAsyncEndpoint(ctx, s.T()) + + operationID := testvars.New(s.T()).Any().String() + startResp, err := env.startNexusOperation(ctx, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + RequestId: "first-request", + CompletionCallbacks: []*commonpb.Callback{nexusCompletionCallback("http://localhost/cb1")}, + }) + s.NoError(err) + s.True(startResp.GetStarted()) + + attachReq := &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + RequestId: "second-request", + CompletionCallbacks: []*commonpb.Callback{nexusCompletionCallback("http://localhost/cb2")}, + IdConflictPolicy: enumspb.NEXUS_OPERATION_ID_CONFLICT_POLICY_USE_EXISTING, + OnConflictOptions: &nexusoperationpb.OnConflictOptions{ + AttachRequestId: true, + AttachCompletionCallbacks: true, + }, + } + attachResp, err := env.startNexusOperation(ctx, attachReq) + s.NoError(err) + s.False(attachResp.GetStarted(), "the second request must not have created an operation") + s.Equal(startResp.GetRunId(), attachResp.GetRunId()) + + callbackInfos := env.describeNexusOperation(ctx, s.T(), operationID).GetCompletionCallbacks() + s.Require().Len(callbackInfos, 2) + s.Equal("http://localhost/cb1", callbackInfos[0].GetInfo().GetCallback().GetNexus().GetUrl()) + s.Equal("http://localhost/cb2", callbackInfos[1].GetInfo().GetCallback().GetNexus().GetUrl()) + s.Equal(enumspb.CALLBACK_STATE_STANDBY, callbackInfos[0].GetInfo().GetState()) + s.Equal(enumspb.CALLBACK_STATE_STANDBY, callbackInfos[1].GetInfo().GetState()) + + // Replaying the same attach must not duplicate any of the callbacks. + _, err = env.startNexusOperation(ctx, attachReq) + s.NoError(err) + newDescribeResp := env.describeNexusOperation(ctx, s.T(), operationID) + s.Len(newDescribeResp.GetCompletionCallbacks(), 2) + }) + + // Confirm that SANO fails with callback kinds that are not enabled. + s.Run("RejectNonNexusCallbacks", func(s *NexusStandaloneCallbacksTestSuite) { + for _, tc := range []struct { + name string + callback *commonpb.Callback + errMsg string + }{ + { + name: "worker", + callback: &commonpb.Callback{Variant: &commonpb.Callback_Worker_{Worker: &commonpb.Callback_Worker{ + TaskQueueName: "completions-task-queue", + Service: "HTTPAdapter", + Operation: "DeliverAsWebhook", + }}}, + // The callback is well-formed, but the Worker kind is not enabled. (See newTestEnv.) + errMsg: "worker callbacks are not enabled for this execution type", + }, + { + name: "unset", + callback: &commonpb.Callback{}, + errMsg: "unknown callback variant", + }, + } { + s.Run(tc.name, func(s *NexusStandaloneCallbacksTestSuite) { + resp, err := env.startNexusOperation(s.Context(), &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: testvars.New(s.T()).Any().String(), + Endpoint: alwaysSuccessEndpointName, + CompletionCallbacks: []*commonpb.Callback{tc.callback}, + }) + s.Nil(resp) + + var unimplementedErr *serviceerror.Unimplemented + s.ErrorAs(err, &unimplementedErr) + s.ErrorContains(err, tc.errMsg) + }) + } + }) + + // A link riding on a completion callback is validated like one attached to the request itself, + // including when the request carries no links of its own. + s.Run("RejectInvalidCallbackLinks", func(s *NexusStandaloneCallbacksTestSuite) { + cb := nexusCompletionCallback("http://localhost/cb") + cb.Links = []*commonpb.Link{{ + // A workflow_event link with none of its required fields set. + Variant: &commonpb.Link_WorkflowEvent_{WorkflowEvent: &commonpb.Link_WorkflowEvent{}}, + }} + + resp, err := env.startNexusOperation(s.Context(), &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: testvars.New(s.T()).Any().String(), + Endpoint: alwaysSuccessEndpointName, + CompletionCallbacks: []*commonpb.Callback{cb}, + }) + s.Nil(resp) + + var invalidArgErr *serviceerror.InvalidArgument + s.ErrorAs(err, &invalidArgErr) + s.ErrorContains(err, "workflow event link must not have an empty namespace field") + }) + + // Links and callbacks are attached by independent options, so a single request may carry both. + s.Run("AttachLinksAndCallbacksOnConflict", func(s *NexusStandaloneCallbacksTestSuite) { + ctx := s.Context() + t := s.T() + + endpointName := env.createRandomExternalNexusServer(s.Context(), t, nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + return &nexus.HandlerStartOperationResultAsync{OperationToken: "test-operation-token"}, nil + }, + }) + + firstLink := standaloneNexusTestLink(env, "first-wf") + secondLink := standaloneNexusTestLink(env, "second-wf") + + operationID := testvars.New(t).Any().String() + startResp, err := env.startNexusOperation(ctx, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + RequestId: "first-request", + CompletionCallbacks: []*commonpb.Callback{nexusCompletionCallback("http://localhost/cb1")}, + Links: []*commonpb.Link{firstLink}, + }) + s.NoError(err) + s.True(startResp.GetStarted()) + + attachResp, err := env.startNexusOperation(ctx, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + RequestId: "second-request", + CompletionCallbacks: []*commonpb.Callback{nexusCompletionCallback("http://localhost/cb2")}, + Links: []*commonpb.Link{secondLink}, + IdConflictPolicy: enumspb.NEXUS_OPERATION_ID_CONFLICT_POLICY_USE_EXISTING, + OnConflictOptions: &nexusoperationpb.OnConflictOptions{ + AttachRequestId: true, + AttachCompletionCallbacks: true, + AttachLinks: true, + }, + }) + s.NoError(err) + s.False(attachResp.GetStarted(), "the second request must not have created an operation") + + descResp, err := env.FrontendClient().DescribeNexusOperationExecution(s.Context(), &workflowservice.DescribeNexusOperationExecutionRequest{ + Namespace: env.Namespace().String(), + OperationId: operationID, + }) + s.NoError(err) + // Links are stored per request ID, so their relative order is non-deterministic. + protorequire.ProtoElementsMatch(t, + []*commonpb.Link{firstLink, secondLink}, + descResp.GetInfo().GetLinks()) + }) +} + +// TestCallbacksDisabled confirms the per-namespace feature flag gates the whole surface, including +// the on-conflict attach path. +func (s *NexusStandaloneCallbacksTestSuite) TestCallbacksDisabled() { + // Test environment with SANO not supporting completion callbacks. + env := s.newTestEnv(false) + endpointName := env.createRandomExternalNexusServer(s.Context(), s.T(), nexustest.Handler{}) + cbs := []*commonpb.Callback{nexusCompletionCallback("http://localhost/cb")} + + s.Run("StartWithCallbacksFails", func(s *NexusStandaloneCallbacksTestSuite) { + _, err := env.startNexusOperation(s.Context(), &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: testvars.New(s.T()).Any().String(), + Endpoint: endpointName, + CompletionCallbacks: cbs, + }) + s.ErrorContains(err, "completion callbacks are not enabled for this namespace") + }) + + s.Run("OnConflictAttachCallbacksFails", func(s *NexusStandaloneCallbacksTestSuite) { + _, err := env.startNexusOperation(s.Context(), &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: testvars.New(s.T()).Any().String(), + Endpoint: endpointName, + CompletionCallbacks: cbs, + IdConflictPolicy: enumspb.NEXUS_OPERATION_ID_CONFLICT_POLICY_USE_EXISTING, + OnConflictOptions: &nexusoperationpb.OnConflictOptions{ + AttachRequestId: true, + AttachCompletionCallbacks: true, + }, + }) + s.ErrorContains(err, "completion callbacks are not enabled for this namespace") + }) +} diff --git a/tests/nexus_standalone_test.go b/tests/nexus_standalone_test.go index 4daea03d738..fab45bce9dc 100644 --- a/tests/nexus_standalone_test.go +++ b/tests/nexus_standalone_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/google/uuid" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -15,6 +16,7 @@ import ( enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" nexuspb "go.temporal.io/api/nexus/v1" + nexusoperationpb "go.temporal.io/api/nexusoperation/v1" sdkpb "go.temporal.io/api/sdk/v1" "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" @@ -224,6 +226,143 @@ func (s *NexusStandaloneTestSuite) TestStartStandaloneNexusOperation() { }) } +// standaloneNexusTestLink returns a new workflow event link. +func standaloneNexusTestLink(env *NexusTestEnv, workflowID string) *commonpb.Link { + return &commonpb.Link{ + Variant: &commonpb.Link_WorkflowEvent_{ + WorkflowEvent: &commonpb.Link_WorkflowEvent{ + Namespace: env.Namespace().String(), + WorkflowId: workflowID, + RunId: "wf-run-id", + Reference: &commonpb.Link_WorkflowEvent_EventRef{ + EventRef: &commonpb.Link_WorkflowEvent_EventReference{ + EventId: 1, + EventType: enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, + }, + }, + }, + }, + } +} + +// TestStandaloneNexusOperationLinks covers links a caller attaches to a standalone Nexus operation, +// on start and via on_conflict_options, along with the limits enforced on both paths. +func (s *NexusStandaloneTestSuite) TestStandaloneNexusOperationLinks() { + ctx := context.Background() + env := s.newTestEnv() + + // The async operation remains STARTED, so it remains open for on-conflict attaches. + endpointName := env.createAsyncEndpoint(ctx, s.T()) + + // Calls Describe- and returns the attached links. + describeLinks := func(s *NexusStandaloneTestSuite, operationID string) []*commonpb.Link { + t := s.T() + t.Helper() + descResp, err := env.FrontendClient().DescribeNexusOperationExecution( + ctx, + &workflowservice.DescribeNexusOperationExecutionRequest{ + Namespace: env.Namespace().String(), + OperationId: operationID, + }) + require.NoError(t, err) + return descResp.GetInfo().GetLinks() + } + + s.Run("AttachedOnStart", func(s *NexusStandaloneTestSuite) { + t := s.T() + operationID := uuid.NewString() + links := []*commonpb.Link{ + standaloneNexusTestLink(env, "start-wf-1"), + standaloneNexusTestLink(env, "start-wf-2"), + } + + startResp, err := s.startNexusOperation(env, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + Links: links, + }) + s.NoError(err) + s.True(startResp.GetStarted()) + + gotLinks := describeLinks(s, operationID) + protorequire.ProtoSliceEqual(t, links, gotLinks) + }) + + s.Run("AttachLinksOnConflictMergesLinks", func(s *NexusStandaloneTestSuite) { + t := s.T() + operationID := uuid.NewString() + firstLink := standaloneNexusTestLink(env, "merge-wf-1") + secondLink := standaloneNexusTestLink(env, "merge-wf-2") + + startResp, err := s.startNexusOperation(env, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + RequestId: "first-request", + Links: []*commonpb.Link{firstLink}, + }) + s.NoError(err) + s.True(startResp.GetStarted()) + + // Call Start again with the same operation ID, with the conflict policy to + // use the existing SANO but attach links. + attachReq := &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + RequestId: "second-request", + Links: []*commonpb.Link{secondLink}, + IdConflictPolicy: enumspb.NEXUS_OPERATION_ID_CONFLICT_POLICY_USE_EXISTING, + OnConflictOptions: &nexusoperationpb.OnConflictOptions{ + AttachLinks: true, + }, + } + attachResp, err := s.startNexusOperation(env, attachReq) + s.NoError(err) + s.False(attachResp.GetStarted()) + s.Equal(startResp.GetRunId(), attachResp.GetRunId()) + + expected := []*commonpb.Link{firstLink, secondLink} + // Links are stored per request ID, so their relative order is non-deterministic. + gotLinks := describeLinks(s, operationID) + protorequire.ProtoElementsMatch(t, expected, gotLinks) + + // Replaying the same request must not duplicate the links it already attached. + _, err = s.startNexusOperation(env, attachReq) + s.NoError(err) + gotLinks = describeLinks(s, operationID) + protorequire.ProtoElementsMatch(t, expected, gotLinks) + }) + + s.Run("LinksIgnoredOnConflictWithoutAttachLinks", func(s *NexusStandaloneTestSuite) { + t := s.T() + operationID := uuid.NewString() + firstLink := standaloneNexusTestLink(env, "ignored-wf-1") + + _, err := s.startNexusOperation(env, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + RequestId: "first-request", + Links: []*commonpb.Link{firstLink}, + }) + s.NoError(err) + + _, err = s.startNexusOperation(env, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: operationID, + Endpoint: endpointName, + RequestId: "second-request", + Links: []*commonpb.Link{standaloneNexusTestLink(env, "ignored-wf-2")}, + IdConflictPolicy: enumspb.NEXUS_OPERATION_ID_CONFLICT_POLICY_USE_EXISTING, + // attach_links intentionally omitted — the second request's links must be dropped. + }) + s.NoError(err) + + gotLinks := describeLinks(s, operationID) + protorequire.ProtoSliceEqual(t, []*commonpb.Link{firstLink}, gotLinks) + }) + + // The per-request and per-execution link caps are covered by unit tests + // TestNewStandaloneOperationAttachesLinks and TestOperationAttachLinks. +} + func (s *NexusStandaloneTestSuite) TestDescribeStandaloneNexusOperation() { s.Run("NotFound", func(s *NexusStandaloneTestSuite) { env := s.newTestEnv() @@ -1203,7 +1342,11 @@ func (s *NexusStandaloneTestSuite) TestTerminateStandaloneNexusOperation() { s.Contains(err.Error(), "already terminated") }) - s.Run("AlreadyCanceled", func(s *NexusStandaloneTestSuite) { + // Covers a *pending cancellation request*, which leaves the operation open. Terminating an + // operation that already reached the terminal CANCELED status is rejected instead; that path + // requires a handler-side cancel completion, so it is covered by the unit test + // TestTerminateRejectedForClosedOperation. + s.Run("AfterCancelRequested", func(s *NexusStandaloneTestSuite) { env := s.newTestEnv() endpointName := env.createRandomNexusEndpoint(s.Context(), s.T()).GetSpec().GetName() @@ -1231,7 +1374,8 @@ func (s *NexusStandaloneTestSuite) TestTerminateStandaloneNexusOperation() { }) s.NoError(err) - // Verify state changed to terminated (terminate overrides cancel request). + // Verify state changed to terminated (terminate overrides cancel request). Because the + // operation does not support cancellation, and didn't process the cancellation request. descResp, err := env.FrontendClient().DescribeNexusOperationExecution(s.Context(), &workflowservice.DescribeNexusOperationExecutionRequest{ Namespace: env.Namespace().String(), OperationId: "test-op", diff --git a/tests/nexus_test_base.go b/tests/nexus_test_base.go index 02cdf3676d5..31b90f570aa 100644 --- a/tests/nexus_test_base.go +++ b/tests/nexus_test_base.go @@ -1,14 +1,17 @@ package tests import ( + "cmp" "context" "errors" "net/http/httptest" "testing" + "time" "github.com/google/uuid" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" + callbackpb "go.temporal.io/api/callback/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" nexuspb "go.temporal.io/api/nexus/v1" @@ -19,7 +22,9 @@ import ( cnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/nexus/nexustest" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/types/known/durationpb" ) type NexusTestEnv struct { @@ -34,6 +39,38 @@ func newNexusTestEnv(t *testing.T, useTemporalFailures bool, opts ...testcore.Te } } +// startNexusOperation starts a standalone Nexus operation, applying defaults for +// required fields tests usually don't care about. +func (env *NexusTestEnv) startNexusOperation( + ctx context.Context, + req *workflowservice.StartNexusOperationExecutionRequest, +) (*workflowservice.StartNexusOperationExecutionResponse, error) { + req.Namespace = cmp.Or(req.Namespace, env.Namespace().String()) + req.Service = cmp.Or(req.Service, "test-service") + req.Operation = cmp.Or(req.Operation, "test-operation") + req.RequestId = cmp.Or(req.RequestId, env.Tv().RequestID()) + if req.ScheduleToCloseTimeout == nil { + req.ScheduleToCloseTimeout = durationpb.New(10 * time.Minute) + } + + return env.FrontendClient().StartNexusOperationExecution(ctx, req) +} + +// describeNexusOperation describes a standalone Nexus operation by ID, including its outcome. +func (env *NexusTestEnv) describeNexusOperation( + ctx context.Context, + t require.TestingT, + operationID string, +) *workflowservice.DescribeNexusOperationExecutionResponse { + descResp, err := env.FrontendClient().DescribeNexusOperationExecution(ctx, &workflowservice.DescribeNexusOperationExecutionRequest{ + Namespace: env.Namespace().String(), + OperationId: operationID, + IncludeOutcome: true, + }) + require.NoError(t, err) + return descResp +} + func (env *NexusTestEnv) createNexusEndpoint(ctx context.Context, t *testing.T, name string, taskQueue string) *nexuspb.Endpoint { resp, err := env.OperatorClient().CreateNexusEndpoint(ctx, &operatorservice.CreateNexusEndpointRequest{ Spec: &nexuspb.EndpointSpec{ @@ -115,6 +152,76 @@ func (env *NexusTestEnv) dispatchByNamespaceAndTaskQueueURL(namespace string, ta }) } +// createSyncSuccessEndpoint registers an endpoint whose handler completes every operation +// synchronously with the supplied result payload. Shutdown as part with the test's Cleanup. +func (env *NexusTestEnv) createSyncSuccessEndpoint(ctx context.Context, t *testing.T, result string) string { + return env.createRandomExternalNexusServer(ctx, t, nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + return &nexus.HandlerStartOperationResultSync[any]{Value: result}, nil + }, + }) +} + +// createSyncFailureEndpoint registers an endpoint whose handler fails every operation +// synchronously with the supplied message. Shutdown as part with the test's Cleanup. +func (env *NexusTestEnv) createSyncFailureEndpoint(ctx context.Context, t *testing.T, message string) string { + return env.createRandomExternalNexusServer(ctx, t, nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + return nil, &nexus.OperationError{ + State: nexus.OperationStateFailed, + Cause: &nexus.FailureError{Failure: nexus.Failure{Message: message}}, + } + }, + }) +} + +// awaitCallbackInfo polls DescribeNexusOperationExecution until the operation's single completion +// callback reaches wantState, then returns it. +func (env *NexusTestEnv) awaitCallbackInfo( + ctx context.Context, + t testing.TB, + operationID string, + wantState enumspb.CallbackState, +) *callbackpb.CallbackInfo { + t.Helper() + + var cbInfo *callbackpb.CallbackInfo + await.Require(ctx, t, func(c *await.T) { + cbs := env.describeNexusOperation(c.Context(), c, operationID).GetCompletionCallbacks() + require.Len(c, cbs, 1) + cbInfo = cbs[0].GetInfo() + require.NotNil(c, cbInfo) + require.Equal(c, wantState, cbInfo.GetState()) + }, 10*time.Second, 100*time.Millisecond) + return cbInfo +} + +// createAsyncEndpoint registers an endpoint whose handler leaves every operation running async, +// so calls to the endpoint remain in the STARTED state until it is completed by other means. +// (e.g. the Nexus operation gets canceled or terminated.) +func (env *NexusTestEnv) createAsyncEndpoint(ctx context.Context, t *testing.T) string { + return env.createRandomExternalNexusServer(ctx, t, nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + return &nexus.HandlerStartOperationResultAsync{OperationToken: "test-operation-token"}, nil + }, + }) +} + // nexusTaskResponse represents a successful response from a nexus task handler. // A nil response indicates no response should be sent (e.g., handler timed out). type nexusTaskResponse struct { @@ -418,3 +525,11 @@ func newNexusCompletionHandler(t *testing.T) (*completionHandler, string) { }) return ch, srv.URL } + +// nexusCompletionCallback builds a Nexus-variant completion callback targeting url, typically the URL +// returned by [newNexusCompletionHandler]. +func nexusCompletionCallback(url string) *commonpb.Callback { + return &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{Url: url}}, + } +} diff --git a/tests/nexus_workflow_update_test.go b/tests/nexus_workflow_update_test.go index 8a98071aca4..22964e83b72 100644 --- a/tests/nexus_workflow_update_test.go +++ b/tests/nexus_workflow_update_test.go @@ -593,6 +593,83 @@ func (s *NexusWorkflowUpdateTestSuite) TestDescribeWorkflowShowsUpdateCallbacks( s.NoError(env.SdkClient().SignalWorkflow(ctx, run.GetID(), run.GetRunID(), "stop", nil)) } +func (s *NexusWorkflowUpdateTestSuite) TestWorkflowUpdateRejectsNonNexusCallbacks() { + env := newNexusTestEnv(s.T(), true, enableUpdateCallbacksOpts()...) + ctx := s.Context() + taskQueue := testcore.RandomizeStr(s.T().Name()) + + wf := newUpdateChildWorkflow(false) + s.startWorker(env, taskQueue, wf) + + run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{ + TaskQueue: taskQueue, + }, wf, "initial input") + s.NoError(err) + + tests := []struct { + Name string + Callback *commonpb.Callback + ErrMsg string + }{ + { + Name: "worker", + Callback: &commonpb.Callback{ + Variant: &commonpb.Callback_Worker_{ + Worker: &commonpb.Callback_Worker{ + TaskQueueName: "completions-task-queue", + Service: "HTTPAdapter", + Operation: "DeliverAsWebhook", + }, + }, + }, + // The validator rejects the Worker variant explicitly, before it reaches + // the unknown-variant fallback. + ErrMsg: "worker callbacks are not enabled for this execution type", + }, + { + Name: "nil variant", + Callback: &commonpb.Callback{}, + ErrMsg: "unknown callback variant", + }, + } + + // Not using subtests since the the test scenarios are all using the same + // workflow and test environment. + for _, test := range tests { + resp, err := env.FrontendClient().UpdateWorkflowExecution(ctx, &workflowservice.UpdateWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: run.GetID(), + RunId: run.GetRunID(), + }, + WaitPolicy: &updatepb.WaitPolicy{ + LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED, + }, + Request: &updatepb.Request{ + Meta: &updatepb.Meta{ + UpdateId: testcore.RandomizeStr("update-id"), + }, + Input: &updatepb.Input{ + Name: "update", + Args: &commonpb.Payloads{ + Payloads: []*commonpb.Payload{testcore.MustToPayload(s.T(), "test")}, + }, + }, + RequestId: uuid.NewString(), + CompletionCallbacks: []*commonpb.Callback{test.Callback}, + }, + }) + s.Nil(resp, test.Name) + s.ErrorContains(err, test.ErrMsg, test.Name) + } + + // The requested updates never reached the workflow. Signal it to stop gracefully. + s.NoError(env.SdkClient().SignalWorkflow(ctx, run.GetID(), run.GetRunID(), "stop", nil)) + var result string + s.NoError(run.Get(ctx, &result)) + s.Equal("done: initial input", result) +} + // TestWorkflowUpdateCallbackAfterResetInflightUpdate verifies that when a workflow is // reset while an update with completion callbacks is in-flight (accepted but not completed), // the update is reapplied in the new run and the callback fires when the update completes.