From 20e755a773ad6c6587ddf7d12c36f61f59e28816 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 2 Oct 2025 07:56:45 -0700 Subject: [PATCH 1/9] Update Workflow/Activity priority --- service/frontend/workflow_handler.go | 3 + service/frontend/workflow_handler_test.go | 101 ++ .../api/recordactivitytaskstarted/api.go | 7 +- service/history/api/startworkflow/api.go | 28 +- .../history/api/updateactivityoptions/api.go | 41 + .../api/updateactivityoptions/api_test.go | 113 ++- .../history/api/updateworkflowoptions/api.go | 46 +- .../api/updateworkflowoptions/api_test.go | 155 ++- .../history/historybuilder/event_factory.go | 36 +- .../history/historybuilder/history_builder.go | 28 +- .../history_builder_categorization_test.go | 2 +- service/history/interfaces/mutable_state.go | 22 +- .../history/interfaces/mutable_state_mock.go | 212 +---- service/history/ndc/events_reapplier_test.go | 2 +- service/history/ndc/workflow_resetter.go | 5 +- service/history/ndc/workflow_resetter_test.go | 47 +- service/history/workflow/activity.go | 1 + .../history/workflow/mutable_state_impl.go | 721 ++------------ .../workflow/mutable_state_impl_test.go | 887 ++---------------- service/matching/forwarder.go | 2 +- service/matching/pri_forwarder.go | 1 + tests/priority_fairness_test.go | 400 ++++---- 22 files changed, 834 insertions(+), 2026 deletions(-) diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index 041a8e91afd..b2b86366a60 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -5987,6 +5987,9 @@ func (wh *WorkflowHandler) UpdateActivityOptions( if request.GetActivity() == nil { return nil, errActivityIDOrTypeNotSet } + if err := priorities.Validate(request.GetActivityOptions().GetPriority()); err != nil { + return nil, err + } namespaceID, err := wh.namespaceRegistry.GetNamespaceID(namespace.Name(request.GetNamespace())) if err != nil { diff --git a/service/frontend/workflow_handler_test.go b/service/frontend/workflow_handler_test.go index 28097f59e51..9fce1ddaa0c 100644 --- a/service/frontend/workflow_handler_test.go +++ b/service/frontend/workflow_handler_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + activitypb "go.temporal.io/api/activity/v1" batchpb "go.temporal.io/api/batch/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" @@ -914,6 +915,31 @@ func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Failed_InvalidAggregat s.ErrorContains(err, "cannot attach more than 10 links per request, got 11") } +func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Priority() { + config := s.newConfig() + wh := s.getWorkflowHandler(config) + + s.mockSearchAttributesMapperProvider.EXPECT().GetMapper(gomock.Any()).Return(nil, nil).AnyTimes() + + request := &workflowservice.StartWorkflowExecutionRequest{ + Namespace: s.testNamespace.String(), + WorkflowId: "workflow-id", + WorkflowType: &commonpb.WorkflowType{ + Name: "workflow-type", + }, + TaskQueue: &taskqueuepb.TaskQueue{ + Name: "task-queue", + }, + Priority: &commonpb.Priority{PriorityKey: -1}, + } + + _, err := wh.StartWorkflowExecution(context.Background(), request) + var invalidArg *serviceerror.InvalidArgument + s.ErrorAs(err, &invalidArg) + s.ErrorContains(err, "PriorityKey can't be negative") + // NOTE: only testing a single validation scenario here; the priority validation has its own unit tests +} + func (s *WorkflowHandlerSuite) TestSignalWithStartWorkflowExecution_InvalidWorkflowIdConflictPolicy() { config := s.newConfig() wh := s.getWorkflowHandler(config) @@ -1011,6 +1037,32 @@ func (s *WorkflowHandlerSuite) TestSignalWithStartWorkflowExecution_Failed_Inval s.ErrorContains(err, "link exceeds allowed size of 4000") } +func (s *WorkflowHandlerSuite) TestSignalWithStartWorkflowExecution_Priority() { + config := s.newConfig() + wh := s.getWorkflowHandler(config) + + s.mockSearchAttributesMapperProvider.EXPECT().GetMapper(gomock.Any()).Return(nil, nil).AnyTimes() + + request := &workflowservice.SignalWithStartWorkflowExecutionRequest{ + Namespace: s.testNamespace.String(), + WorkflowId: "workflow-id", + WorkflowType: &commonpb.WorkflowType{ + Name: "workflow-type", + }, + TaskQueue: &taskqueuepb.TaskQueue{ + Name: "task-queue", + }, + SignalName: "signal-name", + Priority: &commonpb.Priority{PriorityKey: -1}, + } + + _, err := wh.SignalWithStartWorkflowExecution(context.Background(), request) + var invalidArg *serviceerror.InvalidArgument + s.ErrorAs(err, &invalidArg) + s.ErrorContains(err, "PriorityKey can't be negative") + // NOTE: only testing a single validation scenario here; the priority validation has its own unit tests +} + func (s *WorkflowHandlerSuite) TestSignalWorkflowExecution_Failed_InvalidLinks() { s.mockSearchAttributesMapperProvider.EXPECT().GetMapper(gomock.Any()).AnyTimes().Return(nil, nil) config := s.newConfig() @@ -3958,3 +4010,52 @@ func (s *WorkflowHandlerSuite) TestUpdateTaskQueueConfig_Validation() { s.NotNil(resp) }) } + +func (s *WorkflowHandlerSuite) TestUpdateWorkflowExecutionOptions_Priority() { + config := s.newConfig() + wh := s.getWorkflowHandler(config) + + request := &workflowservice.UpdateWorkflowExecutionOptionsRequest{ + Namespace: s.testNamespace.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: "workflow-id", + RunId: "run-id", + }, + WorkflowExecutionOptions: &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{PriorityKey: -1}, + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"priority"}}, + } + + _, err := wh.UpdateWorkflowExecutionOptions(context.Background(), request) + var invalidArg *serviceerror.InvalidArgument + s.ErrorAs(err, &invalidArg) + s.ErrorContains(err, "PriorityKey can't be negative") + // NOTE: only testing a single validation scenario here; the priority validation has its own unit tests +} + +func (s *WorkflowHandlerSuite) TestUpdateActivityOptions_Priority() { + config := s.newConfig() + wh := s.getWorkflowHandler(config) + + request := &workflowservice.UpdateActivityOptionsRequest{ + Namespace: s.testNamespace.String(), + Execution: &commonpb.WorkflowExecution{ + WorkflowId: "workflow-id", + RunId: "run-id", + }, + Activity: &workflowservice.UpdateActivityOptionsRequest_Id{ + Id: "activity-id", + }, + ActivityOptions: &activitypb.ActivityOptions{ + Priority: &commonpb.Priority{PriorityKey: -1}, + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"priority"}}, + } + + _, err := wh.UpdateActivityOptions(context.Background(), request) + var invalidArg *serviceerror.InvalidArgument + s.ErrorAs(err, &invalidArg) + s.ErrorContains(err, "PriorityKey can't be negative") + // NOTE: only testing a single validation scenario here; the priority validation has its own unit tests +} diff --git a/service/history/api/recordactivitytaskstarted/api.go b/service/history/api/recordactivitytaskstarted/api.go index 0c68a219994..4a8f083c721 100644 --- a/service/history/api/recordactivitytaskstarted/api.go +++ b/service/history/api/recordactivitytaskstarted/api.go @@ -165,12 +165,11 @@ func recordActivityTaskStarted( } if ai.Stamp != request.Stamp { - // activity has changes before task is started. - // ErrActivityStampMismatch is the error to indicate that requested activity has mismatched stamp + // This happens when the workflow task was rescheduled. errorMessage := fmt.Sprintf( - "Activity task with this stamp not found. Id: %s,: type: %s, current stamp: %d", + "Activity task rejected; stamp has changed. Id: %s,: type: %s, current stamp: %d", ai.ActivityId, ai.ActivityType.Name, ai.Stamp) - return nil, rejectCodeUndefined, serviceerror.NewNotFound(errorMessage) + return nil, rejectCodeUndefined, serviceerrors.NewObsoleteMatchingTask(errorMessage) } wfBehavior := mutableState.GetEffectiveVersioningBehavior() diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index 6b4bd314de6..d7b06478fc6 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -20,8 +20,8 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/persistence/visibility/manager" "go.temporal.io/server/common/primitives" - "go.temporal.io/server/common/softassert" "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/consts" @@ -54,6 +54,7 @@ type Starter struct { shardContext historyi.ShardContext workflowConsistencyChecker api.WorkflowConsistencyChecker tokenSerializer *tasktoken.Serializer + visibilityManager manager.VisibilityManager request *historyservice.StartWorkflowExecutionRequest namespace *namespace.Namespace createOrUpdateLeaseFn api.CreateOrUpdateLeaseFunc @@ -84,6 +85,7 @@ func NewStarter( shardContext historyi.ShardContext, workflowConsistencyChecker api.WorkflowConsistencyChecker, tokenSerializer *tasktoken.Serializer, + visibilityManager manager.VisibilityManager, request *historyservice.StartWorkflowExecutionRequest, createLeaseFn api.CreateOrUpdateLeaseFunc, ) (*Starter, error) { @@ -98,6 +100,7 @@ func NewStarter( shardContext: shardContext, workflowConsistencyChecker: workflowConsistencyChecker, tokenSerializer: tokenSerializer, + visibilityManager: visibilityManager, request: request, namespace: namespaceEntry, createOrUpdateLeaseFn: createLeaseFn, @@ -251,11 +254,7 @@ func (s *Starter) prepareNewWorkflow(workflowID string) (*creationParams, error) workflowTaskInfo := mutableState.GetStartedWorkflowTask() if s.requestEagerStart() && workflowTaskInfo == nil { - return nil, softassert.UnexpectedInternalErr( - s.shardContext.GetLogger(), - "unexpected error: mutable state did not have a started workflow task", - nil, - ) + return nil, serviceerror.NewInternal("unexpected error: mutable state did not have a started workflow task") } workflowSnapshot, eventBatches, err := mutableState.CloseTransactionAsSnapshot( historyi.TransactionPolicyActive, @@ -264,11 +263,7 @@ func (s *Starter) prepareNewWorkflow(workflowID string) (*creationParams, error) return nil, err } if len(eventBatches) != 1 { - return nil, softassert.UnexpectedInternalErr( - s.shardContext.GetLogger(), - "unable to create 1st event batch", - nil, - ) + return nil, serviceerror.NewInternal("unable to create 1st event batch") } return &creationParams{ @@ -670,7 +665,7 @@ func (s *Starter) handleUseExistingWorkflowOnConflictOptions( requestID, completionCallbacks, links, - "", + nil, ) return api.UpdateWorkflowWithoutWorkflowTask, err }, @@ -747,6 +742,15 @@ func (s *Starter) generateResponse( }, nil } + if err := api.ProcessOutgoingSearchAttributes( + shardCtx.GetSearchAttributesProvider(), + shardCtx.GetSearchAttributesMapperProvider(), + historyEvents, + s.namespace.Name(), + s.visibilityManager); err != nil { + return nil, err + } + clock, err := shardCtx.NewVectorClock() if err != nil { return nil, err diff --git a/service/history/api/updateactivityoptions/api.go b/service/history/api/updateactivityoptions/api.go index 7b1014c2526..14758086304 100644 --- a/service/history/api/updateactivityoptions/api.go +++ b/service/history/api/updateactivityoptions/api.go @@ -148,6 +148,7 @@ func processActivityOptionsUpdate( ScheduleToStartTimeout: ai.ScheduleToStartTimeout, StartToCloseTimeout: ai.StartToCloseTimeout, HeartbeatTimeout: ai.HeartbeatTimeout, + Priority: ai.Priority, RetryPolicy: &commonpb.RetryPolicy{ BackoffCoefficient: ai.RetryBackoffCoefficient, InitialInterval: ai.RetryInitialInterval, @@ -202,10 +203,48 @@ func mergeActivityOptions( mergeInto.HeartbeatTimeout = mergeFrom.HeartbeatTimeout } + if _, ok := updateFields["priority"]; ok { + mergeInto.Priority = mergeFrom.Priority + } + + if _, ok := updateFields["priority.priorityKey"]; ok { + if mergeFrom.Priority == nil { + return serviceerror.NewInvalidArgument("Priority is not provided") + } + if mergeInto.Priority == nil { + mergeInto.Priority = &commonpb.Priority{} + } + mergeInto.Priority.PriorityKey = mergeFrom.Priority.PriorityKey + } + + if _, ok := updateFields["priority.fairnessKey"]; ok { + if mergeFrom.Priority == nil { + return serviceerror.NewInvalidArgument("Priority is not provided") + } + if mergeInto.Priority == nil { + mergeInto.Priority = &commonpb.Priority{} + } + mergeInto.Priority.FairnessKey = mergeFrom.Priority.FairnessKey + } + + if _, ok := updateFields["priority.fairnessWeight"]; ok { + if mergeFrom.Priority == nil { + return serviceerror.NewInvalidArgument("Priority is not provided") + } + if mergeInto.Priority == nil { + mergeInto.Priority = &commonpb.Priority{} + } + mergeInto.Priority.FairnessWeight = mergeFrom.Priority.FairnessWeight + } + if mergeInto.RetryPolicy == nil { mergeInto.RetryPolicy = &commonpb.RetryPolicy{} } + if _, ok := updateFields["retryPolicy"]; ok { + mergeInto.RetryPolicy = mergeFrom.RetryPolicy + } + if _, ok := updateFields["retryPolicy.initialInterval"]; ok { if mergeFrom.RetryPolicy == nil { return serviceerror.NewInvalidArgument("RetryPolicy is not provided") @@ -295,6 +334,7 @@ func updateActivityOptions( activityInfo.ScheduleToStartTimeout = activityOptions.ScheduleToStartTimeout activityInfo.StartToCloseTimeout = activityOptions.StartToCloseTimeout activityInfo.HeartbeatTimeout = activityOptions.HeartbeatTimeout + activityInfo.Priority = activityOptions.Priority activityInfo.RetryMaximumInterval = activityOptions.RetryPolicy.MaximumInterval activityInfo.RetryBackoffCoefficient = activityOptions.RetryPolicy.BackoffCoefficient activityInfo.RetryInitialInterval = activityOptions.RetryPolicy.InitialInterval @@ -375,6 +415,7 @@ func restoreOriginalOptions( ScheduleToStartTimeout: originalOptions.ScheduleToStartTimeout, StartToCloseTimeout: originalOptions.StartToCloseTimeout, HeartbeatTimeout: originalOptions.HeartbeatTimeout, + Priority: originalOptions.Priority, RetryPolicy: originalOptions.RetryPolicy, } diff --git a/service/history/api/updateactivityoptions/api_test.go b/service/history/api/updateactivityoptions/api_test.go index 7f1aebd4059..0480abcd6f1 100644 --- a/service/history/api/updateactivityoptions/api_test.go +++ b/service/history/api/updateactivityoptions/api_test.go @@ -37,13 +37,17 @@ import ( ) func TestApplyActivityOptionsAcceptance(t *testing.T) { - - options := &activitypb.ActivityOptions{ + updateOptions := &activitypb.ActivityOptions{ TaskQueue: &taskqueuepb.TaskQueue{Name: "task_queue_name"}, ScheduleToCloseTimeout: durationpb.New(time.Second), StartToCloseTimeout: durationpb.New(time.Second), ScheduleToStartTimeout: durationpb.New(time.Second), HeartbeatTimeout: durationpb.New(time.Second), + Priority: &commonpb.Priority{ + PriorityKey: 42, + FairnessKey: "test_key", + FairnessWeight: 5.0, + }, RetryPolicy: &commonpb.RetryPolicy{ MaximumInterval: durationpb.New(time.Second), MaximumAttempts: 5, @@ -60,10 +64,10 @@ func TestApplyActivityOptionsAcceptance(t *testing.T) { mask *fieldmaskpb.FieldMask }{ { - name: "full mix - CamelCase", - mergeFrom: options, + name: "Top-level fields with CamelCase", + mergeFrom: updateOptions, mergeInto: &activitypb.ActivityOptions{}, - expected: options, + expected: updateOptions, mask: &fieldmaskpb.FieldMask{ Paths: []string{ "TaskQueue.Name", @@ -71,18 +75,16 @@ func TestApplyActivityOptionsAcceptance(t *testing.T) { "ScheduleToStartTimeout", "StartToCloseTimeout", "HeartbeatTimeout", - "RetryPolicy.BackoffCoefficient", - "RetryPolicy.InitialInterval", - "RetryPolicy.MaximumInterval", - "RetryPolicy.MaximumAttempts", + "Priority", + "RetryPolicy", }, }, }, { - name: "full mix - snake_case", - mergeFrom: options, + name: "Top-level fields with snake_case", + mergeFrom: updateOptions, mergeInto: &activitypb.ActivityOptions{}, - expected: options, + expected: updateOptions, mask: &fieldmaskpb.FieldMask{ Paths: []string{ "task_queue.name", @@ -90,38 +92,54 @@ func TestApplyActivityOptionsAcceptance(t *testing.T) { "schedule_to_start_timeout", "start_to_close_timeout", "heartbeat_timeout", - "retry_policy.backoff_coefficient", - "retry_policy.initial_interval", - "retry_policy.maximum_interval", - "retry_policy.maximum_attempts", + "priority", + "retry_policy", }, }, }, { - name: "partial", + name: "Sub-fields", mergeFrom: &activitypb.ActivityOptions{ - TaskQueue: &taskqueuepb.TaskQueue{Name: "task_queue_name"}, - ScheduleToCloseTimeout: durationpb.New(time.Second), - ScheduleToStartTimeout: durationpb.New(time.Second), + Priority: &commonpb.Priority{ + PriorityKey: 99, + FairnessKey: "newKey", + FairnessWeight: 7.5, + }, RetryPolicy: &commonpb.RetryPolicy{ - MaximumInterval: durationpb.New(time.Second), - MaximumAttempts: 5, + MaximumInterval: durationpb.New(time.Second), + MaximumAttempts: 5, + BackoffCoefficient: 1.0, + InitialInterval: durationpb.New(time.Second), }, }, mergeInto: &activitypb.ActivityOptions{ - StartToCloseTimeout: durationpb.New(time.Second), - HeartbeatTimeout: durationpb.New(time.Second), + Priority: &commonpb.Priority{ + PriorityKey: 10, + FairnessKey: "oldKey", + FairnessWeight: 1.0, + }, + RetryPolicy: &commonpb.RetryPolicy{}, + }, + expected: &activitypb.ActivityOptions{ + Priority: &commonpb.Priority{ + PriorityKey: 99, + FairnessKey: "newKey", + FairnessWeight: 7.5, + }, RetryPolicy: &commonpb.RetryPolicy{ + MaximumInterval: durationpb.New(time.Second), + MaximumAttempts: 5, BackoffCoefficient: 1.0, InitialInterval: durationpb.New(time.Second), }, }, - expected: options, mask: &fieldmaskpb.FieldMask{ Paths: []string{ - "task_queue.name", - "schedule_to_close_timeout", - "schedule_to_start_timeout", + "priority.priority_key", + "priority.fairness_key", + "priority.fairness_weight", + "retry_policy.backoff_coefficient", + "retry_policy.initial_interval", "retry_policy.maximum_interval", "retry_policy.maximum_attempts", }, @@ -145,7 +163,7 @@ func TestApplyActivityOptionsAcceptance(t *testing.T) { assert.Equal(t, tc.mergeInto.ScheduleToStartTimeout, tc.expected.ScheduleToStartTimeout, "ScheduleToStartTimeout") assert.Equal(t, tc.mergeInto.StartToCloseTimeout, tc.expected.StartToCloseTimeout, "StartToCloseTimeout") assert.Equal(t, tc.mergeInto.HeartbeatTimeout, tc.expected.HeartbeatTimeout, "HeartbeatTimeout") - + assert.Equal(t, tc.mergeInto.Priority, tc.expected.Priority, "Priority") } } @@ -153,23 +171,35 @@ func TestApplyActivityOptionsErrors(t *testing.T) { var err error err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"retry_policy.maximum_interval"}})) - assert.Error(t, err) + assert.ErrorContains(t, err, "RetryPolicy is not provided") err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"retry_policy.maximum_attempts"}})) - assert.Error(t, err) + assert.ErrorContains(t, err, "RetryPolicy is not provided") err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"retry_policy.backoff_coefficient"}})) - assert.Error(t, err) + assert.ErrorContains(t, err, "RetryPolicy is not provided") err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"retry_policy.initial_interval"}})) - assert.Error(t, err) + assert.ErrorContains(t, err, "RetryPolicy is not provided") err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"taskQueue.name"}})) - assert.Error(t, err) + assert.ErrorContains(t, err, "TaskQueue is not provided") + + err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, + util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"priority.priority_key"}})) + assert.ErrorContains(t, err, "Priority is not provided") + + err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, + util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"priority.fairness_key"}})) + assert.ErrorContains(t, err, "Priority is not provided") + + err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, + util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"priority.fairness_weight"}})) + assert.ErrorContains(t, err, "Priority is not provided") } @@ -180,6 +210,11 @@ func TestApplyActivityOptionsReset(t *testing.T) { ScheduleToStartTimeout: durationpb.New(time.Second), StartToCloseTimeout: durationpb.New(time.Second), HeartbeatTimeout: durationpb.New(time.Second), + Priority: &commonpb.Priority{ + PriorityKey: 42, + FairnessKey: "test_key", + FairnessWeight: 5.0, + }, RetryPolicy: &commonpb.RetryPolicy{ MaximumInterval: durationpb.New(time.Second), MaximumAttempts: 5, @@ -194,6 +229,9 @@ func TestApplyActivityOptionsReset(t *testing.T) { "schedule_to_start_timeout", "start_to_close_timeout", "heartbeat_timeout", + "priority.priority_key", + "priority.fairness_key", + "priority.fairness_weight", "retry_policy.backoff_coefficient", "retry_policy.initial_interval", "retry_policy.maximum_interval", @@ -205,6 +243,9 @@ func TestApplyActivityOptionsReset(t *testing.T) { err := mergeActivityOptions(options, &activitypb.ActivityOptions{ + Priority: &commonpb.Priority{ + PriorityKey: 10, + }, RetryPolicy: &commonpb.RetryPolicy{ MaximumAttempts: 5, BackoffCoefficient: 1.0, @@ -218,6 +259,10 @@ func TestApplyActivityOptionsReset(t *testing.T) { assert.Nil(t, options.StartToCloseTimeout) assert.Nil(t, options.HeartbeatTimeout) + assert.Equal(t, int32(10), options.Priority.PriorityKey) + assert.Equal(t, "", options.Priority.FairnessKey) + assert.Equal(t, float32(0), options.Priority.FairnessWeight) + assert.Nil(t, options.RetryPolicy.InitialInterval) assert.Nil(t, options.RetryPolicy.MaximumInterval) } diff --git a/service/history/api/updateworkflowoptions/api.go b/service/history/api/updateworkflowoptions/api.go index 0e225bf7e58..623c8d7219d 100644 --- a/service/history/api/updateworkflowoptions/api.go +++ b/service/history/api/updateworkflowoptions/api.go @@ -1,8 +1,10 @@ package updateworkflowoptions import ( + "cmp" "context" + commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/serviceerror" workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/server/api/historyservice/v1" @@ -104,8 +106,14 @@ func MergeAndApply( if mergedOpts.GetVersioningOverride() == nil { unsetOverride = true } - _, err = ms.AddWorkflowExecutionOptionsUpdatedEvent(mergedOpts.GetVersioningOverride(), unsetOverride, "", nil, nil, identity) - if err != nil { + if _, err = ms.AddWorkflowExecutionOptionsUpdatedEvent( + mergedOpts.GetVersioningOverride(), + unsetOverride, + "", + nil, + nil, + mergedOpts.GetPriority(), + ); err != nil { return nil, hasChanges, err } return mergedOpts, hasChanges, nil @@ -120,6 +128,11 @@ func getOptionsFromMutableState(ms historyi.MutableState) *workflowpb.WorkflowEx } opts.VersioningOverride = override } + if priority := ms.GetExecutionInfo().GetPriority(); priority != nil { + if prio, ok := proto.Clone(priority).(*commonpb.Priority); ok { + opts.Priority = prio + } + } return opts } @@ -133,10 +146,30 @@ func mergeWorkflowExecutionOptions( return nil, err } updateFields := util.ParseFieldMask(updateMask) - if _, ok := updateFields["versioningOverride"]; ok { - mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() + + // ==== Priority + + if _, ok := updateFields["priority"]; ok { + mergeInto.Priority = mergeFrom.GetPriority() + } + + if _, ok := updateFields["priority.priorityKey"]; ok { + mergeInto.Priority = cmp.Or(mergeInto.Priority, &commonpb.Priority{}) + mergeInto.Priority.PriorityKey = mergeFrom.GetPriority().GetPriorityKey() } + if _, ok := updateFields["priority.fairnessKey"]; ok { + mergeInto.Priority = cmp.Or(mergeInto.Priority, &commonpb.Priority{}) + mergeInto.Priority.FairnessKey = mergeFrom.Priority.GetFairnessKey() + } + + if _, ok := updateFields["priority.fairnessWeight"]; ok { + mergeInto.Priority = cmp.Or(mergeInto.Priority, &commonpb.Priority{}) + mergeInto.Priority.FairnessWeight = mergeFrom.Priority.GetFairnessWeight() + } + + // ==== VersioningOverride + if _, ok := updateFields["versioningOverride.deployment"]; ok { if _, ok := updateFields["versioningOverride.behavior"]; !ok { return nil, serviceerror.NewInvalidArgument("versioning_override fields must be updated together") @@ -150,5 +183,10 @@ func mergeWorkflowExecutionOptions( } mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() } + + if _, ok := updateFields["versioningOverride"]; ok { + mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() + } + return mergeInto, nil } diff --git a/service/history/api/updateworkflowoptions/api_test.go b/service/history/api/updateworkflowoptions/api_test.go index 951445c7b9e..b3d74a2dd77 100644 --- a/service/history/api/updateworkflowoptions/api_test.go +++ b/service/history/api/updateworkflowoptions/api_test.go @@ -14,7 +14,7 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/historyservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" - "go.temporal.io/server/chasm" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/cluster/clustertest" "go.temporal.io/server/common/locks" @@ -125,6 +125,147 @@ func TestMergeOptions_FooMask(t *testing.T) { assert.Error(t, err) } +func TestMergeOptions_Priority(t *testing.T) { + updateMask := &fieldmaskpb.FieldMask{Paths: []string{"priority"}} + + // Merge priority into empty options + merged, err := mergeWorkflowExecutionOptions( + emptyOptions, + &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{PriorityKey: 10}, + }, updateMask) + assert.NoError(t, err) + assert.Equal(t, int32(10), merged.Priority.PriorityKey) + + // Update priority with all fields set + merged, err = mergeWorkflowExecutionOptions( + merged, + &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{ + PriorityKey: 20, + FairnessKey: "key1", + FairnessWeight: 3.5, + }, + }, updateMask) + assert.NoError(t, err) + assert.Equal(t, int32(20), merged.Priority.PriorityKey) + assert.Equal(t, "key1", merged.Priority.FairnessKey) + assert.Equal(t, float32(3.5), merged.Priority.FairnessWeight) + + // Replace with partial priority and clear unspecified fields + merged, err = mergeWorkflowExecutionOptions( + merged, + &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{PriorityKey: 30}, + }, updateMask) + assert.NoError(t, err) + assert.Equal(t, int32(30), merged.Priority.PriorityKey) + assert.Equal(t, "", merged.Priority.FairnessKey) // cleared + assert.Equal(t, float32(0), merged.Priority.FairnessWeight) // cleared +} + +func TestMergeOptions_Priority_FairnessKey(t *testing.T) { + updateMask := &fieldmaskpb.FieldMask{Paths: []string{"priority.fairness_key"}} + + init := &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{ + PriorityKey: 50, + FairnessWeight: 2.5, + }, + } + + merged, err := mergeWorkflowExecutionOptions( + init, + &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{FairnessKey: "key1"}, + }, updateMask) + assert.NoError(t, err) + assert.Equal(t, "key1", merged.Priority.FairnessKey) + assert.Equal(t, int32(50), merged.Priority.PriorityKey) // preserved + assert.Equal(t, float32(2.5), merged.Priority.FairnessWeight) // preserved +} + +func TestMergeOptions_Priority_FairnessWeight(t *testing.T) { + updateMask := &fieldmaskpb.FieldMask{Paths: []string{"priority.fairness_weight"}} + + init := &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{ + PriorityKey: 75, + FairnessKey: "existingKey", + }, + } + + merged, err := mergeWorkflowExecutionOptions( + init, + &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{FairnessWeight: 5}, + }, updateMask) + assert.NoError(t, err) + assert.Equal(t, float32(5), merged.Priority.FairnessWeight) + assert.Equal(t, int32(75), merged.Priority.PriorityKey) // preserved + assert.Equal(t, "existingKey", merged.Priority.FairnessKey) // preserved +} + +func TestMergeOptions_Priority_PriorityKey(t *testing.T) { + priorityKeyMask := &fieldmaskpb.FieldMask{Paths: []string{"priority.priority_key"}} + + init := &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{ + FairnessKey: "key1", + FairnessWeight: 3.0, + }, + } + + merged, err := mergeWorkflowExecutionOptions( + init, + &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{PriorityKey: 100}, + }, priorityKeyMask) + assert.NoError(t, err) + assert.Equal(t, int32(100), merged.Priority.PriorityKey) + assert.Equal(t, "key1", merged.Priority.FairnessKey) // preserved + assert.Equal(t, float32(3.0), merged.Priority.FairnessWeight) // preserved +} + +func TestMergeOptions_Priority_ClearPriority(t *testing.T) { + updateMask := &fieldmaskpb.FieldMask{Paths: []string{"priority"}} + + init := &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{ + PriorityKey: 100, + FairnessKey: "key", + FairnessWeight: 5, + }, + } + + merged, err := mergeWorkflowExecutionOptions( + init, + &workflowpb.WorkflowExecutionOptions{ + Priority: nil, + }, updateMask) + assert.NoError(t, err) + assert.Nil(t, merged.Priority) +} + +func TestMergeOptions_Priority_MultipleFields(t *testing.T) { + allFieldsMask := &fieldmaskpb.FieldMask{ + Paths: []string{"priority.priority_key", "priority.fairness_key", "priority.fairness_weight"}} + + merged, err := mergeWorkflowExecutionOptions( + emptyOptions, + &workflowpb.WorkflowExecutionOptions{ + Priority: &commonpb.Priority{ + PriorityKey: 99, + FairnessKey: "testKey", + FairnessWeight: 7.5, + }, + }, allFieldsMask) + assert.NoError(t, err) + assert.Equal(t, int32(99), merged.Priority.PriorityKey) + assert.Equal(t, "testKey", merged.Priority.FairnessKey) + assert.Equal(t, float32(7.5), merged.Priority.FairnessWeight) +} + type ( // updateWorkflowOptionsSuite contains tests for the UpdateWorkflowOptions API. updateWorkflowOptionsSuite struct { @@ -178,7 +319,7 @@ func (s *updateWorkflowOptionsSuite) SetupTest() { s.currentContext.EXPECT().LoadMutableState(gomock.Any(), s.shardContext).Return(s.currentMutableState, nil) s.workflowCache = wcache.NewMockCache(s.controller) - s.workflowCache.EXPECT().GetOrCreateChasmExecution(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), chasm.WorkflowArchetypeID, locks.PriorityHigh). + s.workflowCache.EXPECT().GetOrCreateChasmEntity(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), chasmworkflow.Archetype, locks.PriorityHigh). Return(s.currentContext, wcache.NoopReleaseFn, nil) s.workflowConsistencyChecker = api.NewWorkflowConsistencyChecker( @@ -192,7 +333,6 @@ func (s *updateWorkflowOptionsSuite) TearDownTest() { } func (s *updateWorkflowOptionsSuite) TestInvoke_Success() { - expectedOverrideOptions := &workflowpb.WorkflowExecutionOptions{ VersioningOverride: &workflowpb.VersioningOverride{ Behavior: enumspb.VERSIONING_BEHAVIOR_PINNED, @@ -200,7 +340,14 @@ func (s *updateWorkflowOptionsSuite) TestInvoke_Success() { }, } s.currentMutableState.EXPECT().IsWorkflowExecutionRunning().Return(true) - s.currentMutableState.EXPECT().AddWorkflowExecutionOptionsUpdatedEvent(expectedOverrideOptions.VersioningOverride, false, "", nil, nil, "").Return(&historypb.HistoryEvent{}, nil) + s.currentMutableState.EXPECT().AddWorkflowExecutionOptionsUpdatedEvent( + expectedOverrideOptions.VersioningOverride, + false, + "", + nil, + nil, + nil, + ).Return(&historypb.HistoryEvent{}, nil) s.currentContext.EXPECT().UpdateWorkflowExecutionAsActive(gomock.Any(), s.shardContext).Return(nil) updateReq := &historyservice.UpdateWorkflowExecutionOptionsRequest{ diff --git a/service/history/historybuilder/event_factory.go b/service/history/historybuilder/event_factory.go index 86365c2ee57..e7576add50b 100644 --- a/service/history/historybuilder/event_factory.go +++ b/service/history/historybuilder/event_factory.go @@ -384,7 +384,7 @@ func (b *EventFactory) CreateWorkflowExecutionOptionsUpdatedEvent( attachRequestID string, attachCompletionCallbacks []*commonpb.Callback, links []*commonpb.Link, - identity string, + priority *commonpb.Priority, ) *historypb.HistoryEvent { event := b.createHistoryEvent(enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED, b.timeSource.Now()) event.Attributes = &historypb.HistoryEvent_WorkflowExecutionOptionsUpdatedEventAttributes{ @@ -393,7 +393,7 @@ func (b *EventFactory) CreateWorkflowExecutionOptionsUpdatedEvent( UnsetVersioningOverride: unsetVersioningOverride, AttachedRequestId: attachRequestID, AttachedCompletionCallbacks: attachCompletionCallbacks, - Identity: identity, + Priority: priority, }, } event.Links = links @@ -1009,38 +1009,6 @@ func (b *EventFactory) CreateChildWorkflowExecutionTimedOutEvent( return event } -func (b *EventFactory) CreateWorkflowExecutionPausedEvent( - identity string, - reason string, - requestID string, -) *historypb.HistoryEvent { - event := b.createHistoryEvent(enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_PAUSED, b.timeSource.Now()) - event.Attributes = &historypb.HistoryEvent_WorkflowExecutionPausedEventAttributes{ - WorkflowExecutionPausedEventAttributes: &historypb.WorkflowExecutionPausedEventAttributes{ - Identity: identity, - Reason: reason, - RequestId: requestID, - }, - } - return event -} - -func (b *EventFactory) CreateWorkflowExecutionUnpausedEvent( - identity string, - reason string, - requestID string, -) *historypb.HistoryEvent { - event := b.createHistoryEvent(enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UNPAUSED, b.timeSource.Now()) - event.Attributes = &historypb.HistoryEvent_WorkflowExecutionUnpausedEventAttributes{ - WorkflowExecutionUnpausedEventAttributes: &historypb.WorkflowExecutionUnpausedEventAttributes{ - Identity: identity, - Reason: reason, - RequestId: requestID, - }, - } - return event -} - func (b *EventFactory) createHistoryEvent( eventType enumspb.EventType, time time.Time, diff --git a/service/history/historybuilder/history_builder.go b/service/history/historybuilder/history_builder.go index 5df04bc736f..6935c7244b9 100644 --- a/service/history/historybuilder/history_builder.go +++ b/service/history/historybuilder/history_builder.go @@ -276,30 +276,6 @@ func (b *HistoryBuilder) AddWorkflowTaskFailedEvent( return event } -func (b *HistoryBuilder) AddWorkflowExecutionPausedEvent( - identity string, - reason string, - requestID string, -) *historypb.HistoryEvent { - event := b.CreateWorkflowExecutionPausedEvent(identity, reason, requestID) - // Mark the event as 'worker may ignore' so that older SDKs can safely ignore it. - event.WorkerMayIgnore = true - event, _ = b.add(event) - return event -} - -func (b *HistoryBuilder) AddWorkflowExecutionUnpausedEvent( - identity string, - reason string, - requestID string, -) *historypb.HistoryEvent { - event := b.CreateWorkflowExecutionUnpausedEvent(identity, reason, requestID) - // Mark the event as 'worker may ignore' so that older SDKs can safely ignore it. - event.WorkerMayIgnore = true - event, _ = b.add(event) - return event -} - func (b *HistoryBuilder) AddActivityTaskScheduledEvent( workflowTaskCompletedEventID int64, command *commandpb.ScheduleActivityTaskCommandAttributes, @@ -452,7 +428,7 @@ func (b *HistoryBuilder) AddWorkflowExecutionOptionsUpdatedEvent( attachRequestID string, attachCompletionCallbacks []*commonpb.Callback, links []*commonpb.Link, - identity string, + priority *commonpb.Priority, ) *historypb.HistoryEvent { event := b.EventFactory.CreateWorkflowExecutionOptionsUpdatedEvent( worker_versioning.ConvertOverrideToV32(versioningOverride), @@ -460,7 +436,7 @@ func (b *HistoryBuilder) AddWorkflowExecutionOptionsUpdatedEvent( attachRequestID, attachCompletionCallbacks, links, - identity, + priority, ) event, _ = b.EventStore.add(event) return event diff --git a/service/history/historybuilder/history_builder_categorization_test.go b/service/history/historybuilder/history_builder_categorization_test.go index 45275714f33..be945937182 100644 --- a/service/history/historybuilder/history_builder_categorization_test.go +++ b/service/history/historybuilder/history_builder_categorization_test.go @@ -208,7 +208,7 @@ func TestHistoryBuilder_FlushBufferToCurrentBatch(t *testing.T) { t.Errorf("expected 1 event in memBufferBatch got %d", len(hb.memBufferBatch)) } // add another event to memBufferBatch - hb.AddWorkflowExecutionOptionsUpdatedEvent(nil, false, "request-id-1", nil, nil, "") + hb.AddWorkflowExecutionOptionsUpdatedEvent(nil, false, "request-id-1", nil, nil, nil) if len(hb.memBufferBatch) != 2 { t.Errorf("expected 2 event in memBufferBatch got %d", len(hb.memBufferBatch)) } diff --git a/service/history/interfaces/mutable_state.go b/service/history/interfaces/mutable_state.go index 89d496d930e..c81b82d191c 100644 --- a/service/history/interfaces/mutable_state.go +++ b/service/history/interfaces/mutable_state.go @@ -23,8 +23,6 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" taskqueuespb "go.temporal.io/server/api/taskqueue/v1" workflowspb "go.temporal.io/server/api/workflow/v1" - "go.temporal.io/server/chasm" - chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" @@ -118,7 +116,7 @@ type ( attachRequestID string, attachCompletionCallbacks []*commonpb.Callback, links []*commonpb.Link, - identity string, + priority *commonpb.Priority, ) (*historypb.HistoryEvent, error) AddWorkflowExecutionUpdateAcceptedEvent(protocolInstanceID string, acceptedRequestMessageId string, acceptedRequestSequencingEventId int64, acceptedRequest *updatepb.Request) (*historypb.HistoryEvent, error) AddWorkflowExecutionUpdateCompletedEvent(acceptedEventID int64, updResp *updatepb.Response) (*historypb.HistoryEvent, error) @@ -215,10 +213,6 @@ type ( TaskQueueScheduleToStartTimeout(name string) (*taskqueuepb.TaskQueue, *durationpb.Duration) IsWorkflowExecutionRunning() bool - AddWorkflowExecutionPausedEvent(identity string, reason string, requestID string) (*historypb.HistoryEvent, error) - ApplyWorkflowExecutionPausedEvent(event *historypb.HistoryEvent) error - AddWorkflowExecutionUnpausedEvent(identity string, reason string, requestID string) (*historypb.HistoryEvent, error) - ApplyWorkflowExecutionUnpausedEvent(event *historypb.HistoryEvent) error IsResourceDuplicated(resourceDedupKey definition.DeduplicationID) bool IsWorkflowPendingOnWorkflowTaskBackoff() bool UpdateDuplicatedResource(resourceDedupKey definition.DeduplicationID) @@ -293,8 +287,6 @@ type ( AddTasks(tasks ...tasks.Task) PopTasks() map[tasks.Category][]tasks.Task - DeleteCHASMPureTasks(maxScheduledTime time.Time) - SetUpdateCondition(int64, int64) GetUpdateCondition() (int64, int64) @@ -302,11 +294,6 @@ type ( CheckSpeculativeWorkflowTaskTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) bool RemoveSpeculativeWorkflowTaskTimeoutTask() - SetWorkflowTaskScheduleToStartTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) - SetWorkflowTaskStartToCloseTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) - GetWorkflowTaskScheduleToStartTimeoutTask() *tasks.WorkflowTaskTimeoutTask - GetWorkflowTaskStartToCloseTimeoutTask() *tasks.WorkflowTaskTimeoutTask - IsDirty() bool IsTransitionHistoryEnabled() bool // StartTransaction sets up the mutable state for transacting. @@ -330,9 +317,6 @@ type ( IsWorkflow() bool ChasmTree() ChasmTree - ChasmEnabled() bool - ChasmWorkflowComponent(ctx context.Context) (*chasmworkflow.Workflow, chasm.MutableContext, error) - ChasmWorkflowComponentReadOnly(ctx context.Context) (*chasmworkflow.Workflow, chasm.Context, error) // NextTransitionCount returns the next state transition count from the state transition history. // If state transition history is empty (e.g. when disabled or fresh mutable state), returns 0. @@ -367,9 +351,7 @@ type ( // activities. // If there is a pending workflow task that is not started yet, it'll be rescheduled after // transition start. - StartDeploymentTransition(deployment *deploymentpb.Deployment, revisionNumber int64) error - GetVersioningRevisionNumber() int64 - SetVersioningRevisionNumber(revisionNumber int64) + StartDeploymentTransition(deployment *deploymentpb.Deployment) error AddReapplyCandidateEvent(event *historypb.HistoryEvent) GetReapplyCandidateEvents() []*historypb.HistoryEvent diff --git a/service/history/interfaces/mutable_state_mock.go b/service/history/interfaces/mutable_state_mock.go index ef97a9fd5e1..fd29ce5d197 100644 --- a/service/history/interfaces/mutable_state_mock.go +++ b/service/history/interfaces/mutable_state_mock.go @@ -31,8 +31,6 @@ import ( persistence "go.temporal.io/server/api/persistence/v1" taskqueue0 "go.temporal.io/server/api/taskqueue/v1" workflow0 "go.temporal.io/server/api/workflow/v1" - chasm "go.temporal.io/server/chasm" - workflow1 "go.temporal.io/server/chasm/lib/workflow" definition "go.temporal.io/server/common/definition" namespace "go.temporal.io/server/common/namespace" persistence0 "go.temporal.io/server/common/persistence" @@ -639,33 +637,18 @@ func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionCanceledEvent(arg0, } // AddWorkflowExecutionOptionsUpdatedEvent mocks base method. -func (m *MockMutableState) AddWorkflowExecutionOptionsUpdatedEvent(versioningOverride *workflow.VersioningOverride, unsetVersioningOverride bool, attachRequestID string, attachCompletionCallbacks []*common.Callback, links []*common.Link, identity string) (*history.HistoryEvent, error) { +func (m *MockMutableState) AddWorkflowExecutionOptionsUpdatedEvent(versioningOverride *workflow.VersioningOverride, unsetVersioningOverride bool, attachRequestID string, attachCompletionCallbacks []*common.Callback, links []*common.Link, priority *common.Priority) (*history.HistoryEvent, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddWorkflowExecutionOptionsUpdatedEvent", versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, identity) + ret := m.ctrl.Call(m, "AddWorkflowExecutionOptionsUpdatedEvent", versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, priority) ret0, _ := ret[0].(*history.HistoryEvent) ret1, _ := ret[1].(error) return ret0, ret1 } // AddWorkflowExecutionOptionsUpdatedEvent indicates an expected call of AddWorkflowExecutionOptionsUpdatedEvent. -func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionOptionsUpdatedEvent(versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, identity any) *gomock.Call { +func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionOptionsUpdatedEvent(versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, priority any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowExecutionOptionsUpdatedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowExecutionOptionsUpdatedEvent), versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, identity) -} - -// AddWorkflowExecutionPausedEvent mocks base method. -func (m *MockMutableState) AddWorkflowExecutionPausedEvent(identity, reason, requestID string) (*history.HistoryEvent, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddWorkflowExecutionPausedEvent", identity, reason, requestID) - ret0, _ := ret[0].(*history.HistoryEvent) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// AddWorkflowExecutionPausedEvent indicates an expected call of AddWorkflowExecutionPausedEvent. -func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionPausedEvent(identity, reason, requestID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowExecutionPausedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowExecutionPausedEvent), identity, reason, requestID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowExecutionOptionsUpdatedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowExecutionOptionsUpdatedEvent), versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, priority) } // AddWorkflowExecutionSignaled mocks base method. @@ -743,21 +726,6 @@ func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionTerminatedEvent(firs return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowExecutionTerminatedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowExecutionTerminatedEvent), firstEventID, reason, details, identity, deleteAfterTerminate, links) } -// AddWorkflowExecutionUnpausedEvent mocks base method. -func (m *MockMutableState) AddWorkflowExecutionUnpausedEvent(identity, reason, requestID string) (*history.HistoryEvent, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddWorkflowExecutionUnpausedEvent", identity, reason, requestID) - ret0, _ := ret[0].(*history.HistoryEvent) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// AddWorkflowExecutionUnpausedEvent indicates an expected call of AddWorkflowExecutionUnpausedEvent. -func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionUnpausedEvent(identity, reason, requestID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowExecutionUnpausedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowExecutionUnpausedEvent), identity, reason, requestID) -} - // AddWorkflowExecutionUpdateAcceptedEvent mocks base method. func (m *MockMutableState) AddWorkflowExecutionUpdateAcceptedEvent(protocolInstanceID, acceptedRequestMessageId string, acceptedRequestSequencingEventId int64, acceptedRequest *update.Request) (*history.HistoryEvent, error) { m.ctrl.T.Helper() @@ -1418,20 +1386,6 @@ func (mr *MockMutableStateMockRecorder) ApplyWorkflowExecutionOptionsUpdatedEven return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyWorkflowExecutionOptionsUpdatedEvent", reflect.TypeOf((*MockMutableState)(nil).ApplyWorkflowExecutionOptionsUpdatedEvent), event) } -// ApplyWorkflowExecutionPausedEvent mocks base method. -func (m *MockMutableState) ApplyWorkflowExecutionPausedEvent(event *history.HistoryEvent) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ApplyWorkflowExecutionPausedEvent", event) - ret0, _ := ret[0].(error) - return ret0 -} - -// ApplyWorkflowExecutionPausedEvent indicates an expected call of ApplyWorkflowExecutionPausedEvent. -func (mr *MockMutableStateMockRecorder) ApplyWorkflowExecutionPausedEvent(event any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyWorkflowExecutionPausedEvent", reflect.TypeOf((*MockMutableState)(nil).ApplyWorkflowExecutionPausedEvent), event) -} - // ApplyWorkflowExecutionSignaled mocks base method. func (m *MockMutableState) ApplyWorkflowExecutionSignaled(arg0 *history.HistoryEvent) error { m.ctrl.T.Helper() @@ -1488,20 +1442,6 @@ func (mr *MockMutableStateMockRecorder) ApplyWorkflowExecutionTimedoutEvent(arg0 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyWorkflowExecutionTimedoutEvent", reflect.TypeOf((*MockMutableState)(nil).ApplyWorkflowExecutionTimedoutEvent), arg0, arg1) } -// ApplyWorkflowExecutionUnpausedEvent mocks base method. -func (m *MockMutableState) ApplyWorkflowExecutionUnpausedEvent(event *history.HistoryEvent) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ApplyWorkflowExecutionUnpausedEvent", event) - ret0, _ := ret[0].(error) - return ret0 -} - -// ApplyWorkflowExecutionUnpausedEvent indicates an expected call of ApplyWorkflowExecutionUnpausedEvent. -func (mr *MockMutableStateMockRecorder) ApplyWorkflowExecutionUnpausedEvent(event any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyWorkflowExecutionUnpausedEvent", reflect.TypeOf((*MockMutableState)(nil).ApplyWorkflowExecutionUnpausedEvent), event) -} - // ApplyWorkflowExecutionUpdateAcceptedEvent mocks base method. func (m *MockMutableState) ApplyWorkflowExecutionUpdateAcceptedEvent(arg0 *history.HistoryEvent) error { m.ctrl.T.Helper() @@ -1640,20 +1580,6 @@ func (mr *MockMutableStateMockRecorder) AttachRequestID(requestID, eventType, ev return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttachRequestID", reflect.TypeOf((*MockMutableState)(nil).AttachRequestID), requestID, eventType, eventID) } -// ChasmEnabled mocks base method. -func (m *MockMutableState) ChasmEnabled() bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ChasmEnabled") - ret0, _ := ret[0].(bool) - return ret0 -} - -// ChasmEnabled indicates an expected call of ChasmEnabled. -func (mr *MockMutableStateMockRecorder) ChasmEnabled() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChasmEnabled", reflect.TypeOf((*MockMutableState)(nil).ChasmEnabled)) -} - // ChasmTree mocks base method. func (m *MockMutableState) ChasmTree() ChasmTree { m.ctrl.T.Helper() @@ -1668,38 +1594,6 @@ func (mr *MockMutableStateMockRecorder) ChasmTree() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChasmTree", reflect.TypeOf((*MockMutableState)(nil).ChasmTree)) } -// ChasmWorkflowComponent mocks base method. -func (m *MockMutableState) ChasmWorkflowComponent(ctx context.Context) (*workflow1.Workflow, chasm.MutableContext, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ChasmWorkflowComponent", ctx) - ret0, _ := ret[0].(*workflow1.Workflow) - ret1, _ := ret[1].(chasm.MutableContext) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// ChasmWorkflowComponent indicates an expected call of ChasmWorkflowComponent. -func (mr *MockMutableStateMockRecorder) ChasmWorkflowComponent(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChasmWorkflowComponent", reflect.TypeOf((*MockMutableState)(nil).ChasmWorkflowComponent), ctx) -} - -// ChasmWorkflowComponentReadOnly mocks base method. -func (m *MockMutableState) ChasmWorkflowComponentReadOnly(ctx context.Context) (*workflow1.Workflow, chasm.Context, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ChasmWorkflowComponentReadOnly", ctx) - ret0, _ := ret[0].(*workflow1.Workflow) - ret1, _ := ret[1].(chasm.Context) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// ChasmWorkflowComponentReadOnly indicates an expected call of ChasmWorkflowComponentReadOnly. -func (mr *MockMutableStateMockRecorder) ChasmWorkflowComponentReadOnly(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChasmWorkflowComponentReadOnly", reflect.TypeOf((*MockMutableState)(nil).ChasmWorkflowComponentReadOnly), ctx) -} - // CheckResettable mocks base method. func (m *MockMutableState) CheckResettable() error { m.ctrl.T.Helper() @@ -1842,18 +1736,6 @@ func (mr *MockMutableStateMockRecorder) CurrentVersionedTransition() *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CurrentVersionedTransition", reflect.TypeOf((*MockMutableState)(nil).CurrentVersionedTransition)) } -// DeleteCHASMPureTasks mocks base method. -func (m *MockMutableState) DeleteCHASMPureTasks(maxScheduledTime time.Time) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "DeleteCHASMPureTasks", maxScheduledTime) -} - -// DeleteCHASMPureTasks indicates an expected call of DeleteCHASMPureTasks. -func (mr *MockMutableStateMockRecorder) DeleteCHASMPureTasks(maxScheduledTime any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCHASMPureTasks", reflect.TypeOf((*MockMutableState)(nil).DeleteCHASMPureTasks), maxScheduledTime) -} - // DeleteSignalRequested mocks base method. func (m *MockMutableState) DeleteSignalRequested(requestID string) { m.ctrl.T.Helper() @@ -2690,20 +2572,6 @@ func (mr *MockMutableStateMockRecorder) GetUserTimerInfoByEventID(arg0 any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserTimerInfoByEventID", reflect.TypeOf((*MockMutableState)(nil).GetUserTimerInfoByEventID), arg0) } -// GetVersioningRevisionNumber mocks base method. -func (m *MockMutableState) GetVersioningRevisionNumber() int64 { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetVersioningRevisionNumber") - ret0, _ := ret[0].(int64) - return ret0 -} - -// GetVersioningRevisionNumber indicates an expected call of GetVersioningRevisionNumber. -func (mr *MockMutableStateMockRecorder) GetVersioningRevisionNumber() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVersioningRevisionNumber", reflect.TypeOf((*MockMutableState)(nil).GetVersioningRevisionNumber)) -} - // GetWorkflowCloseTime mocks base method. func (m *MockMutableState) GetWorkflowCloseTime(ctx context.Context) (time.Time, error) { m.ctrl.T.Helper() @@ -2777,34 +2645,6 @@ func (mr *MockMutableStateMockRecorder) GetWorkflowTaskByID(scheduledEventID any return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkflowTaskByID", reflect.TypeOf((*MockMutableState)(nil).GetWorkflowTaskByID), scheduledEventID) } -// GetWorkflowTaskScheduleToStartTimeoutTask mocks base method. -func (m *MockMutableState) GetWorkflowTaskScheduleToStartTimeoutTask() *tasks.WorkflowTaskTimeoutTask { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetWorkflowTaskScheduleToStartTimeoutTask") - ret0, _ := ret[0].(*tasks.WorkflowTaskTimeoutTask) - return ret0 -} - -// GetWorkflowTaskScheduleToStartTimeoutTask indicates an expected call of GetWorkflowTaskScheduleToStartTimeoutTask. -func (mr *MockMutableStateMockRecorder) GetWorkflowTaskScheduleToStartTimeoutTask() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkflowTaskScheduleToStartTimeoutTask", reflect.TypeOf((*MockMutableState)(nil).GetWorkflowTaskScheduleToStartTimeoutTask)) -} - -// GetWorkflowTaskStartToCloseTimeoutTask mocks base method. -func (m *MockMutableState) GetWorkflowTaskStartToCloseTimeoutTask() *tasks.WorkflowTaskTimeoutTask { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetWorkflowTaskStartToCloseTimeoutTask") - ret0, _ := ret[0].(*tasks.WorkflowTaskTimeoutTask) - return ret0 -} - -// GetWorkflowTaskStartToCloseTimeoutTask indicates an expected call of GetWorkflowTaskStartToCloseTimeoutTask. -func (mr *MockMutableStateMockRecorder) GetWorkflowTaskStartToCloseTimeoutTask() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkflowTaskStartToCloseTimeoutTask", reflect.TypeOf((*MockMutableState)(nil).GetWorkflowTaskStartToCloseTimeoutTask)) -} - // GetWorkflowType mocks base method. func (m *MockMutableState) GetWorkflowType() *common.WorkflowType { m.ctrl.T.Helper() @@ -3406,42 +3246,6 @@ func (mr *MockMutableStateMockRecorder) SetUpdateCondition(arg0, arg1 any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUpdateCondition", reflect.TypeOf((*MockMutableState)(nil).SetUpdateCondition), arg0, arg1) } -// SetVersioningRevisionNumber mocks base method. -func (m *MockMutableState) SetVersioningRevisionNumber(revisionNumber int64) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "SetVersioningRevisionNumber", revisionNumber) -} - -// SetVersioningRevisionNumber indicates an expected call of SetVersioningRevisionNumber. -func (mr *MockMutableStateMockRecorder) SetVersioningRevisionNumber(revisionNumber any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetVersioningRevisionNumber", reflect.TypeOf((*MockMutableState)(nil).SetVersioningRevisionNumber), revisionNumber) -} - -// SetWorkflowTaskScheduleToStartTimeoutTask mocks base method. -func (m *MockMutableState) SetWorkflowTaskScheduleToStartTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "SetWorkflowTaskScheduleToStartTimeoutTask", task) -} - -// SetWorkflowTaskScheduleToStartTimeoutTask indicates an expected call of SetWorkflowTaskScheduleToStartTimeoutTask. -func (mr *MockMutableStateMockRecorder) SetWorkflowTaskScheduleToStartTimeoutTask(task any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetWorkflowTaskScheduleToStartTimeoutTask", reflect.TypeOf((*MockMutableState)(nil).SetWorkflowTaskScheduleToStartTimeoutTask), task) -} - -// SetWorkflowTaskStartToCloseTimeoutTask mocks base method. -func (m *MockMutableState) SetWorkflowTaskStartToCloseTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "SetWorkflowTaskStartToCloseTimeoutTask", task) -} - -// SetWorkflowTaskStartToCloseTimeoutTask indicates an expected call of SetWorkflowTaskStartToCloseTimeoutTask. -func (mr *MockMutableStateMockRecorder) SetWorkflowTaskStartToCloseTimeoutTask(task any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetWorkflowTaskStartToCloseTimeoutTask", reflect.TypeOf((*MockMutableState)(nil).SetWorkflowTaskStartToCloseTimeoutTask), task) -} - // ShouldResetActivityTimerTaskMask mocks base method. func (m *MockMutableState) ShouldResetActivityTimerTaskMask(current, incoming *persistence.ActivityInfo) bool { m.ctrl.T.Helper() @@ -3457,17 +3261,17 @@ func (mr *MockMutableStateMockRecorder) ShouldResetActivityTimerTaskMask(current } // StartDeploymentTransition mocks base method. -func (m *MockMutableState) StartDeploymentTransition(arg0 *deployment.Deployment, revisionNumber int64) error { +func (m *MockMutableState) StartDeploymentTransition(arg0 *deployment.Deployment) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StartDeploymentTransition", arg0, revisionNumber) + ret := m.ctrl.Call(m, "StartDeploymentTransition", arg0) ret0, _ := ret[0].(error) return ret0 } // StartDeploymentTransition indicates an expected call of StartDeploymentTransition. -func (mr *MockMutableStateMockRecorder) StartDeploymentTransition(arg0, revisionNumber any) *gomock.Call { +func (mr *MockMutableStateMockRecorder) StartDeploymentTransition(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartDeploymentTransition", reflect.TypeOf((*MockMutableState)(nil).StartDeploymentTransition), arg0, revisionNumber) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartDeploymentTransition", reflect.TypeOf((*MockMutableState)(nil).StartDeploymentTransition), arg0) } // StartTransaction mocks base method. diff --git a/service/history/ndc/events_reapplier_test.go b/service/history/ndc/events_reapplier_test.go index ecd8a3a23de..b8928d44a32 100644 --- a/service/history/ndc/events_reapplier_test.go +++ b/service/history/ndc/events_reapplier_test.go @@ -111,7 +111,7 @@ func (s *nDCEventReapplicationSuite) TestReapplyEvents_AppliedEvent_WorkflowExec attr.GetAttachedRequestId(), attr.GetAttachedCompletionCallbacks(), event.Links, - event.GetWorkflowExecutionOptionsUpdatedEventAttributes().GetIdentity(), + attr.GetPriority(), ).Return(event, nil) msCurrent.EXPECT().HSM().Return(s.hsmNode).AnyTimes() msCurrent.EXPECT().IsWorkflowPendingOnWorkflowTaskBackoff().Return(true) diff --git a/service/history/ndc/workflow_resetter.go b/service/history/ndc/workflow_resetter.go index 4abf39ab31f..c7537b5b328 100644 --- a/service/history/ndc/workflow_resetter.go +++ b/service/history/ndc/workflow_resetter.go @@ -953,7 +953,7 @@ func reapplyEvents( requestID, callbacks, event.Links, - attr.GetIdentity(), + attr.Priority, ); err != nil { return reappliedEvents, err } @@ -1167,8 +1167,7 @@ func (r *workflowResetterImpl) performPostResetOperations(ctx context.Context, r for _, operation := range postResetOperations { switch op := operation.GetVariant().(type) { case *workflowpb.PostResetOperation_UpdateWorkflowOptions_: - // TODO(carlydf): Put the reset requester in the event so that with state-based replication this code will run on the passive side. - _, _, err := updateworkflowoptions.MergeAndApply(resetMS, op.UpdateWorkflowOptions.GetWorkflowExecutionOptions(), op.UpdateWorkflowOptions.GetUpdateMask(), "") + _, _, err := updateworkflowoptions.MergeAndApply(resetMS, op.UpdateWorkflowOptions.GetWorkflowExecutionOptions(), op.UpdateWorkflowOptions.GetUpdateMask()) if err != nil { return err } diff --git a/service/history/ndc/workflow_resetter_test.go b/service/history/ndc/workflow_resetter_test.go index 22b23e9d49e..74987e5f803 100644 --- a/service/history/ndc/workflow_resetter_test.go +++ b/service/history/ndc/workflow_resetter_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/google/uuid" + "github.com/pborman/uuid" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" commonpb "go.temporal.io/api/common/v1" @@ -21,7 +21,7 @@ import ( historyspb "go.temporal.io/server/api/history/v1" "go.temporal.io/server/api/historyservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" - "go.temporal.io/server/chasm" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common" "go.temporal.io/server/common/collection" "go.temporal.io/server/common/definition" @@ -113,9 +113,9 @@ func (s *workflowResetterSuite) SetupTest() { s.namespaceID = tests.NamespaceID s.workflowID = "some random workflow ID" - s.baseRunID = uuid.NewString() - s.currentRunID = uuid.NewString() - s.resetRunID = uuid.NewString() + s.baseRunID = uuid.New() + s.currentRunID = uuid.New() + s.resetRunID = uuid.New() } func (s *workflowResetterSuite) TearDownTest() { @@ -196,7 +196,6 @@ func (s *workflowResetterSuite) TestPersistToDB_CurrentTerminated() { s.mockTransaction.EXPECT().UpdateWorkflowExecution( gomock.Any(), persistence.UpdateWorkflowModeUpdateCurrent, - chasm.WorkflowArchetypeID, int64(0), currentMutation, currentEventsSeq, @@ -261,7 +260,6 @@ func (s *workflowResetterSuite) TestPersistToDB_CurrentNotTerminated() { s.mockTransaction.EXPECT().UpdateWorkflowExecution( gomock.Any(), persistence.UpdateWorkflowModeUpdateCurrent, - chasm.WorkflowArchetypeID, int64(0), currentMutation, currentEventsSeq, @@ -285,7 +283,7 @@ func (s *workflowResetterSuite) TestReplayResetWorkflow() { baseRebuildLastEventVersion := int64(12) resetBranchToken := []byte("some random reset branch token") - resetRequestID := uuid.NewString() + resetRequestID := uuid.New() resetHistorySize := int64(4411) resetMutableState := historyi.NewMockMutableState(s.controller) @@ -335,10 +333,10 @@ func (s *workflowResetterSuite) TestReplayResetWorkflow() { } func (s *workflowResetterSuite) TestFailWorkflowTask_NoWorkflowTask() { - baseRunID := uuid.NewString() + baseRunID := uuid.New() baseRebuildLastEventID := int64(1234) baseRebuildLastEventVersion := int64(5678) - resetRunID := uuid.NewString() + resetRunID := uuid.New() resetReason := "some random reset reason" mutableState := historyi.NewMockMutableState(s.controller) @@ -356,17 +354,17 @@ func (s *workflowResetterSuite) TestFailWorkflowTask_NoWorkflowTask() { } func (s *workflowResetterSuite) TestFailWorkflowTask_WorkflowTaskScheduled() { - baseRunID := uuid.NewString() + baseRunID := uuid.New() baseRebuildLastEventID := int64(1234) baseRebuildLastEventVersion := int64(5678) - resetRunID := uuid.NewString() + resetRunID := uuid.New() resetReason := "some random reset reason" mutableState := historyi.NewMockMutableState(s.controller) workflowTaskSchedule := &historyi.WorkflowTaskInfo{ ScheduledEventID: baseRebuildLastEventID - 12, StartedEventID: common.EmptyEventID, - RequestID: uuid.NewString(), + RequestID: uuid.New(), TaskQueue: &taskqueuepb.TaskQueue{ Name: "random task queue name", Kind: enumspb.TASK_QUEUE_KIND_NORMAL, @@ -413,17 +411,17 @@ func (s *workflowResetterSuite) TestFailWorkflowTask_WorkflowTaskScheduled() { } func (s *workflowResetterSuite) TestFailWorkflowTask_WorkflowTaskStarted() { - baseRunID := uuid.NewString() + baseRunID := uuid.New() baseRebuildLastEventID := int64(1234) baseRebuildLastEventVersion := int64(5678) - resetRunID := uuid.NewString() + resetRunID := uuid.New() resetReason := "some random reset reason" mutableState := historyi.NewMockMutableState(s.controller) workflowTask := &historyi.WorkflowTaskInfo{ ScheduledEventID: baseRebuildLastEventID - 12, StartedEventID: baseRebuildLastEventID - 10, - RequestID: uuid.NewString(), + RequestID: uuid.New(), TaskQueue: &taskqueuepb.TaskQueue{ Name: "random task queue name", Kind: enumspb.TASK_QUEUE_KIND_NORMAL, @@ -629,7 +627,7 @@ func (s *workflowResetterSuite) TestReapplyContinueAsNewWorkflowEvents_WithConti baseNextEventID := int64(456) baseBranchToken := []byte("some random base branch token") - newRunID := uuid.NewString() + newRunID := uuid.New() newFirstEventID := common.FirstEventID newNextEventID := int64(6) newBranchToken := []byte("some random new branch token") @@ -714,10 +712,10 @@ func (s *workflowResetterSuite) TestReapplyContinueAsNewWorkflowEvents_WithConti resetContext.EXPECT().Lock(gomock.Any(), locks.PriorityHigh).Return(nil) resetContext.EXPECT().Unlock() resetContext.EXPECT().IsDirty().Return(false).AnyTimes() + resetContext.EXPECT().SetArchetype(chasmworkflow.Archetype).Times(1) resetMutableState := historyi.NewMockMutableState(s.controller) resetContextCacheKey := wcache.Key{ WorkflowKey: definition.NewWorkflowKey(s.namespaceID.String(), s.workflowID, newRunID), - ArchetypeID: chasm.WorkflowArchetypeID, ShardUUID: s.mockShard.GetOwner(), } resetContext.EXPECT().LoadMutableState(gomock.Any(), s.mockShard).Return(resetMutableState, nil) @@ -758,7 +756,7 @@ func (s *workflowResetterSuite) TestReapplyWorkflowEvents() { nextEventID := int64(6) branchToken := []byte("some random branch token") - newRunID := uuid.NewString() + newRunID := uuid.New() event1 := &historypb.HistoryEvent{ EventId: 1, EventType: enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, @@ -828,8 +826,8 @@ func (s *workflowResetterSuite) TestReapplyEvents_WithPendingChildren() { testInitiatedEventID := int64(123) testChildWFType := &commonpb.WorkflowType{Name: "TEST-CHILD-WF-TYPE"} testChildWFExecution := &commonpb.WorkflowExecution{ - WorkflowId: uuid.NewString(), - RunId: uuid.NewString(), + WorkflowId: uuid.New(), + RunId: uuid.New(), } testStartEventHeader := &commonpb.Header{} @@ -1026,7 +1024,6 @@ func (s *workflowResetterSuite) TestReapplyEvents_WithNoPendingChildren() { } func (s *workflowResetterSuite) TestReapplyEvents() { - event1 := &historypb.HistoryEvent{ EventId: 101, EventType: enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED, @@ -1192,7 +1189,7 @@ func (s *workflowResetterSuite) TestReapplyEvents() { attr.GetAttachedRequestId(), attr.GetAttachedCompletionCallbacks(), event.Links, - attr.GetIdentity(), + attr.GetPriority(), ).Return(&historypb.HistoryEvent{}, nil) case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED: attr := event.GetWorkflowExecutionSignaledEventAttributes() @@ -1470,7 +1467,7 @@ func (s *workflowResetterSuite) TestWorkflowRestartAfterExecutionTimeout() { resetReason := "some random reset reason" resetBranchToken := []byte("some random reset branch token") - resetRequestID := uuid.NewString() + resetRequestID := uuid.New() resetHistorySize := int64(4411) resetMutableState := historyi.NewMockMutableState(s.controller) executionInfos := make(map[int64]*persistencespb.ChildExecutionInfo) @@ -1478,7 +1475,7 @@ func (s *workflowResetterSuite) TestWorkflowRestartAfterExecutionTimeout() { workflowTaskSchedule := &historyi.WorkflowTaskInfo{ ScheduledEventID: baseRebuildLastEventID - 12, StartedEventID: common.EmptyEventID, - RequestID: uuid.NewString(), + RequestID: uuid.New(), TaskQueue: &taskqueuepb.TaskQueue{ Name: "random task queue name", Kind: enumspb.TASK_QUEUE_KIND_NORMAL, diff --git a/service/history/workflow/activity.go b/service/history/workflow/activity.go index e4034d95b98..fa600268256 100644 --- a/service/history/workflow/activity.go +++ b/service/history/workflow/activity.go @@ -224,6 +224,7 @@ func GetPendingActivityInfo( ScheduleToStartTimeout: ai.ScheduleToStartTimeout, StartToCloseTimeout: ai.StartToCloseTimeout, HeartbeatTimeout: ai.HeartbeatTimeout, + Priority: ai.Priority, RetryPolicy: &commonpb.RetryPolicy{ InitialInterval: ai.RetryInitialInterval, diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 267b71cb21e..09125fbc2bf 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -11,8 +11,8 @@ import ( "strings" "time" - "github.com/google/uuid" "github.com/nexus-rpc/sdk-go/nexus" + "github.com/pborman/uuid" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" deploymentpb "go.temporal.io/api/deployment/v1" @@ -48,16 +48,13 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" - "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/payload" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/transitionhistory" "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/searchattribute" - "go.temporal.io/server/common/searchattribute/sadefs" serviceerrors "go.temporal.io/server/common/serviceerror" - "go.temporal.io/server/common/softassert" "go.temporal.io/server/common/util" "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/components/callbacks" @@ -216,33 +213,8 @@ type ( InsertTasks map[tasks.Category][]tasks.Task - // BestEffortDeleteTasks holds keys of history tasks to be deleted after a successful - // persistence update. This deletion is done on best effort basis. Persistence layer can ignore it without - // any errors. - BestEffortDeleteTasks map[tasks.Category][]tasks.Key - speculativeWorkflowTaskTimeoutTask *tasks.WorkflowTaskTimeoutTask - // In-memory storage for workflow task timeout tasks. These are set when timeout tasks are - // generated and used to delete them when the workflow task completes. Not persisted to storage. - wftScheduleToStartTimeoutTask *tasks.WorkflowTaskTimeoutTask - wftStartToCloseTimeoutTask *tasks.WorkflowTaskTimeoutTask - - // In-memory storage for CHASM pure tasks. These are set when CHASM pure tasks are generated and used to - // delete them when then are no longer needed. (i.e. when the task's scheduled time is after that of the - // earliest valid CHASM pure task's). - // - // Those pure tasks are mostly reverse ordered by their scheduled time (the VisibilityTimestamp field). - // Since a physical pure task is only generated when there's no other pure task with an earlier scheduled time, - // simply appending new pure tasks to the end of the slice maintains the order. - // - // NOTE: shard context may move those tasks' scheduled time to the future if they are earlier than the timer queue's - // max read level (otherwise those tasks won't be loaded), which may potentially break the reverse order. - // That is fine, however, as in the worst case we just delete fewer tasks than we could have, but we will never delete - // tasks that are still needed (all tasks deleted are those having an earlier scheduled time than what's needed). - // Task deletion is just a best-effort optimization after all, so not complicating the logic to account for that here. - chasmPureTasks []*tasks.ChasmTaskPure - // Do not rely on this, this is only updated on // Load() and closeTransactionXXX methods. So when // a transaction is in progress, this value will be @@ -327,7 +299,6 @@ func NewMutableState( namespaceEntry: namespaceEntry, appliedEvents: make(map[string]struct{}), InsertTasks: make(map[tasks.Category][]tasks.Task), - BestEffortDeleteTasks: make(map[tasks.Category][]tasks.Key), transitionHistoryEnabled: shard.GetConfig().EnableTransitionHistory(), visibilityUpdated: false, executionStateUpdated: false, @@ -532,7 +503,7 @@ func NewMutableStateFromDB( if shard.GetConfig().EnableChasm() { var err error - mutableState.chasmTree, err = chasm.NewTreeFromDB( + mutableState.chasmTree, err = chasm.NewTree( dbRecord.ChasmNodes, shard.ChasmRegistry(), shard.GetTimeSource(), @@ -620,7 +591,8 @@ func (ms *MutableStateImpl) mustInitHSM() { } func (ms *MutableStateImpl) IsWorkflow() bool { - return ms.chasmTree.ArchetypeID() == chasm.WorkflowArchetypeID + archetype := ms.chasmTree.Archetype() + return archetype == chasmworkflow.Archetype || archetype == "" } func (ms *MutableStateImpl) HSM() *hsm.Node { @@ -631,77 +603,13 @@ func (ms *MutableStateImpl) ChasmTree() historyi.ChasmTree { return ms.chasmTree } -// ChasmEnabled returns true if the mutable state has a real chasm tree. -// The chasmTree is initialized with a noopChasmTree which is then overwritten with an actual chasm tree if chasm is -// enabled when the mutable state is created. Once the EnableChasm dynamic config is removed and the tree is always -// initialized, this helper can be removed. -func (ms *MutableStateImpl) ChasmEnabled() bool { - _, isNoop := ms.chasmTree.(*noopChasmTree) - return !isNoop -} - -// chasmCallbacksEnabled returns true if CHASM callbacks are enabled for this workflow. -func (ms *MutableStateImpl) chasmCallbacksEnabled() bool { - if !ms.ChasmEnabled() { - return false - } - - // Check the callback library's EnableCallbacks config via history config - return ms.shard.GetConfig().EnableCHASMCallbacks(ms.GetNamespaceEntry().Name().String()) -} - -// ChasmWorkflowComponent gets the root workflow component from the CHASM tree. -// Returns the workflow component (which is *chasmworkflow.Workflow) and the CHASM mutable context. -// This method is for write operations. Callers can type assert to *chasmworkflow.Workflow if needed. -func (ms *MutableStateImpl) ChasmWorkflowComponent(ctx context.Context) (*chasmworkflow.Workflow, chasm.MutableContext, error) { - chasmCtx := chasm.NewMutableContext(ctx, ms.chasmTree.(*chasm.Node)) - rootComponent, err := ms.chasmTree.ComponentByPath(chasmCtx, nil) - if err != nil { - return nil, nil, err - } - wf, ok := rootComponent.(*chasmworkflow.Workflow) - if !ok { - return nil, nil, serviceerror.NewInternalf("expected workflow component, but got %T", rootComponent) - } - return wf, chasmCtx, nil -} - -func (ms *MutableStateImpl) ensureChasmWorkflowComponent(ctx context.Context) { - // Initialize chasm tree once for new workflows. - // Using context.Background() because this is done outside an actual request context and the - // chasmworkflow.NewWorkflow does not actually use it currently. - root, ok := ms.chasmTree.(*chasm.Node) - softassert.That(ms.logger, ok, "chasmTree cast failed") - - if root.ArchetypeID() == chasm.UnspecifiedArchetypeID { - mutableContext := chasm.NewMutableContext(ctx, root) - root.SetRootComponent(chasmworkflow.NewWorkflow(mutableContext, chasm.NewMSPointer(ms))) - } -} - -// ChasmWorkflowComponentReadOnly gets the root workflow component from the CHASM tree. -// Returns both the workflow component and a read-only CHASM context. -// This method is for read-only operations. -func (ms *MutableStateImpl) ChasmWorkflowComponentReadOnly(ctx context.Context) (*chasmworkflow.Workflow, chasm.Context, error) { - chasmCtx := chasm.NewContext(ctx, ms.chasmTree.(*chasm.Node)) - rootComponent, err := ms.chasmTree.ComponentByPath(chasmCtx, nil) - if err != nil { - return nil, nil, err - } - wf, ok := rootComponent.(*chasmworkflow.Workflow) - if !ok { - return nil, nil, serviceerror.NewInternalf("expected workflow component, but got %T", rootComponent) - } - return wf, chasmCtx, nil -} - // GetNexusCompletion converts a workflow completion event into a [nexus.OperationCompletion]. // Completions may be sent to arbitrary third parties, we intentionally do not include any termination reasons, and // expose only failure messages. func (ms *MutableStateImpl) GetNexusCompletion( ctx context.Context, requestID string, -) (nexusrpc.OperationCompletion, error) { +) (nexus.OperationCompletion, error) { ce, err := ms.GetCompletionEvent(ctx) if err != nil { return nil, err @@ -745,10 +653,9 @@ func (ms *MutableStateImpl) GetNexusCompletion( // Nexus does not support it. p = payloads[0] } - completion, err := nexusrpc.NewOperationCompletionSuccessful(p, nexusrpc.OperationCompletionSuccessfulOptions{ + completion, err := nexus.NewOperationCompletionSuccessful(p, nexus.OperationCompletionSuccessfulOptions{ Serializer: commonnexus.PayloadSerializer, StartTime: ms.executionState.GetStartTime().AsTime(), - CloseTime: ce.GetEventTime().AsTime(), Links: []nexus.Link{startLink}, }) if err != nil { @@ -760,11 +667,10 @@ func (ms *MutableStateImpl) GetNexusCompletion( if err != nil { return nil, err } - return nexusrpc.NewOperationCompletionUnsuccessful( + return nexus.NewOperationCompletionUnsuccessful( &nexus.OperationError{State: nexus.OperationStateFailed, Cause: &nexus.FailureError{Failure: f}}, - nexusrpc.OperationCompletionUnsuccessfulOptions{ + nexus.OperationCompletionUnsuccessfulOptions{ StartTime: ms.executionState.GetStartTime().AsTime(), - CloseTime: ce.GetEventTime().AsTime(), Links: []nexus.Link{startLink}, }) case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_CANCELED: @@ -779,14 +685,13 @@ func (ms *MutableStateImpl) GetNexusCompletion( if err != nil { return nil, err } - return nexusrpc.NewOperationCompletionUnsuccessful( + return nexus.NewOperationCompletionUnsuccessful( &nexus.OperationError{ State: nexus.OperationStateCanceled, Cause: &nexus.FailureError{Failure: f}, }, - nexusrpc.OperationCompletionUnsuccessfulOptions{ + nexus.OperationCompletionUnsuccessfulOptions{ StartTime: ms.executionState.GetStartTime().AsTime(), - CloseTime: ce.GetEventTime().AsTime(), Links: []nexus.Link{startLink}, }) case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_TERMINATED: @@ -799,11 +704,10 @@ func (ms *MutableStateImpl) GetNexusCompletion( if err != nil { return nil, err } - return nexusrpc.NewOperationCompletionUnsuccessful( + return nexus.NewOperationCompletionUnsuccessful( &nexus.OperationError{State: nexus.OperationStateFailed, Cause: &nexus.FailureError{Failure: f}}, - nexusrpc.OperationCompletionUnsuccessfulOptions{ + nexus.OperationCompletionUnsuccessfulOptions{ StartTime: ms.executionState.GetStartTime().AsTime(), - CloseTime: ce.GetEventTime().AsTime(), Links: []nexus.Link{startLink}, }) case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_TIMED_OUT: @@ -819,14 +723,13 @@ func (ms *MutableStateImpl) GetNexusCompletion( if err != nil { return nil, err } - return nexusrpc.NewOperationCompletionUnsuccessful( + return nexus.NewOperationCompletionUnsuccessful( &nexus.OperationError{ State: nexus.OperationStateFailed, Cause: &nexus.FailureError{Failure: f}, }, - nexusrpc.OperationCompletionUnsuccessfulOptions{ + nexus.OperationCompletionUnsuccessfulOptions{ StartTime: ms.executionState.GetStartTime().AsTime(), - CloseTime: ce.GetEventTime().AsTime(), Links: []nexus.Link{startLink}, }) } @@ -903,21 +806,8 @@ func (ms *MutableStateImpl) SetHistoryTree( ) error { // NOTE: Unfortunately execution timeout and run timeout are not yet initialized into ms.executionInfo at this point. // TODO: Consider explicitly initializing mutable state with these timeout parameters instead of passing them in. - workflowKey := ms.GetWorkflowKey() - - archetypeID := ms.ChasmTree().ArchetypeID() - if archetypeID != chasm.WorkflowArchetypeID { - return softassert.UnexpectedInternalErr( - ms.logger, - "Backfilling history not supported for non-workflow archetype", - nil, - tag.ArchetypeID(archetypeID), - tag.WorkflowNamespaceID(workflowKey.NamespaceID), - tag.WorkflowID(workflowKey.WorkflowID), - tag.WorkflowRunID(workflowKey.RunID), - ) - } + workflowKey := ms.GetWorkflowKey() var retentionDuration *durationpb.Duration if duration := ms.namespaceEntry.Retention(); duration > 0 { retentionDuration = durationpb.New(duration) @@ -2212,25 +2102,14 @@ func (ms *MutableStateImpl) IsTransientWorkflowTask() bool { func (ms *MutableStateImpl) ClearTransientWorkflowTask() error { if !ms.HasStartedWorkflowTask() { - return softassert.UnexpectedInternalErr( - ms.logger, - "cannot clear transient workflow task when task is missing", - nil, - ) + return serviceerror.NewInternal("cannot clear transient workflow task when task is missing") } if !ms.IsTransientWorkflowTask() { - return softassert.UnexpectedInternalErr( - ms.logger, - "cannot clear transient workflow task when task is not transient", - nil, - ) + return serviceerror.NewInternal("cannot clear transient workflow task when task is not transient") } + // this is transient workflow task if ms.HasBufferedEvents() { - return softassert.UnexpectedInternalErr( - ms.logger, - "cannot clear transient workflow task when there are buffered events", - nil, - ) + return serviceerror.NewInternal("cannot clear transient workflow task when there are buffered events") } // no buffered event emptyWorkflowTaskInfo := &historyi.WorkflowTaskInfo{ @@ -2394,7 +2273,6 @@ func (ms *MutableStateImpl) HasRequestID( } func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( - ctx context.Context, parentExecutionInfo *workflowspb.ParentExecutionInfo, execution *commonpb.WorkflowExecution, previousExecutionState historyi.MutableState, @@ -2435,7 +2313,7 @@ func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( // for other fields as well. runTimeout := command.GetWorkflowRunTimeout() - completionCallbacks, err := getCompletionCallbacksAsProtoSlice(ctx, previousExecutionState) + completionCallbacks, err := getCompletionCallbacksAsProtoSlice(previousExecutionState) if err != nil { return nil, err } @@ -2452,9 +2330,9 @@ func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( inheritedPinnedVersion = worker_versioning.ExternalWorkerDeploymentVersionFromDeployment(previousExecutionState.GetEffectiveDeployment()) newTQ := command.GetTaskQueue().GetName() if newTQ != previousExecutionInfo.GetTaskQueue() { - newTQInPinnedVersion, err = IsWFTaskQueueInVersionDetector(ctx, ms.GetNamespaceEntry().ID().String(), newTQ, inheritedPinnedVersion) + newTQInPinnedVersion, err = IsWFTaskQueueInVersionDetector(context.Background(), ms.GetNamespaceEntry().ID().String(), newTQ, inheritedPinnedVersion) if err != nil { - return nil, fmt.Errorf("error determining child task queue presence in inherited version: %w", err) + return nil, errors.New(fmt.Sprintf("error determining child task queue presence in inherited version: %s", err.Error())) } if !newTQInPinnedVersion { inheritedPinnedVersion = nil @@ -2472,35 +2350,8 @@ func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( } } - // New run initiated by ContinueAsNew of an AUTO_UPGRADE workflow execution will inherit the previous run's - // deployment version and revision number iff the new run's Task Queue belongs to source deployment version. - var sourceDeploymentVersion *deploymentpb.WorkerDeploymentVersion - var sourceDeploymentRevisionNumber int64 - if previousExecutionState.GetEffectiveVersioningBehavior() == enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE { - sourceDeploymentVersion = worker_versioning.ExternalWorkerDeploymentVersionFromDeployment(previousExecutionState.GetEffectiveDeployment()) - sourceDeploymentRevisionNumber = previousExecutionState.GetVersioningRevisionNumber() - - newTQ := command.GetTaskQueue().GetName() - if newTQ != previousExecutionInfo.GetTaskQueue() { - // Cross-TQ CAN: check if new TQ is in parent's deployment - TQInSourceDeploymentVersion, err := IsWFTaskQueueInVersionDetector( - ctx, - ms.GetNamespaceEntry().ID().String(), - newTQ, - sourceDeploymentVersion, - ) - if err != nil { - return nil, fmt.Errorf("error determining CAN task queue presence in auto upgrade deployment: %w", err) - } - if !TQInSourceDeploymentVersion { - sourceDeploymentVersion = nil - sourceDeploymentRevisionNumber = 0 - } - } - } - createRequest := &workflowservice.StartWorkflowExecutionRequest{ - RequestId: uuid.NewString(), + RequestId: uuid.New(), Namespace: ms.namespaceEntry.Name().String(), WorkflowId: execution.WorkflowId, TaskQueue: tq, @@ -2563,14 +2414,6 @@ func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( req.WorkflowExecutionExpirationTime = timestamppb.New(workflowTimeoutTime) } - // Add InheritedAutoUpgradeInfo if InheritedPinnedVersion is not set and source deployment version and revision number are set. - if sourceDeploymentVersion != nil && sourceDeploymentRevisionNumber != 0 && inheritedPinnedVersion == nil { - req.InheritedAutoUpgradeInfo = &deploymentpb.InheritedAutoUpgradeInfo{ - SourceDeploymentVersion: sourceDeploymentVersion, - SourceDeploymentRevisionNumber: sourceDeploymentRevisionNumber, - } - } - event, err := ms.AddWorkflowExecutionStartedEventWithOptions( execution, req, @@ -2901,163 +2744,19 @@ func (ms *MutableStateImpl) ApplyWorkflowExecutionStartedEvent( ms.approximateSize += ms.executionInfo.Size() ms.approximateSize += ms.executionState.Size() - // Populate the versioningInfo if the inheritedAutoUpgradeInfo is present. - if event.GetInheritedAutoUpgradeInfo() != nil { - ms.SetVersioningRevisionNumber(event.GetInheritedAutoUpgradeInfo().GetSourceDeploymentRevisionNumber()) - // TODO (Shivam): Remove this once you make SetDeploymentVersion and SetVersioningBehavior methods with nil checks - if ms.executionInfo.VersioningInfo == nil { - ms.executionInfo.VersioningInfo = &workflowpb.WorkflowExecutionVersioningInfo{} - } - ms.executionInfo.VersioningInfo.DeploymentVersion = event.GetInheritedAutoUpgradeInfo().GetSourceDeploymentVersion() - // Assume AutoUpgrade behavior for the first workflow task. - ms.executionInfo.VersioningInfo.Behavior = enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE - } - ms.writeEventToCache(startEvent) return nil } -func (ms *MutableStateImpl) AddWorkflowExecutionPausedEvent( - identity string, - reason string, - requestID string, -) (*historypb.HistoryEvent, error) { - opTag := tag.WorkflowActionWorkflowPaused - if err := ms.checkMutability(opTag); err != nil { - return nil, err - } - event := ms.hBuilder.AddWorkflowExecutionPausedEvent(identity, reason, requestID) - if err := ms.ApplyWorkflowExecutionPausedEvent(event); err != nil { - return nil, err - } - return event, nil -} - -// ApplyWorkflowExecutionPausedEvent applies the paused event to the mutable state. It updates the workflow execution status to paused and sets the pause info. -func (ms *MutableStateImpl) ApplyWorkflowExecutionPausedEvent(event *historypb.HistoryEvent) error { - // Update workflow status. - if _, err := ms.UpdateWorkflowStateStatus(ms.executionState.GetState(), enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED); err != nil { - return err - } - // Set pause info in mutable state. - ms.executionInfo.PauseInfo = &persistencespb.WorkflowPauseInfo{ - PauseTime: timestamppb.New(event.GetEventTime().AsTime()), - Identity: event.GetWorkflowExecutionPausedEventAttributes().GetIdentity(), - Reason: event.GetWorkflowExecutionPausedEventAttributes().GetReason(), - RequestId: event.GetWorkflowExecutionPausedEventAttributes().GetRequestId(), - } - - // Update approximate size of the mutable state. This will be decreased when the pause info is removed (when the workflow is unpaused) - ms.approximateSize += ms.executionInfo.PauseInfo.Size() - - // Invalidate all the pending activities. Do not mark individual activities as paused. - for _, ai := range ms.GetPendingActivityInfos() { - if err := ms.UpdateActivity(ai.ScheduledEventId, func(activityInfo *persistencespb.ActivityInfo, _ historyi.MutableState) error { - activityInfo.Stamp = activityInfo.Stamp + 1 - return nil - }); err != nil { - return err - } - } - - // Invalidate pending workflow task by incrementing the persisted stamp. - // This ensures subsequent task dispatch detects the change. - if ms.HasPendingWorkflowTask() { - ms.executionInfo.WorkflowTaskStamp += 1 - ms.workflowTaskManager.UpdateWorkflowTask(ms.GetPendingWorkflowTask()) - } - return nil -} - -func (ms *MutableStateImpl) AddWorkflowExecutionUnpausedEvent( - identity string, - reason string, - requestID string, -) (*historypb.HistoryEvent, error) { - opTag := tag.WorkflowActionWorkflowUnpaused - if err := ms.checkMutability(opTag); err != nil { - return nil, err - } - event := ms.hBuilder.AddWorkflowExecutionUnpausedEvent(identity, reason, requestID) - if err := ms.ApplyWorkflowExecutionUnpausedEvent(event); err != nil { - return nil, err - } - return event, nil -} - -// ApplyWorkflowExecutionUnpausedEvent applies the unpaused event to the mutable state. It updates the workflow execution status to running and clears the pause info. -func (ms *MutableStateImpl) ApplyWorkflowExecutionUnpausedEvent(event *historypb.HistoryEvent) error { - // Update workflow status. - if _, err := ms.UpdateWorkflowStateStatus(ms.executionState.GetState(), enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING); err != nil { - return err - } - - // save pauseInfoSize before clearing so that we can adjust approximate size later before returning success - pauseInfoSize := 0 - if ms.executionInfo.PauseInfo != nil { - pauseInfoSize = ms.GetExecutionInfo().GetPauseInfo().Size() - // Clear pause info in mutable state. - ms.executionInfo.PauseInfo = nil - } - - // Reschedule any pending activities - // Note: workflow task is scheduled in the unpause API. So no need to schedule it here. - for _, ai := range ms.GetPendingActivityInfos() { - // Bump activity stamp to force replication so that the passive cluster can recreate the activity task. - if err := ms.UpdateActivity(ai.ScheduledEventId, func(activityInfo *persistencespb.ActivityInfo, _ historyi.MutableState) error { - activityInfo.Stamp = activityInfo.Stamp + 1 - return nil - }); err != nil { - return err - } - - // Check activity scheduled time and generate activity retry task if scheduled time is in the future. - if ai.GetScheduledTime().AsTime().After(ms.timeSource.Now().UTC()) { - if err := ms.taskGenerator.GenerateActivityRetryTasks(ai); err != nil { - return err - } - } else { - // Generate activity task to resend the activity to matching immediately. - if err := ms.taskGenerator.GenerateActivityTasks(ai.ScheduledEventId); err != nil { - return err - } - } - } - - // Update approximate size of the mutable state. - ms.approximateSize -= pauseInfoSize - return nil -} - func (ms *MutableStateImpl) addCompletionCallbacks( event *historypb.HistoryEvent, requestID string, completionCallbacks []*commonpb.Callback, -) error { - if len(completionCallbacks) == 0 { - return nil - } - if ms.chasmCallbacksEnabled() { - // Initialize chasm tree once for new workflows. - // Using context.Background() because this is done outside an actual request context and the - // chasmworkflow.NewWorkflow does not actually use it currently. - ms.ensureChasmWorkflowComponent(context.Background()) - return ms.addCompletionCallbacksChasm(event, requestID, completionCallbacks) - } - - return ms.addCompletionCallbacksHsm(event, requestID, completionCallbacks) -} - -// addCompletionCallbacksHsm creates completion callbacks using the HSM implementation. -func (ms *MutableStateImpl) addCompletionCallbacksHsm( - event *historypb.HistoryEvent, - requestID string, - completionCallbacks []*commonpb.Callback, ) error { coll := callbacks.MachineCollection(ms.HSM()) maxCallbacksPerWorkflow := ms.config.MaxCallbacksPerWorkflow(ms.GetNamespaceEntry().Name().String()) if len(completionCallbacks)+coll.Size() > maxCallbacksPerWorkflow { - return serviceerror.NewFailedPreconditionf( + return serviceerror.NewInvalidArgumentf( "cannot attach more than %d callbacks to a workflow (%d callbacks already attached)", maxCallbacksPerWorkflow, coll.Size(), @@ -3075,8 +2774,11 @@ func (ms *MutableStateImpl) addCompletionCallbacksHsm( Header: variant.Nexus.GetHeader(), }, } - default: - return serviceerror.NewInvalidArgumentf("unknown callback variant: %T", cb.Variant) + case *commonpb.Callback_Internal_: + err := proto.Unmarshal(cb.GetInternal().GetData(), persistenceCB) + if err != nil { + return err + } } machine := callbacks.NewCallback(requestID, event.EventTime, callbacks.NewWorkflowClosedTrigger(), persistenceCB) id := "" @@ -3096,21 +2798,6 @@ func (ms *MutableStateImpl) addCompletionCallbacksHsm( return nil } -// addCompletionCallbacksChasm creates completion callbacks using the CHASM implementation. -func (ms *MutableStateImpl) addCompletionCallbacksChasm( - event *historypb.HistoryEvent, - requestID string, - completionCallbacks []*commonpb.Callback, -) error { - wf, ctx, err := ms.ChasmWorkflowComponent(context.Background()) - if err != nil { - return err - } - - maxCallbacksPerWorkflow := ms.config.MaxCHASMCallbacksPerWorkflow(ms.GetNamespaceEntry().Name().String()) - return wf.AddCompletionCallbacks(ctx, event.EventTime, requestID, completionCallbacks, maxCallbacksPerWorkflow) -} - // AddFirstWorkflowTaskScheduled adds the first workflow task scheduled event unless it should be delayed as indicated // by the startEvent's FirstWorkflowTaskBackoff. // Returns the workflow task's scheduled event ID if a task was scheduled, 0 otherwise. @@ -3235,10 +2922,10 @@ func (ms *MutableStateImpl) updateBinaryChecksumSearchAttribute() error { if exeInfo.SearchAttributes == nil { exeInfo.SearchAttributes = make(map[string]*commonpb.Payload, 1) } - if proto.Equal(exeInfo.SearchAttributes[sadefs.BinaryChecksums], checksumsPayload) { + if proto.Equal(exeInfo.SearchAttributes[searchattribute.BinaryChecksums], checksumsPayload) { return nil // unchanged } - ms.updateSearchAttributes(map[string]*commonpb.Payload{sadefs.BinaryChecksums: checksumsPayload}) + ms.updateSearchAttributes(map[string]*commonpb.Payload{searchattribute.BinaryChecksums: checksumsPayload}) return ms.taskGenerator.GenerateUpsertVisibilityTask() } @@ -3338,7 +3025,7 @@ func (ms *MutableStateImpl) ApplyBuildIdRedirect( ms.GetExecutionInfo().BuildIdRedirectCounter = redirectCounter // Re-scheduling pending workflow and activity tasks. - err = ms.reschedulePendingWorkflowTask() + err = ms.reschedulePendingWorkflowTask(false) if err != nil { return err } @@ -3352,16 +3039,6 @@ func (ms *MutableStateImpl) ApplyBuildIdRedirect( // TODO: skip task generation also when activity is in backoff period continue } - - // need to update stamp so the passive side regenerate the task - err := ms.UpdateActivity(ai.ScheduledEventId, func(info *persistencespb.ActivityInfo, state historyi.MutableState) error { - info.Stamp++ - return nil - }) - if err != nil { - return err - } - // we only need to resend the activities to matching, no need to update timer tasks. err = ms.taskGenerator.GenerateActivityTasks(ai.ScheduledEventId) if err != nil { @@ -3416,7 +3093,7 @@ func (ms *MutableStateImpl) loadBuildIds() ([]string, error) { if searchAttributes == nil { return []string{}, nil } - saPayload, found := searchAttributes[sadefs.BuildIds] + saPayload, found := searchAttributes[searchattribute.BuildIds] if !found { return []string{}, nil } @@ -3524,7 +3201,7 @@ func (ms *MutableStateImpl) saveBuildIds(buildIds []string, maxSearchAttributeVa return err } if len(buildIds) == 0 || len(saPayload.GetData()) <= maxSearchAttributeValueSize { - ms.updateSearchAttributes(map[string]*commonpb.Payload{sadefs.BuildIds: saPayload}) + ms.updateSearchAttributes(map[string]*commonpb.Payload{searchattribute.BuildIds: saPayload}) break } if len(buildIds) == 1 { @@ -3549,21 +3226,21 @@ func (ms *MutableStateImpl) saveDeploymentSearchAttributes(deployment, version, return err } if len(deploymentPayload.GetData()) <= maxSearchAttributeValueSize { // we know the string won't really be over, but still check - saPayloads[sadefs.TemporalWorkerDeployment] = deploymentPayload + saPayloads[searchattribute.TemporalWorkerDeployment] = deploymentPayload } versionPayload, err := searchattribute.EncodeValue(version, enumspb.INDEXED_VALUE_TYPE_KEYWORD) if err != nil { return err } if len(versionPayload.GetData()) <= maxSearchAttributeValueSize { // we know the string won't really be over, but still check - saPayloads[sadefs.TemporalWorkerDeploymentVersion] = versionPayload + saPayloads[searchattribute.TemporalWorkerDeploymentVersion] = versionPayload } behaviorPayload, err := searchattribute.EncodeValue(behavior, enumspb.INDEXED_VALUE_TYPE_KEYWORD) if err != nil { return err } if len(behaviorPayload.GetData()) <= maxSearchAttributeValueSize { // we know the string won't really be over, but still check - saPayloads[sadefs.TemporalWorkflowVersioningBehavior] = behaviorPayload + saPayloads[searchattribute.TemporalWorkflowVersioningBehavior] = behaviorPayload } ms.updateSearchAttributes(saPayloads) return nil @@ -3575,15 +3252,15 @@ func (ms *MutableStateImpl) addBuildIdAndDeploymentInfoToSearchAttributesWithNoV if err != nil { return false, err } - existingDeployment, err := ms.loadSearchAttributeString(sadefs.TemporalWorkerDeployment) + existingDeployment, err := ms.loadSearchAttributeString(searchattribute.TemporalWorkerDeployment) if err != nil { return false, err } - existingVersion, err := ms.loadSearchAttributeString(sadefs.TemporalWorkerDeploymentVersion) + existingVersion, err := ms.loadSearchAttributeString(searchattribute.TemporalWorkerDeploymentVersion) if err != nil { return false, err } - existingBehavior, err := ms.loadSearchAttributeString(sadefs.TemporalWorkflowVersioningBehavior) + existingBehavior, err := ms.loadSearchAttributeString(searchattribute.TemporalWorkflowVersioningBehavior) if err != nil { return false, err } @@ -4289,7 +3966,6 @@ func (ms *MutableStateImpl) AddCompletedWorkflowEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( event.GetEventTime().AsTime(), false, - false, // skipCloseTransferTask ); err != nil { return nil, err } @@ -4333,7 +4009,6 @@ func (ms *MutableStateImpl) AddFailWorkflowEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( event.GetEventTime().AsTime(), false, - false, // skipCloseTransferTask ); err != nil { return nil, err } @@ -4381,7 +4056,6 @@ func (ms *MutableStateImpl) AddTimeoutWorkflowEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( event.GetEventTime().AsTime(), false, - false, // skipCloseTransferTask ); err != nil { return nil, err } @@ -4464,7 +4138,6 @@ func (ms *MutableStateImpl) AddWorkflowExecutionCanceledEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( event.GetEventTime().AsTime(), false, - false, // skipCloseTransferTask ); err != nil { return nil, err } @@ -4997,7 +4670,6 @@ func (ms *MutableStateImpl) AddWorkflowExecutionTerminatedEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( event.GetEventTime().AsTime(), deleteAfterTerminate, - false, // skipCloseTransferTask ); err != nil { return nil, err } @@ -5183,7 +4855,7 @@ func (ms *MutableStateImpl) AddWorkflowExecutionOptionsUpdatedEvent( attachRequestID string, attachCompletionCallbacks []*commonpb.Callback, links []*commonpb.Link, - identity string, + priority *commonpb.Priority, ) (*historypb.HistoryEvent, error) { if err := ms.checkMutability(tag.WorkflowActionWorkflowOptionsUpdated); err != nil { return nil, err @@ -5194,7 +4866,7 @@ func (ms *MutableStateImpl) AddWorkflowExecutionOptionsUpdatedEvent( attachRequestID, attachCompletionCallbacks, links, - identity, + priority, ) prevEffectiveVersioningBehavior := ms.GetEffectiveVersioningBehavior() prevEffectiveDeployment := ms.GetEffectiveDeployment() @@ -5233,34 +4905,38 @@ func (ms *MutableStateImpl) ApplyWorkflowExecutionOptionsUpdatedEvent(event *his return err } + // Update priority. + if attributes.GetPriority() != nil { + ms.executionInfo.Priority = attributes.GetPriority() + requestReschedulePendingWorkflowTask = true + } + // Update attached request ID. if attributes.GetAttachedRequestId() != "" { ms.AttachRequestID(attributes.GetAttachedRequestId(), event.EventType, event.EventId) } // Update completion callbacks. - if err := ms.addCompletionCallbacks( - event, - attributes.GetAttachedRequestId(), - attributes.GetAttachedCompletionCallbacks(), - ); err != nil { - return err + if len(attributes.GetAttachedCompletionCallbacks()) > 0 { + if err := ms.addCompletionCallbacks( + event, + attributes.GetAttachedRequestId(), + attributes.GetAttachedCompletionCallbacks(), + ); err != nil { + return err + } } // Finally, reschedule the pending workflow task if so requested. if requestReschedulePendingWorkflowTask { - return ms.reschedulePendingWorkflowTask() + return ms.reschedulePendingWorkflowTask(true) } return nil } func (ms *MutableStateImpl) updateVersioningOverride( override *workflowpb.VersioningOverride, -) (bool, error) { - previousEffectiveDeployment := ms.GetEffectiveDeployment() - previousEffectiveVersioningBehavior := ms.GetEffectiveVersioningBehavior() - var requestReschedulePendingWorkflowTask bool - +) (requestReschedulePendingWorkflowTask bool, err error) { if override != nil { if ms.GetExecutionInfo().GetVersioningInfo() == nil { ms.GetExecutionInfo().VersioningInfo = &workflowpb.WorkflowExecutionVersioningInfo{} @@ -5322,6 +4998,8 @@ func (ms *MutableStateImpl) updateVersioningOverride( ms.GetExecutionInfo().WorkerDeploymentName = "" } + previousEffectiveDeployment := ms.GetEffectiveDeployment() + previousEffectiveVersioningBehavior := ms.GetEffectiveVersioningBehavior() if !proto.Equal(ms.GetEffectiveDeployment(), previousEffectiveDeployment) || ms.GetEffectiveVersioningBehavior() != previousEffectiveVersioningBehavior { // TODO (carly) part 2: if safe mode, do replay test on new deployment if deployment changed, if fail, revert changes and abort @@ -5364,12 +5042,13 @@ func (ms *MutableStateImpl) updateVersioningOverride( // For v3 versioned workflows (ms.GetEffectiveVersioningBehavior() != UNSPECIFIED), this will update the reachability // search attribute based on the execution_info.deployment and/or override deployment if one exists. limit := ms.config.SearchAttributesSizeOfValueLimit(ms.namespaceEntry.Name().String()) - if err := ms.updateBuildIdsAndDeploymentSearchAttributes(nil, limit); err != nil { - return requestReschedulePendingWorkflowTask, err + if err = ms.updateBuildIdsAndDeploymentSearchAttributes(nil, limit); err != nil { + return } } - return requestReschedulePendingWorkflowTask, ms.reschedulePendingActivities() + err = ms.reschedulePendingActivities() + return } func (ms *MutableStateImpl) ApplyWorkflowExecutionTerminatedEvent( @@ -5457,7 +5136,7 @@ func (ms *MutableStateImpl) AddContinueAsNewEvent( } var err error - newRunID := uuid.NewString() + newRunID := uuid.New() newExecution := commonpb.WorkflowExecution{ WorkflowId: ms.executionInfo.WorkflowId, RunId: newRunID, @@ -5517,7 +5196,6 @@ func (ms *MutableStateImpl) AddContinueAsNewEvent( } if _, err = newMutableState.addWorkflowExecutionStartedEventForContinueAsNew( - ctx, parentInfo, &newExecution, ms, @@ -5548,7 +5226,6 @@ func (ms *MutableStateImpl) AddContinueAsNewEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( continueAsNewEvent.GetEventTime().AsTime(), false, - false, // skipCloseTransferTask ); err != nil { return nil, nil, err } @@ -6042,9 +5719,6 @@ func (ms *MutableStateImpl) RetryActivity( activityInfo.RequestId = "" activityInfo.RetryLastFailure = ms.truncateRetryableActivityFailure(activityFailure) activityInfo.Attempt++ - if ms.config.EnableActivityRetryStampIncrement() { - activityInfo.Stamp++ - } return nil }); err != nil { return enumspb.RETRY_STATE_INTERNAL_SERVER_ERROR, err @@ -6118,14 +5792,12 @@ func (ms *MutableStateImpl) updateActivityInfoForRetries( _ = ms.UpdateActivity(ai.ScheduledEventId, func(activityInfo *persistencespb.ActivityInfo, mutableState historyi.MutableState) error { mutableStateImpl, ok := mutableState.(*MutableStateImpl) if ok { - isActivityRetryStampIncrementEnabled := ms.config.EnableActivityRetryStampIncrement() ai = UpdateActivityInfoForRetries( activityInfo, mutableStateImpl.GetCurrentVersion(), nextAttempt, mutableStateImpl.truncateRetryableActivityFailure(activityFailure), timestamppb.New(nextScheduledTime), - isActivityRetryStampIncrementEnabled, ) } return nil @@ -6208,123 +5880,14 @@ func (ms *MutableStateImpl) updatePauseInfoSearchAttribute() error { exeInfo.SearchAttributes = make(map[string]*commonpb.Payload, 1) } - if proto.Equal(exeInfo.SearchAttributes[sadefs.TemporalPauseInfo], pauseInfoPayload) { + if proto.Equal(exeInfo.SearchAttributes[searchattribute.TemporalPauseInfo], pauseInfoPayload) { return nil // unchanged } - ms.updateSearchAttributes(map[string]*commonpb.Payload{sadefs.TemporalPauseInfo: pauseInfoPayload}) - return ms.taskGenerator.GenerateUpsertVisibilityTask() -} - -func (ms *MutableStateImpl) UpdateReportedProblemsSearchAttribute() error { - var reportedProblems []string - switch wftFailure := ms.executionInfo.LastWorkflowTaskFailure.(type) { - case *persistencespb.WorkflowExecutionInfo_LastWorkflowTaskFailureCause: - reportedProblems = []string{ - "category=WorkflowTaskFailed", - fmt.Sprintf("cause=WorkflowTaskFailedCause%s", wftFailure.LastWorkflowTaskFailureCause.String()), - } - case *persistencespb.WorkflowExecutionInfo_LastWorkflowTaskTimedOutType: - reportedProblems = []string{ - "category=WorkflowTaskTimedOut", - fmt.Sprintf("cause=WorkflowTaskTimedOutCause%s", wftFailure.LastWorkflowTaskTimedOutType.String()), - } - } - - reportedProblemsPayload, err := searchattribute.EncodeValue(reportedProblems, enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST) - if err != nil { - return err - } - - exeInfo := ms.executionInfo - if exeInfo.SearchAttributes == nil { - exeInfo.SearchAttributes = make(map[string]*commonpb.Payload, 1) - } - - decodedA, err := searchattribute.DecodeValue(exeInfo.SearchAttributes[sadefs.TemporalReportedProblems], enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, false) - if err != nil { - return err - } - - existingProblems, ok := decodedA.([]string) - if !ok && decodedA != nil { - softassert.Fail(ms.logger, "TemporalReportedProblems payload decoded to unexpected type for logging") - return errors.New("TemporalReportedProblems payload decoded to unexpected type for logging") - } - - if slices.Equal(existingProblems, reportedProblems) { - return nil - } - - // Log the search attribute change - ms.logReportedProblemsChange(existingProblems, reportedProblems) - - ms.updateSearchAttributes(map[string]*commonpb.Payload{sadefs.TemporalReportedProblems: reportedProblemsPayload}) - return ms.taskGenerator.GenerateUpsertVisibilityTask() -} - -func (ms *MutableStateImpl) RemoveReportedProblemsSearchAttribute() error { - if ms.executionInfo.SearchAttributes == nil { - return nil - } - - temporalReportedProblems := ms.executionInfo.SearchAttributes[sadefs.TemporalReportedProblems] - if temporalReportedProblems == nil { - return nil - } - - // Log the removal of the search attribute - ms.logReportedProblemsChange(ms.decodeReportedProblems(temporalReportedProblems), nil) - - ms.executionInfo.LastWorkflowTaskFailure = nil - - // Just remove the search attribute entirely for now - ms.updateSearchAttributes(map[string]*commonpb.Payload{sadefs.TemporalReportedProblems: nil}) + ms.updateSearchAttributes(map[string]*commonpb.Payload{searchattribute.TemporalPauseInfo: pauseInfoPayload}) return ms.taskGenerator.GenerateUpsertVisibilityTask() } -// logReportedProblemsChange logs changes to the TemporalReportedProblems search attribute -func (ms *MutableStateImpl) logReportedProblemsChange(oldPayload, newPayload []string) { - if oldPayload == nil && newPayload != nil { - // Adding search attribute - ms.logger.Info("TemporalReportedProblems search attribute added", - tag.NewStringsTag("reported-problems", newPayload)) - } else if oldPayload != nil && newPayload == nil { - // Removing search attribute - ms.logger.Info("TemporalReportedProblems search attribute removed", - tag.NewStringsTag("previous-reported-problems", oldPayload)) - } else if oldPayload != nil && newPayload != nil { - // Updating search attribute - ms.logger.Info("TemporalReportedProblems search attribute updated", - tag.NewStringsTag("previous-reported-problems", oldPayload), - tag.NewStringsTag("reported-problems", newPayload)) - } -} - -// decodeReportedProblems safely decodes a keyword list payload to []string -func (ms *MutableStateImpl) decodeReportedProblems(p *commonpb.Payload) []string { - if p == nil { - return nil - } - - decoded, err := searchattribute.DecodeValue(p, enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, false) - if err != nil { - ms.logger.Error("Failed to decode TemporalReportedProblems payload for logging", - tag.Error(err)) - softassert.Fail(ms.logger, "Failed to decode TemporalReportedProblems payload for logging") - return []string{} - } - - problems, ok := decoded.([]string) - if !ok { - ms.logger.Error("TemporalReportedProblems payload decoded to unexpected type for logging") - softassert.Fail(ms.logger, "TemporalReportedProblems payload decoded to unexpected type for logging") - return []string{} - } - - return problems -} - func (ms *MutableStateImpl) truncateRetryableActivityFailure( activityFailure *failurepb.Failure, ) *failurepb.Failure { @@ -6364,22 +5927,6 @@ func (ms *MutableStateImpl) processCloseCallbacks() error { return nil } - // Process CHASM callbacks if CHASM is enabled. Note that we check ChasmEnabled() rather than - // chasmCallbacksEnabled() to ensure that callbacks created when both HSM and CHASM callbacks - // were enabled can still be triggered even if the EnableCHASMCallbacks dynamic config is later - // turned off. Once created in CHASM, callbacks should always be processed as long as CHASM is enabled. - if ms.ChasmEnabled() { - if err := ms.processCloseCallbacksChasm(); err != nil { - return err - } - } - - // Always process HSM callbacks as well (a workflow can have both) - return ms.processCloseCallbacksHsm() -} - -// processCloseCallbacksHsm triggers "WorkflowClosed" callbacks using the HSM implementation. -func (ms *MutableStateImpl) processCloseCallbacksHsm() error { coll := callbacks.MachineCollection(ms.HSM()) for _, node := range coll.List() { cb, err := coll.Data(node.Key.ID) @@ -6400,15 +5947,8 @@ func (ms *MutableStateImpl) processCloseCallbacksHsm() error { return nil } -// processCloseCallbacksChasm triggers "WorkflowClosed" callbacks using the CHASM implementation. -func (ms *MutableStateImpl) processCloseCallbacksChasm() error { - wf, ctx, err := ms.ChasmWorkflowComponent(context.Background()) - if err != nil { - return err - } - - return wf.ProcessCloseCallbacks(ctx) -} +// TODO mutable state should generate corresponding transfer / timer tasks according to +// updates accumulated, while currently all transfer / timer tasks are managed manually func (ms *MutableStateImpl) AddTasks( newTasks ...tasks.Task, @@ -6421,47 +5961,14 @@ func (ms *MutableStateImpl) AddTasks( ms.logger.Info("Dropped long duration scheduled task.", tasks.Tags(task)...) continue } - - if chasmPureTask, ok := task.(*tasks.ChasmTaskPure); ok { - ms.chasmPureTasks = append(ms.chasmPureTasks, chasmPureTask) - maxPureTasks := ms.config.ChasmMaxInMemoryPureTasks() - if len(ms.chasmPureTasks) > maxPureTasks { - // Since tasks are reverse ordered by their scheduled time, tasks in the beginning are those - // - Generated a long time ago - // - Scheduled time is far in the future - // both types of tasks are likely to already be persisted in DB and best-effort deletion won't help, - // so drop them from the in-memory list first. - ms.chasmPureTasks = ms.chasmPureTasks[len(ms.chasmPureTasks)-maxPureTasks:] - } - } - ms.InsertTasks[category] = append(ms.InsertTasks[category], task) } } func (ms *MutableStateImpl) PopTasks() map[tasks.Category][]tasks.Task { - insertTasks := ms.InsertTasks + insterTasks := ms.InsertTasks ms.InsertTasks = make(map[tasks.Category][]tasks.Task) - return insertTasks -} - -func (ms *MutableStateImpl) DeleteCHASMPureTasks(maxScheduledTime time.Time) { - for lastTaskIdx := len(ms.chasmPureTasks) - 1; lastTaskIdx >= 0; lastTaskIdx-- { - task := ms.chasmPureTasks[lastTaskIdx] - if !task.GetVisibilityTime().Before(maxScheduledTime) { - ms.chasmPureTasks = ms.chasmPureTasks[:lastTaskIdx+1] - return - } - - ms.BestEffortDeleteTasks[tasks.CategoryTimer] = append( - ms.BestEffortDeleteTasks[tasks.CategoryTimer], - task.GetKey(), - ) - } - - // If we reach here, all tasks have visibility time before maxScheduledTime - // and need to be deleted. - ms.chasmPureTasks = ms.chasmPureTasks[:0] + return insterTasks } func (ms *MutableStateImpl) SetUpdateCondition( @@ -6502,22 +6009,6 @@ func (ms *MutableStateImpl) RemoveSpeculativeWorkflowTaskTimeoutTask() { } } -func (ms *MutableStateImpl) SetWorkflowTaskScheduleToStartTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) { - ms.wftScheduleToStartTimeoutTask = task -} - -func (ms *MutableStateImpl) SetWorkflowTaskStartToCloseTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) { - ms.wftStartToCloseTimeoutTask = task -} - -func (ms *MutableStateImpl) GetWorkflowTaskScheduleToStartTimeoutTask() *tasks.WorkflowTaskTimeoutTask { - return ms.wftScheduleToStartTimeoutTask -} - -func (ms *MutableStateImpl) GetWorkflowTaskStartToCloseTimeoutTask() *tasks.WorkflowTaskTimeoutTask { - return ms.wftStartToCloseTimeoutTask -} - func (ms *MutableStateImpl) GetWorkflowStateStatus() (enumsspb.WorkflowExecutionState, enumspb.WorkflowExecutionStatus) { return ms.executionState.State, ms.executionState.Status } @@ -6643,8 +6134,7 @@ func (ms *MutableStateImpl) CloseTransactionAsMutation( NewBufferedEvents: result.bufferEvents, ClearBufferedEvents: result.clearBuffer, - Tasks: ms.InsertTasks, - BestEffortDeleteTasks: ms.BestEffortDeleteTasks, + Tasks: ms.InsertTasks, Condition: ms.nextEventIDInDB, DBRecordVersion: ms.dbRecordVersion, @@ -6668,11 +6158,7 @@ func (ms *MutableStateImpl) CloseTransactionAsSnapshot( if len(result.bufferEvents) > 0 { // TODO do we need the functionality to generate snapshot with buffered events? - return nil, nil, softassert.UnexpectedInternalErr( - ms.logger, - "cannot generate workflow snapshot with buffered events", - nil, - ) + return nil, nil, serviceerror.NewInternal("cannot generate workflow snapshot with buffered events") } workflowSnapshot := &persistence.WorkflowSnapshot{ @@ -7281,12 +6767,6 @@ func (ms *MutableStateImpl) closeTransactionPrepareReplicationTasks( replicationTasks = append(replicationTasks, ms.syncActivityToReplicationTask(transactionPolicy)...) replicationTasks = append(replicationTasks, ms.dirtyHSMToReplicationTask(transactionPolicy, eventBatches, clearBufferEvents)...) - archetypeID := ms.ChasmTree().ArchetypeID() - isWorkflow := archetypeID == chasm.WorkflowArchetypeID - if !isWorkflow && len(replicationTasks) != 0 { - return softassert.UnexpectedInternalErr(ms.logger, "chasm execution generated workflow replication tasks", nil) - } - if ms.transitionHistoryEnabled { switch transactionPolicy { case historyi.TransactionPolicyActive: @@ -7329,7 +6809,6 @@ func (ms *MutableStateImpl) closeTransactionPrepareReplicationTasks( syncVersionedTransitionTask := &tasks.SyncVersionedTransitionTask{ WorkflowKey: workflowKey, VisibilityTimestamp: now, - ArchetypeID: archetypeID, Priority: enumsspb.TASK_PRIORITY_HIGH, VersionedTransition: currentVersionedTransition, FirstEventID: firstEventID, @@ -7354,22 +6833,16 @@ func (ms *MutableStateImpl) closeTransactionPrepareReplicationTasks( default: panic(fmt.Sprintf("unknown transaction policy: %v", transactionPolicy)) } - } else if isWorkflow { + } else { ms.InsertTasks[tasks.CategoryReplication] = append( ms.InsertTasks[tasks.CategoryReplication], replicationTasks..., ) - } else { - return softassert.UnexpectedInternalErr(ms.logger, "state-based replication not enabled for chasm execution", nil) } if transactionPolicy == historyi.TransactionPolicyPassive && len(ms.InsertTasks[tasks.CategoryReplication]) > 0 { - return softassert.UnexpectedInternalErr( - ms.logger, - "should not generate replication task when close transaction as passive", - nil, - ) + return serviceerror.NewInternal("should not generate replication task when close transaction as passive") } return nil @@ -7424,7 +6897,6 @@ func (ms *MutableStateImpl) cleanupTransaction() error { ) ms.InsertTasks = make(map[tasks.Category][]tasks.Task) - ms.BestEffortDeleteTasks = make(map[tasks.Category][]tasks.Key) // Clear outputs for the next transaction. ms.stateMachineNode.ClearTransactionState() @@ -8413,7 +7885,6 @@ func (ms *MutableStateImpl) syncExecutionInfo(current *persistencespb.WorkflowEx RequestID: incoming.WorkflowTaskRequestId, WorkflowTaskTimeout: incoming.WorkflowTaskTimeout.AsDuration(), Attempt: incoming.WorkflowTaskAttempt, - Stamp: incoming.WorkflowTaskStamp, StartedTime: incoming.WorkflowTaskStartedTime.AsTime(), ScheduledTime: incoming.WorkflowTaskScheduledTime.AsTime(), @@ -8697,7 +8168,7 @@ func (ms *MutableStateImpl) GetEffectiveVersioningBehavior() enumspb.VersioningB // If there is a pending workflow task that is not started yet, it'll be rescheduled after // transition start. // This method must be called with a version different from the effective version. -func (ms *MutableStateImpl) StartDeploymentTransition(deployment *deploymentpb.Deployment, revisionNumber int64) error { +func (ms *MutableStateImpl) StartDeploymentTransition(deployment *deploymentpb.Deployment) error { wfBehavior := ms.GetEffectiveVersioningBehavior() if wfBehavior == enumspb.VERSIONING_BEHAVIOR_PINNED { // WF is pinned so we reject the transition. @@ -8731,7 +8202,7 @@ func (ms *MutableStateImpl) StartDeploymentTransition(deployment *deploymentpb.D // - reschedule the pending WFT so the old one is invalided ms.ClearStickyTaskQueue() - err := ms.reschedulePendingWorkflowTask() + err := ms.reschedulePendingWorkflowTask(false) if err != nil { return err } @@ -8745,22 +8216,9 @@ func (ms *MutableStateImpl) StartDeploymentTransition(deployment *deploymentpb.D ), ).Record(1) - ms.SetVersioningRevisionNumber(revisionNumber) - return nil } -func (ms *MutableStateImpl) GetVersioningRevisionNumber() int64 { - return ms.GetExecutionInfo().GetVersioningInfo().GetRevisionNumber() -} - -func (ms *MutableStateImpl) SetVersioningRevisionNumber(revisionNumber int64) { - if ms.GetExecutionInfo().GetVersioningInfo() == nil { - ms.GetExecutionInfo().VersioningInfo = &workflowpb.WorkflowExecutionVersioningInfo{} - } - ms.GetExecutionInfo().GetVersioningInfo().RevisionNumber = revisionNumber -} - // reschedulePendingActivities reschedules all the activities that are not started, so they are // scheduled against the right queue in matching. func (ms *MutableStateImpl) reschedulePendingActivities() error { @@ -8770,17 +8228,8 @@ func (ms *MutableStateImpl) reschedulePendingActivities() error { // activity already started continue } - - // need to update stamp so the passive side regenerate the task - err := ms.UpdateActivity(ai.ScheduledEventId, func(info *persistencespb.ActivityInfo, state historyi.MutableState) error { - info.Stamp++ - return nil - }) - if err != nil { - return err - } // we only need to resend the activities to matching, no need to update timer tasks. - err = ms.taskGenerator.GenerateActivityTasks(ai.ScheduledEventId) + err := ms.taskGenerator.GenerateActivityTasks(ai.ScheduledEventId) if err != nil { return err } @@ -8791,7 +8240,7 @@ func (ms *MutableStateImpl) reschedulePendingActivities() error { // reschedulePendingWorkflowTask reschedules the pending WFT if it is not started yet. // The currently scheduled WFT will be rejected when attempting to start because its stamp changed. -func (ms *MutableStateImpl) reschedulePendingWorkflowTask() error { +func (ms *MutableStateImpl) reschedulePendingWorkflowTask(invalidatePendingTasks bool) error { // If the WFT is started but not finished, we let it run its course // - once it's completed, failed or timed out a new one will be scheduled. if !ms.HasPendingWorkflowTask() || ms.HasStartedWorkflowTask() { @@ -8806,13 +8255,15 @@ func (ms *MutableStateImpl) reschedulePendingWorkflowTask() error { ms.logInfo("start transition did not reschedule pending speculative task") return nil } - // Reset the attempt; forcing a non-transient workflow task to be scheduled. - ms.executionInfo.WorkflowTaskAttempt = 1 - // Increase the stamp ("version") to invalidate the pending non-speculative WFT. + // If requested, increase the stamp ("version") to invalidate the pending non-speculative WFT. // We don't invalidate speculative WFTs because they are very latency sensitive. - ms.executionInfo.WorkflowTaskStamp += 1 - ms.workflowTaskUpdated = true + if invalidatePendingTasks { + ms.executionInfo.WorkflowTaskStamp += 1 + } + + // Reset the attempt; forcing a non-transient workflow task to be scheduled. + ms.executionInfo.Attempt = 1 return ms.taskGenerator.GenerateScheduleWorkflowTaskTasks(pendingTask.ScheduledEventID) } diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index fb9900ec2ac..108198badb7 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/google/uuid" + "github.com/pborman/uuid" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/uber-go/tally/v4" @@ -32,6 +32,7 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" taskqueuespb "go.temporal.io/server/api/taskqueue/v1" "go.temporal.io/server/chasm" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/definition" @@ -44,7 +45,6 @@ import ( "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/searchattribute" - "go.temporal.io/server/common/searchattribute/sadefs" serviceerror2 "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/common/testing/testvars" @@ -223,7 +223,7 @@ func (s *mutableStateSuite) SetupSubTest() { func (s *mutableStateSuite) TestTransientWorkflowTaskCompletionFirstBatchApplied_ApplyWorkflowTaskCompleted() { version := int64(12) workflowID := "some random workflow ID" - runID := uuid.NewString() + runID := uuid.New() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -257,7 +257,7 @@ func (s *mutableStateSuite) TestTransientWorkflowTaskCompletionFirstBatchApplied func (s *mutableStateSuite) TestTransientWorkflowTaskCompletionFirstBatchApplied_FailoverWorkflowTaskTimeout() { version := int64(12) workflowID := "some random workflow ID" - runID := uuid.NewString() + runID := uuid.New() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -282,7 +282,7 @@ func (s *mutableStateSuite) TestTransientWorkflowTaskCompletionFirstBatchApplied func (s *mutableStateSuite) TestTransientWorkflowTaskCompletionFirstBatchApplied_FailoverWorkflowTaskFailed() { version := int64(12) workflowID := "some random workflow ID" - runID := uuid.NewString() + runID := uuid.New() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -475,161 +475,11 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_UnexpectedSticky() { s.Equal(int64(0), s.mutableState.GetExecutionInfo().GetBuildIdRedirectCounter()) } -func (s *mutableStateSuite) TestPopulateDeleteTasks_WithWorkflowTaskTimeouts() { - // Test that workflow task timeout task references are added to BestEffortDeleteTasks when present. - version := int64(1) - workflowID := "wf-timeout-delete" - runID := uuid.NewString() - s.mutableState = TestGlobalMutableState( - s.mockShard, - s.mockEventsCache, - s.logger, - version, - workflowID, - runID, - ) - - // Create mock timeout tasks that meet the criteria - now := time.Now().UTC() - mockScheduleToStartTask := &tasks.WorkflowTaskTimeoutTask{ - WorkflowKey: definition.NewWorkflowKey( - s.mutableState.GetExecutionInfo().NamespaceId, - workflowID, - runID, - ), - VisibilityTimestamp: now.Add(10 * time.Second), // < 120s - TaskID: 123, - TimeoutType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START, - InMemory: false, // Persisted task - } - - mockStartToCloseTask := &tasks.WorkflowTaskTimeoutTask{ - WorkflowKey: definition.NewWorkflowKey( - s.mutableState.GetExecutionInfo().NamespaceId, - workflowID, - runID, - ), - VisibilityTimestamp: now.Add(30 * time.Second), // < 120s - TaskID: 456, - TimeoutType: enumspb.TIMEOUT_TYPE_START_TO_CLOSE, - InMemory: false, // Persisted task - } - - // Schedule and start a workflow task - wft, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) - s.NoError(err) - - sticky := &taskqueuepb.TaskQueue{Name: "sticky-tq", Kind: enumspb.TASK_QUEUE_KIND_STICKY} - _, wft, err = s.mutableState.AddWorkflowTaskStartedEvent( - wft.ScheduledEventID, - "", - sticky, - "", - nil, - nil, - nil, - false, - ) - s.NoError(err) - - // Set timeout tasks directly in mutable state (simulating what task_generator does) - s.mutableState.SetWorkflowTaskScheduleToStartTimeoutTask(mockScheduleToStartTask) - s.mutableState.SetWorkflowTaskStartToCloseTimeoutTask(mockStartToCloseTask) - // Call UpdateWorkflowTask to persist the workflow task info to ExecutionInfo - s.mutableState.workflowTaskManager.UpdateWorkflowTask(wft) - - // Complete the workflow task - _, err = s.mutableState.AddWorkflowTaskCompletedEvent( - wft, - &workflowservice.RespondWorkflowTaskCompletedRequest{}, - workflowTaskCompletionLimits, - ) - s.NoError(err) - - // Verify that BestEffortDeleteTasks contains the timeout task keys - del := s.mutableState.BestEffortDeleteTasks - s.Contains(del, tasks.CategoryTimer) - s.Equal(2, len(del[tasks.CategoryTimer]), "Should have both ScheduleToStart and StartToClose timeout tasks") - s.Contains(del[tasks.CategoryTimer], mockScheduleToStartTask.GetKey()) - s.Contains(del[tasks.CategoryTimer], mockStartToCloseTask.GetKey()) -} - -func (s *mutableStateSuite) TestPopulateDeleteTasks_LongTimeout_NotIncluded() { - // Test that timeout tasks with very long timeouts (> 120s) are NOT added to BestEffortDeleteTasks. - version := int64(1) - workflowID := "wf-long-timeout" - runID := uuid.NewString() - s.mutableState = TestGlobalMutableState( - s.mockShard, - s.mockEventsCache, - s.logger, - version, - workflowID, - runID, - ) - - // Schedule a workflow task - this sets the scheduled time - wft, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) - s.NoError(err) - - sticky := &taskqueuepb.TaskQueue{Name: "sticky-tq", Kind: enumspb.TASK_QUEUE_KIND_STICKY} - _, wft, err = s.mutableState.AddWorkflowTaskStartedEvent( - wft.ScheduledEventID, - "", - sticky, - "", - nil, - nil, - nil, - false, - ) - s.NoError(err) - - // Get the actual scheduled time from wft (this is what will be used in the calculation) - scheduledTime := wft.ScheduledTime - if scheduledTime.IsZero() { - // If scheduled time is not set in wft, use current time - scheduledTime = time.Now().UTC() - } - - // Create a timeout task with timeout > 120s relative to the actual scheduled time - mockLongTimeoutTask := &tasks.WorkflowTaskTimeoutTask{ - WorkflowKey: definition.NewWorkflowKey( - s.mutableState.GetExecutionInfo().NamespaceId, - workflowID, - runID, - ), - VisibilityTimestamp: scheduledTime.Add(200 * time.Second), // > 120s from scheduled time - TaskID: 123, - TimeoutType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START, - InMemory: false, // Persisted task - } - - // Set the long timeout task directly in mutable state - s.mutableState.SetWorkflowTaskScheduleToStartTimeoutTask(mockLongTimeoutTask) - // Clear the StartToClose task so it doesn't interfere with the test - s.mutableState.SetWorkflowTaskStartToCloseTimeoutTask(nil) - - // Complete the workflow task - _, err = s.mutableState.AddWorkflowTaskCompletedEvent( - wft, - &workflowservice.RespondWorkflowTaskCompletedRequest{}, - workflowTaskCompletionLimits, - ) - s.NoError(err) - - // Verify that BestEffortDeleteTasks does NOT contain the long timeout task - del := s.mutableState.BestEffortDeleteTasks - if timerTasks, exists := del[tasks.CategoryTimer]; exists { - s.Equal(0, len(timerTasks), "Tasks with timeout > 120s should not be added to BestEffortDeleteTasks") - } -} - // creates a mutable state with first WFT completed on Build ID "b1" func (s *mutableStateSuite) createVersionedMutableStateWithCompletedWFT(tq *taskqueuepb.TaskQueue) { version := int64(12) workflowID := "some random workflow ID" - runID := uuid.NewString() + runID := uuid.New() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -902,7 +752,7 @@ func (s *mutableStateSuite) createMutableStateWithVersioningBehavior( ) { version := int64(12) workflowID := "some random workflow ID" - runID := uuid.NewString() + runID := uuid.New() s.mutableState = TestGlobalMutableState( s.mockShard, @@ -919,9 +769,11 @@ func (s *mutableStateSuite) createMutableStateWithVersioningBehavior( s.NoError(err) s.verifyEffectiveDeployment(nil, enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED) - err = s.mutableState.StartDeploymentTransition(deployment, 0) + err = s.mutableState.StartDeploymentTransition(deployment) s.NoError(err) s.verifyEffectiveDeployment(deployment, enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE) + s.EqualValues(1, s.mutableState.executionInfo.Attempt, + "workflow task attempt must be reset to 1 since pending tasks are rescheduled") _, wft, err = s.mutableState.AddWorkflowTaskStartedEvent( wft.ScheduledEventID, @@ -969,7 +821,7 @@ func (s *mutableStateSuite) TestUnpinnedTransition() { s.NoError(err) s.verifyEffectiveDeployment(deployment1, behavior) - err = s.mutableState.StartDeploymentTransition(deployment2, 0) + err = s.mutableState.StartDeploymentTransition(deployment2) s.NoError(err) s.verifyEffectiveDeployment(deployment2, behavior) @@ -1008,7 +860,7 @@ func (s *mutableStateSuite) TestUnpinnedTransitionFailed() { s.NoError(err) s.verifyEffectiveDeployment(deployment1, behavior) - err = s.mutableState.StartDeploymentTransition(deployment2, 0) + err = s.mutableState.StartDeploymentTransition(deployment2) s.NoError(err) s.verifyEffectiveDeployment(deployment2, behavior) @@ -1050,7 +902,7 @@ func (s *mutableStateSuite) TestUnpinnedTransitionTimeout() { s.NoError(err) s.verifyEffectiveDeployment(deployment1, behavior) - err = s.mutableState.StartDeploymentTransition(deployment2, 0) + err = s.mutableState.StartDeploymentTransition(deployment2) s.NoError(err) s.verifyEffectiveDeployment(deployment2, behavior) @@ -1090,7 +942,6 @@ func (s *mutableStateSuite) verifyWorkflowOptionsUpdatedEventAttr( s.Equal(expectedOverride.GetPinnedVersion(), actualOverride.GetPinnedVersion()) //nolint:staticcheck // SA1019: worker versioning v0.31 s.Equal(actualAttr.GetUnsetVersioningOverride(), expectedAttr.GetUnsetVersioningOverride()) - s.Equal(actualAttr.GetIdentity(), expectedAttr.GetIdentity()) } func (s *mutableStateSuite) verifyOverrides( @@ -1121,11 +972,10 @@ func (s *mutableStateSuite) TestOverride_UnpinnedBase_SetPinnedAndUnsetWithEmpty tq := &taskqueuepb.TaskQueue{Name: "tq"} baseBehavior := enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE overrideBehavior := enumspb.VERSIONING_BEHAVIOR_PINNED - id := uuid.NewString() s.createMutableStateWithVersioningBehavior(baseBehavior, deployment1, tq) // set pinned override - event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions2.GetVersioningOverride(), false, "", nil, nil, id) + event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions2.GetVersioningOverride(), false, "", nil, nil, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment2, overrideBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1133,14 +983,12 @@ func (s *mutableStateSuite) TestOverride_UnpinnedBase_SetPinnedAndUnsetWithEmpty &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: pinnedOptions2.GetVersioningOverride(), UnsetVersioningOverride: false, - Identity: id, }, ) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, deployment2) // unset pinned override with boolean - id = uuid.NewString() - event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, id) + event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment1, baseBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1148,7 +996,6 @@ func (s *mutableStateSuite) TestOverride_UnpinnedBase_SetPinnedAndUnsetWithEmpty &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: nil, UnsetVersioningOverride: true, - Identity: id, }, ) s.verifyOverrides(baseBehavior, enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED, deployment1, nil) @@ -1158,11 +1005,10 @@ func (s *mutableStateSuite) TestOverride_PinnedBase_SetUnpinnedAndUnsetWithEmpty tq := &taskqueuepb.TaskQueue{Name: "tq"} baseBehavior := enumspb.VERSIONING_BEHAVIOR_PINNED overrideBehavior := enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE - id := uuid.NewString() s.createMutableStateWithVersioningBehavior(baseBehavior, deployment1, tq) // set unpinned override - event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(unpinnedOptions.GetVersioningOverride(), false, "", nil, nil, id) + event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(unpinnedOptions.GetVersioningOverride(), false, "", nil, nil, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment1, overrideBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1170,14 +1016,12 @@ func (s *mutableStateSuite) TestOverride_PinnedBase_SetUnpinnedAndUnsetWithEmpty &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: unpinnedOptions.GetVersioningOverride(), UnsetVersioningOverride: false, - Identity: id, }, ) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, nil) // unset pinned override with empty - id = uuid.NewString() - event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, id) + event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment1, baseBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1185,7 +1029,6 @@ func (s *mutableStateSuite) TestOverride_PinnedBase_SetUnpinnedAndUnsetWithEmpty &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: nil, UnsetVersioningOverride: true, - Identity: id, }, ) s.verifyOverrides(baseBehavior, enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED, deployment1, nil) @@ -1195,10 +1038,9 @@ func (s *mutableStateSuite) TestOverride_RedirectFails() { tq := &taskqueuepb.TaskQueue{Name: "tq"} baseBehavior := enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE overrideBehavior := enumspb.VERSIONING_BEHAVIOR_PINNED - id := uuid.NewString() s.createMutableStateWithVersioningBehavior(baseBehavior, deployment1, tq) - event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions3.GetVersioningOverride(), false, "", nil, nil, id) + event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions3.GetVersioningOverride(), false, "", nil, nil, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment3, overrideBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1206,13 +1048,12 @@ func (s *mutableStateSuite) TestOverride_RedirectFails() { &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: pinnedOptions3.GetVersioningOverride(), UnsetVersioningOverride: false, - Identity: id, }, ) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, deployment3) // assert that transition fails - err = s.mutableState.StartDeploymentTransition(deployment2, 0) + err = s.mutableState.StartDeploymentTransition(deployment2) s.ErrorIs(err, ErrPinnedWorkflowCannotTransition) s.verifyEffectiveDeployment(deployment3, overrideBehavior) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, deployment3) @@ -1222,10 +1063,9 @@ func (s *mutableStateSuite) TestOverride_BaseDeploymentUpdatedOnCompletion() { tq := &taskqueuepb.TaskQueue{Name: "tq"} baseBehavior := enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE overrideBehavior := enumspb.VERSIONING_BEHAVIOR_PINNED - id := uuid.NewString() s.createMutableStateWithVersioningBehavior(baseBehavior, deployment1, tq) - event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions3.GetVersioningOverride(), false, "", nil, nil, id) + event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions3.GetVersioningOverride(), false, "", nil, nil, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment3, overrideBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1233,13 +1073,12 @@ func (s *mutableStateSuite) TestOverride_BaseDeploymentUpdatedOnCompletion() { &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: pinnedOptions3.GetVersioningOverride(), UnsetVersioningOverride: false, - Identity: id, }, ) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, deployment3) // assert that redirect fails - should be its own test - err = s.mutableState.StartDeploymentTransition(deployment2, 0) + err = s.mutableState.StartDeploymentTransition(deployment2) s.ErrorIs(err, ErrPinnedWorkflowCannotTransition) s.verifyEffectiveDeployment(deployment3, overrideBehavior) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, deployment3) // base deployment still deployment1 here -- good @@ -1276,8 +1115,7 @@ func (s *mutableStateSuite) TestOverride_BaseDeploymentUpdatedOnCompletion() { s.verifyOverrides(baseBehavior, overrideBehavior, deployment2, deployment3) // now we unset the override and check that the base deployment/behavior is in effect - id = uuid.NewString() - event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, id) + event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment2, baseBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1285,7 +1123,6 @@ func (s *mutableStateSuite) TestOverride_BaseDeploymentUpdatedOnCompletion() { &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: nil, UnsetVersioningOverride: true, - Identity: id, }, ) s.verifyOverrides(baseBehavior, enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED, deployment2, nil) @@ -1420,435 +1257,6 @@ func (s *mutableStateSuite) TestChecksumShouldInvalidate() { s.False(s.mutableState.shouldInvalidateCheckum()) } -func (s *mutableStateSuite) TestUpdateWorkflowStateStatus_Table() { - s.SetupSubTest() - cases := []struct { - name string - currentState enumsspb.WorkflowExecutionState - currentStatus enumspb.WorkflowExecutionStatus - toState enumsspb.WorkflowExecutionState - toStatus enumspb.WorkflowExecutionStatus - wantErr bool - }{ - { - name: "created-> {running, running}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, - wantErr: false, - }, - { - name: "created-> {running, paused}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, - wantErr: false, - }, - { - name: "created-> {running, completed}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - wantErr: true, - }, - // CREATED -> CREATED (allowed for RUNNING/PAUSED) - { - name: "created-> {created, running}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, - wantErr: false, - }, - { - name: "created-> {created, paused}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, - wantErr: true, - }, - { - name: "created-> {created, completed} (invalid)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - wantErr: true, - }, - // CREATED -> COMPLETED (allowed only for TERMINATED/TIMED_OUT/CONTINUED_AS_NEW) - { - name: "created-> {completed, terminated}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, - wantErr: false, - }, - { - name: "created-> {completed, timed_out}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TIMED_OUT, - wantErr: false, - }, - { - name: "created-> {completed, continued_as_new}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_CONTINUED_AS_NEW, - wantErr: false, - }, - // CREATED -> ZOMBIE (allowed for RUNNING/PAUSED) - { - name: "created-> {zombie, running}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, - wantErr: false, - }, - { - name: "created-> {zombie, paused}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, - wantErr: false, - }, - // RUNNING state transitions - { - name: "running-> {created, running} (invalid)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, - wantErr: true, - }, - { - name: "running-> {running, paused}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, - wantErr: false, - }, - { - name: "running-> {running, terminated} (invalid)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, - wantErr: true, - }, - { - name: "running-> {completed, completed}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - wantErr: false, - }, - { - name: "running-> {completed, paused} (invalid)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, - wantErr: true, - }, - { - name: "running-> {zombie, running}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, - wantErr: false, - }, - { - name: "running-> {zombie, paused}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, - wantErr: false, - }, - { - name: "running-> {zombie, terminated} (invalid)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, - wantErr: true, - }, - // COMPLETED state transitions - { - name: "completed-> {completed, sameStatus} (no-op)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - currentStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - wantErr: false, - }, - { - name: "completed-> {created, running} (invalid)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - currentStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, - wantErr: true, - }, - { - name: "completed-> {running, running} (invalid)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - currentStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, - wantErr: true, - }, - { - name: "completed-> {zombie, running} (invalid)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - currentStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, - wantErr: true, - }, - { - name: "completed-> {completed, differentStatus} (invalid)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - currentStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_FAILED, - wantErr: true, - }, - // ZOMBIE state transitions - { - name: "zombie-> {created, running}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, - wantErr: false, - }, - { - name: "zombie-> {created, paused}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, - wantErr: true, - }, - { - name: "zombie-> {running, paused}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, - wantErr: false, - }, - { - name: "zombie-> {completed, terminated}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, - wantErr: false, - }, - { - name: "zombie-> {completed, paused} (invalid)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, - wantErr: true, - }, - { - name: "zombie-> {zombie, running}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, - wantErr: false, - }, - { - name: "zombie-> {zombie, terminated} (invalid)", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, - wantErr: true, - }, - // VOID state (no validation) - { - name: "void-> {running, running}", - currentState: enumsspb.WORKFLOW_EXECUTION_STATE_VOID, - toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, - wantErr: false, - }, - } - - for _, c := range cases { - s.Run(c.name, func() { - s.SetupSubTest() - s.mutableState.executionState.State = c.currentState - // default current status to RUNNING unless specified - curStatus := c.currentStatus - if curStatus == 0 { - curStatus = enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING - } - s.mutableState.executionState.Status = curStatus - _, err := s.mutableState.UpdateWorkflowStateStatus(c.toState, c.toStatus) - if c.wantErr { - s.Error(err) - } else { - s.NoError(err) - } - if !c.wantErr { // if the transition was successful, verify the state and status are updated. - s.Equal(c.toState, s.mutableState.executionState.State) - s.Equal(c.toStatus, s.mutableState.executionState.Status) - } - }) - } -} - -func (s *mutableStateSuite) TestAddWorkflowExecutionPausedEvent() { - s.SetupSubTest() - s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() - - tq := &taskqueuepb.TaskQueue{Name: "tq"} - s.createVersionedMutableStateWithCompletedWFT(tq) - - // Complete another WFT to obtain a valid completed event id for scheduling an activity. - wft, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) - s.NoError(err) - _, wft, err = s.mutableState.AddWorkflowTaskStartedEvent( - wft.ScheduledEventID, - "", - tq, - "", - worker_versioning.StampFromBuildId("b1"), - nil, - nil, - false, - ) - s.NoError(err) - completedEvent, err := s.mutableState.AddWorkflowTaskCompletedEvent( - wft, - &workflowservice.RespondWorkflowTaskCompletedRequest{}, - workflowTaskCompletionLimits, - ) - s.NoError(err) - - // Schedule an activity (pending) using the completed WFT event id. - _, activityInfo, err := s.mutableState.AddActivityTaskScheduledEvent( - completedEvent.GetEventId(), - &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: "act-1", - ActivityType: &commonpb.ActivityType{Name: "activity-type"}, - TaskQueue: tq, - }, - false, - ) - s.NoError(err) - prevActivityStamp := activityInfo.Stamp - - // Create a pending workflow task. - pendingWFT, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) - s.NoError(err) - prevWFTStamp := pendingWFT.Stamp - - // Pause and assert stamps incremented. - pausedEvent, err := s.mutableState.AddWorkflowExecutionPausedEvent("tester", "reason", uuid.NewString()) - s.NoError(err) - - updatedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) - s.True(ok) - s.Greater(updatedActivityInfo.Stamp, prevActivityStamp) - - wftInfo := s.mutableState.GetPendingWorkflowTask() - s.NotNil(wftInfo) - s.Greater(wftInfo.Stamp, prevWFTStamp) - - // assert the event is marked as 'worker may ignore' so that older SDKs can safely ignore it. - s.True(pausedEvent.GetWorkerMayIgnore()) -} - -func (s *mutableStateSuite) TestAddWorkflowExecutionUnpausedEvent() { - s.SetupSubTest() - s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() - - tq := &taskqueuepb.TaskQueue{Name: "tq"} - s.createVersionedMutableStateWithCompletedWFT(tq) - - // Complete another WFT to obtain a valid completed event id for scheduling an activity. - wft, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) - s.NoError(err) - _, wft, err = s.mutableState.AddWorkflowTaskStartedEvent( - wft.ScheduledEventID, - "", - tq, - "", - worker_versioning.StampFromBuildId("b1"), - nil, - nil, - false, - ) - s.NoError(err) - completedEvent, err := s.mutableState.AddWorkflowTaskCompletedEvent( - wft, - &workflowservice.RespondWorkflowTaskCompletedRequest{}, - workflowTaskCompletionLimits, - ) - s.NoError(err) - - // Schedule an activity (pending) using the completed WFT event id. - _, activityInfo, err := s.mutableState.AddActivityTaskScheduledEvent( - completedEvent.GetEventId(), - &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: "act-1", - ActivityType: &commonpb.ActivityType{Name: "activity-type"}, - TaskQueue: tq, - }, - false, - ) - s.NoError(err) - // Create a pending workflow task. - pendingWFT, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) - s.NoError(err) - - // Pause first to simulate paused workflow state. - _, err = s.mutableState.AddWorkflowExecutionPausedEvent("tester", "reason", uuid.NewString()) - s.NoError(err) - - // Capture stamps after pause. - pausedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) - s.True(ok) - pausedActivityStamp := pausedActivityInfo.Stamp - pausedWFT := s.mutableState.GetPendingWorkflowTask() - s.NotNil(pausedWFT) - pausedWFTStamp := pausedWFT.Stamp - - // Unpause and verify. - unpausedEvent, err := s.mutableState.AddWorkflowExecutionUnpausedEvent("tester", "reason", uuid.NewString()) - s.NoError(err) - - // PauseInfo should be cleared and status should be RUNNING. - s.Nil(s.mutableState.executionInfo.PauseInfo) - s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, s.mutableState.executionState.Status) - - // Stamps should be incremented again (only for activities) on unpause. - updatedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) - s.True(ok) - s.Greater(updatedActivityInfo.Stamp, pausedActivityStamp) - - currentWFT := s.mutableState.GetPendingWorkflowTask() - s.NotNil(currentWFT) - s.Equal(currentWFT.Stamp, pausedWFTStamp) // workflow task stamp should not change between pause and unpause. - - // assert the event is marked as 'worker may ignore' so that older SDKs can safely ignore it. - s.True(unpausedEvent.GetWorkerMayIgnore()) - - // Ensure the pending workflow task we created earlier still exists (no unexpected removal). - s.Equal(pendingWFT.ScheduledEventID, currentWFT.ScheduledEventID) -} - -func (s *mutableStateSuite) TestPauseWorkflowExecution_FailStateValidation() { - s.SetupSubTest() - s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() - - // Simulate a completed workflow where transitioning status to PAUSED is invalid. - s.mutableState.executionState.State = enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED - s.mutableState.executionState.Status = enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED - prevStatus := s.mutableState.executionState.Status - - _, err := s.mutableState.AddWorkflowExecutionPausedEvent("tester", "test_reason", uuid.NewString()) - s.Error(err) - // Status should remain unchanged and PauseInfo should not be set when validation fails. - s.Equal(prevStatus, s.mutableState.executionState.Status) - s.Nil(s.mutableState.executionInfo.PauseInfo) -} - func (s *mutableStateSuite) TestContinueAsNewMinBackoff() { // set ContinueAsNew min interval to 5s s.mockConfig.WorkflowIdReuseMinimalInterval = func(namespace string) time.Duration { @@ -1905,7 +1313,7 @@ func (s *mutableStateSuite) TestContinueAsNewMinBackoff() { } func (s *mutableStateSuite) TestEventReapplied() { - runID := uuid.NewString() + runID := uuid.New() eventID := int64(1) version := int64(2) dedupResource := definition.NewEventReappliedID(runID, eventID, version) @@ -1919,7 +1327,7 @@ func (s *mutableStateSuite) TestEventReapplied() { func (s *mutableStateSuite) TestTransientWorkflowTaskSchedule_CurrentVersionChanged() { version := int64(2000) workflowID := "some random workflow ID" - runID := uuid.NewString() + runID := uuid.New() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -1954,7 +1362,7 @@ func (s *mutableStateSuite) TestTransientWorkflowTaskSchedule_CurrentVersionChan func (s *mutableStateSuite) TestTransientWorkflowTaskStart_CurrentVersionChanged() { version := int64(2000) workflowID := "some random workflow ID" - runID := uuid.NewString() + runID := uuid.New() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -1988,7 +1396,7 @@ func (s *mutableStateSuite) TestTransientWorkflowTaskStart_CurrentVersionChanged _, _, err = s.mutableState.AddWorkflowTaskStartedEvent( s.mutableState.GetNextEventID(), - uuid.NewString(), + uuid.New(), &taskqueuepb.TaskQueue{Name: f.TaskQueue(enumspb.TASK_QUEUE_TYPE_WORKFLOW).NormalPartition(5).RpcName()}, "random identity", nil, @@ -2024,7 +1432,7 @@ func (s *mutableStateSuite) TestNewMutableStateInChain() { s.logger, 1000, tests.WorkflowID, - uuid.NewString(), + uuid.New(), ) currentMutableState.GetExecutionInfo().WorkflowExecutionTimerTaskStatus = taskStatus @@ -2034,7 +1442,7 @@ func (s *mutableStateSuite) TestNewMutableStateInChain() { s.logger, tests.GlobalNamespaceEntry, tests.WorkflowID, - uuid.NewString(), + uuid.New(), s.mockShard.GetTimeSource().Now(), currentMutableState, ) @@ -2047,7 +1455,7 @@ func (s *mutableStateSuite) TestNewMutableStateInChain() { func (s *mutableStateSuite) TestSanitizedMutableState() { txnID := int64(2000) - runID := uuid.NewString() + runID := uuid.New() mutableState := TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -2148,7 +1556,7 @@ func (s *mutableStateSuite) prepareTransientWorkflowTaskCompletionFirstBatchAppl EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, Attributes: &historypb.HistoryEvent_WorkflowTaskStartedEventAttributes{WorkflowTaskStartedEventAttributes: &historypb.WorkflowTaskStartedEventAttributes{ ScheduledEventId: workflowTaskScheduleEvent.GetEventId(), - RequestId: uuid.NewString(), + RequestId: uuid.New(), }}, } eventID++ @@ -2178,7 +1586,7 @@ func (s *mutableStateSuite) prepareTransientWorkflowTaskCompletionFirstBatchAppl err := s.mutableState.ApplyWorkflowExecutionStartedEvent( nil, execution, - uuid.NewString(), + uuid.New(), workflowStartEvent, ) s.Nil(err) @@ -2236,7 +1644,7 @@ func (s *mutableStateSuite) prepareTransientWorkflowTaskCompletionFirstBatchAppl EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, Attributes: &historypb.HistoryEvent_WorkflowTaskStartedEventAttributes{WorkflowTaskStartedEventAttributes: &historypb.WorkflowTaskStartedEventAttributes{ ScheduledEventId: workflowTaskScheduleEvent.GetEventId(), - RequestId: uuid.NewString(), + RequestId: uuid.New(), }}, } eventID++ @@ -2332,7 +1740,7 @@ func (s *mutableStateSuite) buildWorkflowMutableState() *persistencespb.Workflow TransitionCount: 1024, }, }, - FirstExecutionRunId: uuid.NewString(), + FirstExecutionRunId: uuid.New(), WorkflowExecutionTimerTaskStatus: TimerTaskStatusCreated, } @@ -2374,7 +1782,7 @@ func (s *mutableStateSuite) buildWorkflowMutableState() *persistencespb.Workflow InitiatedEventId: 80, InitiatedEventBatchId: 20, StartedEventId: common.EmptyEventID, - CreateRequestId: uuid.NewString(), + CreateRequestId: uuid.New(), Namespace: tests.Namespace.String(), WorkflowTypeName: "code.uber.internal/test/foobar", }, @@ -2384,7 +1792,7 @@ func (s *mutableStateSuite) buildWorkflowMutableState() *persistencespb.Workflow 70: { Version: failoverVersion, InitiatedEventBatchId: 20, - CancelRequestId: uuid.NewString(), + CancelRequestId: uuid.New(), InitiatedEventId: 70, }, } @@ -2394,7 +1802,7 @@ func (s *mutableStateSuite) buildWorkflowMutableState() *persistencespb.Workflow Version: failoverVersion, InitiatedEventId: 75, InitiatedEventBatchId: 17, - RequestId: uuid.NewString(), + RequestId: uuid.New(), }, } @@ -2671,7 +2079,7 @@ func (s *mutableStateSuite) TestTotalEntitiesCount() { _, _, err = s.mutableState.AddStartChildWorkflowExecutionInitiatedEvent( workflowTaskCompletedEventID, &commandpb.StartChildWorkflowExecutionCommandAttributes{}, - namespace.ID(uuid.NewString()), + namespace.ID(uuid.New()), ) s.NoError(err) @@ -2683,22 +2091,22 @@ func (s *mutableStateSuite) TestTotalEntitiesCount() { _, _, err = s.mutableState.AddRequestCancelExternalWorkflowExecutionInitiatedEvent( workflowTaskCompletedEventID, - uuid.NewString(), + uuid.New(), &commandpb.RequestCancelExternalWorkflowExecutionCommandAttributes{}, - namespace.ID(uuid.NewString()), + namespace.ID(uuid.New()), ) s.NoError(err) _, _, err = s.mutableState.AddSignalExternalWorkflowExecutionInitiatedEvent( workflowTaskCompletedEventID, - uuid.NewString(), + uuid.New(), &commandpb.SignalExternalWorkflowExecutionCommandAttributes{ Execution: &commonpb.WorkflowExecution{ WorkflowId: tests.WorkflowID, RunId: tests.RunID, }, }, - namespace.ID(uuid.NewString()), + namespace.ID(uuid.New()), ) s.NoError(err) @@ -2826,7 +2234,6 @@ func (s *mutableStateSuite) TestRetryWorkflowTask_WithNextRetryDelay() { } func (s *mutableStateSuite) TestRetryActivity_TruncateRetryableFailure() { s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() - s.mockConfig.EnableActivityRetryStampIncrement = dynamicconfig.GetBoolPropertyFn(true) // scheduling, starting & completing workflow task is omitted here @@ -2848,7 +2255,7 @@ func (s *mutableStateSuite) TestRetryActivity_TruncateRetryableFailure() { _, err = s.mutableState.AddActivityTaskStartedEvent( activityInfo, activityInfo.ScheduledEventId, - uuid.NewString(), + uuid.New(), "worker-identity", nil, nil, @@ -2877,63 +2284,16 @@ func (s *mutableStateSuite) TestRetryActivity_TruncateRetryableFailure() { } s.Greater(activityFailure.Size(), failureSizeErrorLimit) - prevStamp := activityInfo.Stamp - retryState, err := s.mutableState.RetryActivity(activityInfo, activityFailure) s.NoError(err) s.Equal(enumspb.RETRY_STATE_IN_PROGRESS, retryState) activityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) s.True(ok) - s.Greater(activityInfo.Stamp, prevStamp) - s.Equal(int32(2), activityInfo.Attempt) s.LessOrEqual(activityInfo.RetryLastFailure.Size(), failureSizeErrorLimit) s.Equal(activityFailure.GetMessage(), activityInfo.RetryLastFailure.Cause.GetMessage()) } -func (s *mutableStateSuite) TestRetryActivity_PausedIncrementsStamp() { - s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() - s.mockConfig.EnableActivityRetryStampIncrement = dynamicconfig.GetBoolPropertyFn(true) - - workflowTaskCompletedEventID := int64(4) - _, activityInfo, err := s.mutableState.AddActivityTaskScheduledEvent( - workflowTaskCompletedEventID, - &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: "6", - ActivityType: &commonpb.ActivityType{Name: "activity-type"}, - TaskQueue: &taskqueuepb.TaskQueue{Name: "task-queue"}, - RetryPolicy: &commonpb.RetryPolicy{ - InitialInterval: timestamp.DurationFromSeconds(1), - }, - }, - false, - ) - s.NoError(err) - - _, err = s.mutableState.AddActivityTaskStartedEvent( - activityInfo, - activityInfo.ScheduledEventId, - uuid.NewString(), - "worker-identity", - nil, - nil, - nil, - ) - s.NoError(err) - - activityInfo.Paused = true - prevStamp := activityInfo.Stamp - - retryState, err := s.mutableState.RetryActivity(activityInfo, &failurepb.Failure{Message: "activity failure"}) - s.NoError(err) - s.Equal(enumspb.RETRY_STATE_IN_PROGRESS, retryState) - - updatedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) - s.True(ok) - s.Greater(updatedActivityInfo.Stamp, prevStamp) - s.Equal(int32(2), updatedActivityInfo.Attempt) -} - func (s *mutableStateSuite) TestupdateBuildIdsAndDeploymentSearchAttributes() { versioned := func(buildId string) *commonpb.WorkerVersionStamp { return &commonpb.WorkerVersionStamp{BuildId: buildId, UseVersioning: true} @@ -3053,9 +2413,9 @@ func (s *mutableStateSuite) TestAddResetPointFromCompletion() { } func (s *mutableStateSuite) TestRolloverAutoResetPointsWithExpiringTime() { - runID1 := uuid.NewString() - runID2 := uuid.NewString() - runID3 := uuid.NewString() + runId1 := uuid.New() + runId2 := uuid.New() + runId3 := uuid.New() retention := 3 * time.Hour base := time.Now() @@ -3067,40 +2427,40 @@ func (s *mutableStateSuite) TestRolloverAutoResetPointsWithExpiringTime() { points := []*workflowpb.ResetPointInfo{ { BuildId: "buildid1", - RunId: runID1, + RunId: runId1, FirstWorkflowTaskCompletedId: 32, ExpireTime: t1, }, { BuildId: "buildid2", - RunId: runID1, + RunId: runId1, FirstWorkflowTaskCompletedId: 63, ExpireTime: t1, }, { BuildId: "buildid3", - RunId: runID2, + RunId: runId2, FirstWorkflowTaskCompletedId: 94, ExpireTime: t2, }, { BuildId: "buildid4", - RunId: runID3, + RunId: runId3, FirstWorkflowTaskCompletedId: 125, }, } - newPoints := rolloverAutoResetPointsWithExpiringTime(&workflowpb.ResetPoints{Points: points}, runID3, now.AsTime(), retention) + newPoints := rolloverAutoResetPointsWithExpiringTime(&workflowpb.ResetPoints{Points: points}, runId3, now.AsTime(), retention) expected := []*workflowpb.ResetPointInfo{ { BuildId: "buildid3", - RunId: runID2, + RunId: runId2, FirstWorkflowTaskCompletedId: 94, ExpireTime: t2, }, { BuildId: "buildid4", - RunId: runID3, + RunId: runId3, FirstWorkflowTaskCompletedId: 125, ExpireTime: t3, }, @@ -3242,7 +2602,7 @@ func (s *mutableStateSuite) TestCloseTransactionUpdateTransition() { }, txFunc: func(ms historyi.MutableState) (*persistencespb.WorkflowExecutionInfo, error) { mockChasmTree := historyi.NewMockChasmTree(s.controller) - mockChasmTree.EXPECT().ArchetypeID().Return(chasm.ArchetypeID(1234)).AnyTimes() + mockChasmTree.EXPECT().Archetype().Return(chasm.Archetype("mock-archetype")).AnyTimes() gomock.InOrder( mockChasmTree.EXPECT().IsStateDirty().Return(true).AnyTimes(), mockChasmTree.EXPECT().CloseTransaction().Return(chasm.NodesMutation{ @@ -3461,7 +2821,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackLastUpdateVersionedTransiti completedEvent := completWorkflowTaskFn(ms) initiatedEvent, _, err := ms.AddRequestCancelExternalWorkflowExecutionInitiatedEvent( completedEvent.GetEventId(), - uuid.NewString(), + uuid.New(), &commandpb.RequestCancelExternalWorkflowExecutionCommandAttributes{}, ms.GetNamespaceEntry().ID(), ) @@ -3488,7 +2848,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackLastUpdateVersionedTransiti completedEvent := completWorkflowTaskFn(ms) initiatedEvent, _, err := ms.AddSignalExternalWorkflowExecutionInitiatedEvent( completedEvent.GetEventId(), - uuid.NewString(), + uuid.New(), &commandpb.SignalExternalWorkflowExecutionCommandAttributes{ Execution: &commonpb.WorkflowExecution{ WorkflowId: "target-workflow-id", @@ -3517,7 +2877,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackLastUpdateVersionedTransiti { name: "SignalRequestedID", testFn: func(ms historyi.MutableState) { - ms.AddSignalRequested(uuid.NewString()) + ms.AddSignalRequested(uuid.New()) _, _, err := ms.CloseTransactionAsMutation(historyi.TransactionPolicyActive) s.NoError(err) @@ -3836,7 +3196,7 @@ func (s *mutableStateSuite) TestCloseTransactionHandleUnknownVersionedTransition } func (s *mutableStateSuite) getBuildIdsFromMutableState() []string { - payload, found := s.mutableState.executionInfo.SearchAttributes[sadefs.BuildIds] + payload, found := s.mutableState.executionInfo.SearchAttributes[searchattribute.BuildIds] if !found { return []string{} } @@ -4113,7 +3473,7 @@ func (s *mutableStateSuite) TestCloseTransactionPrepareReplicationTasks_HistoryT EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, Attributes: &historypb.HistoryEvent_WorkflowTaskStartedEventAttributes{WorkflowTaskStartedEventAttributes: &historypb.WorkflowTaskStartedEventAttributes{ ScheduledEventId: firstEventID, - RequestId: uuid.NewString(), + RequestId: uuid.New(), }}, }, }, @@ -4213,7 +3573,7 @@ func (s *mutableStateSuite) TestCloseTransactionPrepareReplicationTasks_SyncVers EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, Attributes: &historypb.HistoryEvent_WorkflowTaskStartedEventAttributes{WorkflowTaskStartedEventAttributes: &historypb.WorkflowTaskStartedEventAttributes{ ScheduledEventId: firstEventID, - RequestId: uuid.NewString(), + RequestId: uuid.New(), }}, }, }, @@ -4254,7 +3614,6 @@ func (s *mutableStateSuite) TestCloseTransactionPrepareReplicationTasks_SyncVers } expectedTask := &tasks.SyncVersionedTransitionTask{ WorkflowKey: s.mutableState.GetWorkflowKey(), - ArchetypeID: chasm.WorkflowArchetypeID, VisibilityTimestamp: now, Priority: enumsspb.TASK_PRIORITY_HIGH, VersionedTransition: transitionHistory[0], @@ -4266,7 +3625,6 @@ func (s *mutableStateSuite) TestCloseTransactionPrepareReplicationTasks_SyncVers s.True(ok) s.Equal(expectedTask.WorkflowKey, actualTask.WorkflowKey) s.Equal(expectedTask.VersionedTransition, actualTask.VersionedTransition) - s.Equal(expectedTask.ArchetypeID, actualTask.ArchetypeID) s.Equal(3, len(actualTask.TaskEquivalents)) s.Equal(historyTasks[0], actualTask.TaskEquivalents[0]) s.Equal(historyTasks[1], actualTask.TaskEquivalents[1]) @@ -4589,8 +3947,8 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones() { break } childExecution := &commonpb.WorkflowExecution{ - WorkflowId: uuid.NewString(), - RunId: uuid.NewString(), + WorkflowId: uuid.New(), + RunId: uuid.New(), } _, err := mutableState.AddChildWorkflowExecutionStartedEvent( childExecution, @@ -4624,8 +3982,8 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones() { initiatedEventId, s.namespaceEntry.Name(), s.namespaceEntry.ID(), - uuid.NewString(), - uuid.NewString(), + uuid.New(), + uuid.New(), enumspb.CANCEL_EXTERNAL_WORKFLOW_EXECUTION_FAILED_CAUSE_EXTERNAL_WORKFLOW_EXECUTION_NOT_FOUND, ) return &persistencespb.StateMachineTombstone{ @@ -4646,8 +4004,8 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones() { initiatedEventId, s.namespaceEntry.Name(), s.namespaceEntry.ID(), - uuid.NewString(), - uuid.NewString(), + uuid.New(), + uuid.New(), "", enumspb.SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_FAILED_CAUSE_EXTERNAL_WORKFLOW_EXECUTION_NOT_FOUND, ) @@ -4669,7 +4027,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones() { } mockChasmTree := historyi.NewMockChasmTree(s.controller) - mockChasmTree.EXPECT().ArchetypeID().Return(chasm.ArchetypeID(1234)).AnyTimes() + mockChasmTree.EXPECT().Archetype().Return(chasm.Archetype("mock-archetype")).AnyTimes() gomock.InOrder( mockChasmTree.EXPECT().IsStateDirty().Return(true).AnyTimes(), mockChasmTree.EXPECT().CloseTransaction().Return(chasm.NodesMutation{ @@ -4748,7 +4106,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones_CapIfLargerThanL Version: s.namespaceEntry.FailoverVersion(), InitiatedEventId: int64(76 + i), InitiatedEventBatchId: 17, - RequestId: uuid.NewString(), + RequestId: uuid.New(), } } @@ -4760,8 +4118,8 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones_CapIfLargerThanL initiatedEventId, s.namespaceEntry.Name(), s.namespaceEntry.ID(), - uuid.NewString(), - uuid.NewString(), + uuid.New(), + uuid.New(), "", enumspb.SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_FAILED_CAUSE_EXTERNAL_WORKFLOW_EXECUTION_NOT_FOUND, ) @@ -4804,7 +4162,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones_OnlyTrackFirstEm s.Equal(int64(1), tombstoneBatches[0].VersionedTransition.TransitionCount) } -func (s *mutableStateSuite) TestCloseTransactionGenerateCHASMRetentionTask_Workflow() { +func (s *mutableStateSuite) TestCloseTransactionGenerateCHASMRetentionTask() { dbState := s.buildWorkflowMutableState() mutableState, err := NewMutableStateFromDB(s.mockShard, s.mockEventsCache, s.logger, s.namespaceEntry, dbState, 123) @@ -4822,38 +4180,20 @@ func (s *mutableStateSuite) TestCloseTransactionGenerateCHASMRetentionTask_Workf // Is workflow, should not generate retention task mockChasmTree.EXPECT().IsStateDirty().Return(true).AnyTimes() - mockChasmTree.EXPECT().ArchetypeID().Return(chasm.WorkflowArchetypeID).AnyTimes() + mockChasmTree.EXPECT().Archetype().Return(chasmworkflow.Archetype).Times(1) mockChasmTree.EXPECT().CloseTransaction().Return(chasm.NodesMutation{}, nil).AnyTimes() mutation, _, err := mutableState.CloseTransactionAsMutation(historyi.TransactionPolicyActive) s.NoError(err) s.Empty(mutation.Tasks[tasks.CategoryTimer]) -} - -func (s *mutableStateSuite) TestCloseTransactionGenerateCHASMRetentionTask_NonWorkflow() { - dbState := s.buildWorkflowMutableState() - mutableState, err := NewMutableStateFromDB(s.mockShard, s.mockEventsCache, s.logger, s.namespaceEntry, dbState, 123) - s.NoError(err) - - // First close transaction once to get rid of unrelated tasks like UserTimer and ActivityTimeout - _, err = mutableState.StartTransaction(s.namespaceEntry) - s.NoError(err) - _, _, err = mutableState.CloseTransactionAsMutation(historyi.TransactionPolicyActive) - s.NoError(err) - - // Switch to a mock CHASM tree - mockChasmTree := historyi.NewMockChasmTree(s.controller) - mutableState.chasmTree = mockChasmTree - - mockChasmTree.EXPECT().IsStateDirty().Return(true).AnyTimes() - mockChasmTree.EXPECT().ArchetypeID().Return(chasm.WorkflowArchetypeID + 101).AnyTimes() - mockChasmTree.EXPECT().CloseTransaction().Return(chasm.NodesMutation{}, nil).AnyTimes() + // Now make the mutable state non-workflow. + mockChasmTree.EXPECT().Archetype().Return(chasm.Archetype("test-archetype")).Times(2) // One time for each CloseTransactionAsMutation call _, err = mutableState.UpdateWorkflowStateStatus( enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, ) s.NoError(err) - mutation, _, err := mutableState.CloseTransactionAsMutation(historyi.TransactionPolicyActive) + mutation, _, err = mutableState.CloseTransactionAsMutation(historyi.TransactionPolicyActive) s.NoError(err) s.Len(mutation.Tasks[tasks.CategoryTimer], 1) s.Equal(enumsspb.TASK_TYPE_DELETE_HISTORY_EVENT, mutation.Tasks[tasks.CategoryTimer][0].GetType()) @@ -4896,8 +4236,8 @@ func (s *mutableStateSuite) addChangesForStateReplication(state *persistencespb. state.ActivityInfos[90].TimerTaskStatus = TimerTaskStatusCreated state.TimerInfos["25"].ExpiryTime = timestamp.TimeNowPtrUtcAddDuration(time.Hour) state.ChildExecutionInfos[80].StartedEventId = 84 - state.RequestCancelInfos[70].CancelRequestId = uuid.NewString() - state.SignalInfos[75].RequestId = uuid.NewString() + state.RequestCancelInfos[70].CancelRequestId = uuid.New() + state.SignalInfos[75].RequestId = uuid.New() // These infos will be deleted during ApplySnapshot state.ActivityInfos[89] = &persistencespb.ActivityInfo{} @@ -5040,7 +4380,6 @@ func (s *mutableStateSuite) verifyExecutionInfo(current, target, origin *persist s.Equal(target.StickyTaskQueue, current.StickyTaskQueue, "StickyTaskQueue mismatch") s.True(proto.Equal(target.StickyScheduleToStartTimeout, current.StickyScheduleToStartTimeout), "StickyScheduleToStartTimeout mismatch") s.Equal(target.Attempt, current.Attempt, "Attempt mismatch") - s.Equal(target.WorkflowTaskStamp, current.WorkflowTaskStamp, "WorkflowTaskStamp mismatch") s.True(proto.Equal(target.RetryInitialInterval, current.RetryInitialInterval), "RetryInitialInterval mismatch") s.True(proto.Equal(target.RetryMaximumInterval, current.RetryMaximumInterval), "RetryMaximumInterval mismatch") s.Equal(target.RetryMaximumAttempts, current.RetryMaximumAttempts, "RetryMaximumAttempts mismatch") @@ -5824,77 +5163,3 @@ func (s *mutableStateSuite) TestHasRequestID_EmptyExecutionState() { s.False(s.mutableState.HasRequestID(requestID), "Should return false for request ID: %s", requestID) } } - -func (s *mutableStateSuite) TestAddTasks_CHASMPureTask() { - s.mockConfig.ChasmMaxInMemoryPureTasks = dynamicconfig.GetIntPropertyFn(5) - totalTasks := 2 * s.mockConfig.ChasmMaxInMemoryPureTasks() - - visTimestamp := s.mockShard.GetTimeSource().Now() - for i := 0; i < totalTasks; i++ { - task := &tasks.ChasmTaskPure{ - VisibilityTimestamp: visTimestamp, - } - s.mutableState.AddTasks(task) - s.LessOrEqual(len(s.mutableState.chasmPureTasks), s.mockConfig.ChasmMaxInMemoryPureTasks()) - - visTimestamp = visTimestamp.Add(-time.Minute) - } - - s.mockConfig.ChasmMaxInMemoryPureTasks = dynamicconfig.GetIntPropertyFn(2) - s.mutableState.AddTasks(&tasks.ChasmTaskPure{ - VisibilityTimestamp: visTimestamp, - }) - s.Len(s.mutableState.chasmPureTasks, 2) -} - -func (s *mutableStateSuite) TestDeleteCHASMPureTasks() { - now := s.mockShard.GetTimeSource().Now() - - testCases := []struct { - name string - maxScheduledTime time.Time - expectedRemaining int - }{ - { - name: "none", - maxScheduledTime: now, - expectedRemaining: 3, - }, - { - name: "paritial", - maxScheduledTime: now.Add(2 * time.Minute), - expectedRemaining: 2, - }, - { - name: "all", - maxScheduledTime: now.Add(5 * time.Minute), - expectedRemaining: 0, - }, - } - - for _, tc := range testCases { - s.Run(tc.name, func() { - s.mutableState.chasmPureTasks = []*tasks.ChasmTaskPure{ - { - VisibilityTimestamp: now.Add(3 * time.Minute), - }, - { - VisibilityTimestamp: now.Add(2 * time.Minute), - }, - { - VisibilityTimestamp: now.Add(time.Minute), - }, - } - s.mutableState.BestEffortDeleteTasks = make(map[tasks.Category][]tasks.Key) - - s.mutableState.DeleteCHASMPureTasks(tc.maxScheduledTime) - - s.Len(s.mutableState.chasmPureTasks, tc.expectedRemaining) - for _, task := range s.mutableState.chasmPureTasks { - s.False(task.VisibilityTimestamp.Before(tc.maxScheduledTime)) - } - - s.Len(s.mutableState.BestEffortDeleteTasks[tasks.CategoryTimer], 3-tc.expectedRemaining) - }) - } -} diff --git a/service/matching/forwarder.go b/service/matching/forwarder.go index dcf9f846f72..c9eae36f6c6 100644 --- a/service/matching/forwarder.go +++ b/service/matching/forwarder.go @@ -130,8 +130,8 @@ func (fwdr *Forwarder) ForwardTask(ctx context.Context, task *internalTask) erro ScheduleToStartTimeout: expirationDuration, ForwardInfo: fwdr.getForwardInfo(task), VersionDirective: task.event.Data.GetVersionDirective(), - Priority: task.event.Data.GetPriority(), Stamp: task.event.Data.GetStamp(), + Priority: task.event.Data.GetPriority(), }, ) case enumspb.TASK_QUEUE_TYPE_ACTIVITY: diff --git a/service/matching/pri_forwarder.go b/service/matching/pri_forwarder.go index 88c513d7d64..7d433b7c853 100644 --- a/service/matching/pri_forwarder.go +++ b/service/matching/pri_forwarder.go @@ -88,6 +88,7 @@ func (f *priForwarder) ForwardTask(ctx context.Context, task *internalTask) erro ScheduleToStartTimeout: expirationDuration, ForwardInfo: f.getForwardInfo(task), VersionDirective: task.event.Data.GetVersionDirective(), + Stamp: task.event.Data.GetStamp(), Priority: task.event.Data.GetPriority(), Stamp: task.event.Data.GetStamp(), }, diff --git a/tests/priority_fairness_test.go b/tests/priority_fairness_test.go index 2c360514bd0..d4696bb07c6 100644 --- a/tests/priority_fairness_test.go +++ b/tests/priority_fairness_test.go @@ -9,21 +9,23 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + activitypb "go.temporal.io/api/activity/v1" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" - taskqueuepb "go.temporal.io/api/taskqueue/v1" + workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/adminservice/v1" - taskqueuespb "go.temporal.io/server/api/taskqueue/v1" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/payloads" + "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/testing/taskpoller" "go.temporal.io/server/common/testing/testvars" "go.temporal.io/server/tests/testcore" "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/fieldmaskpb" ) type PrioritySuite struct { @@ -68,7 +70,7 @@ func (s *PrioritySuite) TestPriority_Activity_Basic() { _, err := s.TaskPoller().PollAndHandleWorkflowTask( tv, func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.Len(task.History.Events, 3) + s.Equal(3, len(task.History.Events)) var wfidx int _, err := fmt.Sscanf(task.WorkflowExecution.WorkflowId, "wf%d", &wfidx) @@ -77,13 +79,8 @@ func (s *PrioritySuite) TestPriority_Activity_Basic() { var commands []*commandpb.Command for i, pri := range rand.Perm(Levels) { - pri += 1 // 1-based - input, err := payloads.Encode(wfidx, pri) + input, err := payloads.Encode(wfidx, pri+1) s.NoError(err) - priMsg := &commonpb.Priority{PriorityKey: int32(pri)} - if pri == (Levels+1)/2 { - priMsg = nil // nil should be treated as default (3) - } commands = append(commands, &commandpb.Command{ CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ @@ -92,8 +89,10 @@ func (s *PrioritySuite) TestPriority_Activity_Basic() { ActivityType: tv.ActivityType(), TaskQueue: tv.TaskQueue(), ScheduleToCloseTimeout: durationpb.New(time.Minute), - Priority: priMsg, - Input: input, + Priority: &commonpb.Priority{ + PriorityKey: int32(pri + 1), + }, + Input: input, }, }, }) @@ -127,7 +126,7 @@ func (s *PrioritySuite) TestPriority_Activity_Basic() { w := wrongorderness(runs) s.T().Log("wrongorderness:", w) - s.Less(w, 0.1) + s.Less(w, 0.15) } func (s *PrioritySuite) TestSubqueue_Migration() { @@ -155,7 +154,7 @@ func (s *PrioritySuite) TestSubqueue_Migration() { _, err := s.TaskPoller().PollAndHandleWorkflowTask( tv, func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.Len(task.History.Events, 3) + s.Equal(3, len(task.History.Events)) var commands []*commandpb.Command @@ -235,7 +234,6 @@ func wrongorderness(vs []int) float64 { type FairnessSuite struct { testcore.FunctionalTestBase - partitions int } func TestFairnessSuite(t *testing.T) { @@ -244,16 +242,14 @@ func TestFairnessSuite(t *testing.T) { } func (s *FairnessSuite) SetupSuite() { - s.partitions = 1 dynamicConfigOverrides := map[dynamicconfig.Key]any{ - dynamicconfig.MatchingUseNewMatcher.Key(): true, dynamicconfig.MatchingEnableFairness.Key(): true, dynamicconfig.MatchingGetTasksBatchSize.Key(): 20, dynamicconfig.MatchingGetTasksReloadAt.Key(): 5, dynamicconfig.NumPendingActivitiesLimitError.Key(): 1000, - // TODO: disable this and use default later? - dynamicconfig.MatchingNumTaskqueueReadPartitions.Key(): s.partitions, - dynamicconfig.MatchingNumTaskqueueWritePartitions.Key(): s.partitions, + // TODO: disable this later? + dynamicconfig.MatchingNumTaskqueueReadPartitions.Key(): 1, + dynamicconfig.MatchingNumTaskqueueWritePartitions.Key(): 1, } s.FunctionalTestBase.SetupSuiteWithCluster(testcore.WithDynamicConfigOverrides(dynamicConfigOverrides)) } @@ -285,7 +281,7 @@ func (s *FairnessSuite) TestFairness_Activity_Basic() { _, err := s.TaskPoller().PollAndHandleWorkflowTask( tv, func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.Len(task.History.Events, 3) + s.Equal(3, len(task.History.Events)) var wfidx int _, err := fmt.Sscanf(task.WorkflowExecution.WorkflowId, "wf%d", &wfidx) @@ -359,204 +355,194 @@ func unfairness(vs []int) float64 { return float64(totalDelay) / float64(len(firsts)*len(firsts)) } -func (s *FairnessSuite) testMigration(newMatcher, fairness bool) { +// TODO: rename +func (s *FairnessSuite) TestFairness_UpdateWorkflowExecutionOptions_InvalidatesPendingTask() { tv := testvars.New(s.T()) - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - s.OverrideDynamicConfig(dynamicconfig.MatchingEnableMigration, true) - - forTest := func(v any) any { - return []dynamicconfig.ConstrainedValue{ - // test tqs (both wf and activity) - dynamicconfig.ConstrainedValue{ - Constraints: dynamicconfig.Constraints{ - Namespace: s.Namespace().String(), - TaskQueueName: tv.TaskQueue().Name, - }, - Value: v, - }, - // default (match values in SetupSuite to avoid flapping) - dynamicconfig.ConstrainedValue{Value: true}, - } - } - setConfig := func(stage string, newNewMatcher, newFairness bool) { - newMatcher, fairness = newNewMatcher, newFairness - s.T().Log("setting config: "+stage, "newMatcher", newMatcher, "fairness", fairness) - s.OverrideDynamicConfig(dynamicconfig.MatchingUseNewMatcher, forTest(newMatcher)) - s.OverrideDynamicConfig(dynamicconfig.MatchingEnableFairness, forTest(fairness)) - } - waitForTasks := func(tp enumspb.TaskQueueType, onDraining, onActive int64) { - s.T().Helper() - s.EventuallyWithT(func(c *assert.CollectT) { - tasksOnDraining, tasksOnActive, err := s.countTasksByDrainingActive(ctx, tv, tp) - require.NoError(c, err) - require.Equal(c, onDraining, tasksOnDraining) - require.Equal(c, onActive, tasksOnActive) - }, 15*time.Second, 250*time.Millisecond) - } + capture := s.GetTestCluster().Host().CaptureMetricsHandler().StartCapture() + defer s.GetTestCluster().Host().CaptureMetricsHandler().StopCapture(capture) - setConfig("initial", newMatcher, fairness) + ctx, cancel := context.WithTimeout(s.T().Context(), 10*time.Second) + defer cancel() - // start 20 workflows. 20 tasks will be queued on wft queue. - s.T().Log("starting workflows") - for range 20 { - _, err := s.FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ - Namespace: s.Namespace().String(), - WorkflowId: uuid.NewString(), - WorkflowType: tv.WorkflowType(), - TaskQueue: tv.TaskQueue(), + originalPriority := &commonpb.Priority{FairnessKey: "KEY"} + updatedPriority := &commonpb.Priority{FairnessKey: "NEW_KEY"} + + // Queue up new workflow. + startResp, err := s.FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowId: tv.WorkflowID(), + WorkflowType: tv.WorkflowType(), + TaskQueue: tv.TaskQueue(), + Priority: originalPriority, + }) + s.NoError(err) + + // Wait for workflow task to be backlogged. + s.Eventually(func() bool { + resp, err := s.AdminClient().GetTaskQueueTasks(ctx, &adminservice.GetTaskQueueTasksRequest{ + Namespace: s.Namespace().String(), + TaskQueue: tv.TaskQueue().Name, + TaskQueueType: enumspb.TASK_QUEUE_TYPE_WORKFLOW, + BatchSize: 10, + MinPass: 1, }) - s.NoError(err) - } - waitForTasks(enumspb.TASK_QUEUE_TYPE_WORKFLOW, 0, 20) - - processWft := func() { - s.EventuallyWithT(func(c *assert.CollectT) { - _, err := s.TaskPoller().PollAndHandleWorkflowTask( - tv, - func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.Len(task.History.Events, 3) - - var commands []*commandpb.Command - - for i := range 2 { - input, err := payloads.Encode(i) - s.NoError(err) - commands = append(commands, &commandpb.Command{ - CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, - Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ - ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ - ActivityId: fmt.Sprintf("act%d", i), - ActivityType: tv.ActivityType(), - TaskQueue: tv.TaskQueue(), - ScheduleToCloseTimeout: durationpb.New(time.Minute), - Input: input, - }, + return err == nil && len(resp.GetTasks()) == 1 + }, 10*time.Second, 100*time.Millisecond) + + // Update workflow options to set a new priority. + updateResp, err := s.FrontendClient().UpdateWorkflowExecutionOptions(ctx, &workflowservice.UpdateWorkflowExecutionOptionsRequest{ + Namespace: s.Namespace().String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tv.WorkflowID(), + RunId: startResp.GetRunId(), + }, + WorkflowExecutionOptions: &workflowpb.WorkflowExecutionOptions{ + Priority: updatedPriority, + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"priority"}}, + }) + s.NoError(err) + s.NotNil(updateResp.GetWorkflowExecutionOptions()) + s.ProtoEqual(updatedPriority, updateResp.GetWorkflowExecutionOptions().GetPriority()) + + // Query workflow to verify workflow has the updated priority. + descResp, err := s.FrontendClient().DescribeWorkflowExecution(ctx, &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + Execution: &commonpb.WorkflowExecution{ + WorkflowId: tv.WorkflowID(), + RunId: startResp.GetRunId(), + }, + }) + s.NoError(err) + s.NotNil(descResp.GetWorkflowExecutionInfo()) + s.NotNil(descResp.GetWorkflowExecutionInfo().GetPriority()) + s.ProtoEqual(updatedPriority, descResp.GetWorkflowExecutionInfo().GetPriority()) + + // Poll for workflow task and schedule an activity. + _, err = s.TaskPoller().PollAndHandleWorkflowTask( + tv, + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.NotNil(task) + s.ContainsHistoryEvents(` + 3 WorkflowExecutionOptionsUpdated { "Priority": { "FairnessKey": "NEW_KEY" } } + 4 WorkflowTaskStarted + `, task.History.Events) + + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ + ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: tv.ActivityID(), + ActivityType: tv.ActivityType(), + TaskQueue: tv.TaskQueue(), + Priority: originalPriority, + ScheduleToCloseTimeout: durationpb.New(time.Minute), }, - }) - } - - return &workflowservice.RespondWorkflowTaskCompletedRequest{Commands: commands}, nil - }, - taskpoller.WithContext(ctx), - ) - assert.NoError(c, err) - }, 5*time.Second, time.Millisecond) - } - - // process half the workflow tasks and create two activities each. - // 10 tasks will be left on old workflow queue, 20 tasks will be queued on current activity queue. - s.T().Log("processing first half of wfts") - for range 10 { - processWft() - } - waitForTasks(enumspb.TASK_QUEUE_TYPE_WORKFLOW, 0, 10) - waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 0, 20) - - // switch fairness. queues will be reloaded. wft queue should drain old queue. - setConfig("switching fairness", true, !fairness) - - waitForTasks(enumspb.TASK_QUEUE_TYPE_WORKFLOW, 10, 0) - waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 20, 0) - - // process the other half of workflow tasks. these should come from the draining queue now. - // 20 tasks will be queued on new activity queue (still 20 on old). - s.T().Log("processing last half of wfts") - for range 5 { - processWft() - } - waitForTasks(enumspb.TASK_QUEUE_TYPE_WORKFLOW, 5, 0) - waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 20, 10) - for range 5 { - processWft() - } - waitForTasks(enumspb.TASK_QUEUE_TYPE_WORKFLOW, 0, 0) - waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 20, 20) - s.T().Log("wfts done") - - // process activities 1/3 at a time - processActivity := func() { - s.EventuallyWithT(func(c *assert.CollectT) { - _, err := s.TaskPoller().PollAndHandleActivityTask( - tv, - func(task *workflowservice.PollActivityTaskQueueResponse) (*workflowservice.RespondActivityTaskCompletedRequest, error) { - nothing, err := payloads.Encode() - s.NoError(err) - return &workflowservice.RespondActivityTaskCompletedRequest{Result: nothing}, nil + }, + }, }, - taskpoller.WithContext(ctx), - ) - assert.NoError(c, err) - }, 5*time.Second, time.Millisecond) - } - - s.T().Log("processing first 1/3 activities") - for range 13 { - processActivity() - } - waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 7, 20) - - setConfig("switching fairness again", true, !fairness) - waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 20, 7) - - s.T().Log("processing next 1/3 activities") - for range 14 { - processActivity() + }, nil + }, + taskpoller.WithContext(ctx), + ) + s.NoError(err) + + // Verify that 2 workflow tasks were sent to matching; and 1 was marked obsolete. + addWorkflowTaskCount := 0 + obsoleteWorkflowTaskCount := 0 + snap := capture.Snapshot() + for _, rec := range snap[metrics.ClientRequests.Name()] { + for key, val := range rec.Tags { + if key == metrics.OperationTagName && val == metrics.MatchingClientAddWorkflowTaskScope { + addWorkflowTaskCount++ + } + } } - waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 6, 7) - - setConfig("switching fairness last time", true, !fairness) - waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 7, 6) - - s.T().Log("processing last 1/3 activities") - for range 13 { - processActivity() + s.Equal(2, addWorkflowTaskCount, "Expected 2 workflow tasks to be dispatched to matching") + for _, rec := range snap[metrics.ClientFailures.Name()] { + for key, val := range rec.Tags { + if key == metrics.ErrorTypeTagName && val == fmt.Sprintf("%T", serviceerror.ObsoleteMatchingTask{}) { + obsoleteWorkflowTaskCount++ + } + } } - waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 0, 0) -} - -func (s *FairnessSuite) countTasksByDrainingActive(ctx context.Context, tv *testvars.TestVars, tp enumspb.TaskQueueType) ( - tasksOnDraining, tasksOnActive int64, retErr error, -) { - for i := range s.partitions { - res, err := s.AdminClient().DescribeTaskQueuePartition(ctx, &adminservice.DescribeTaskQueuePartitionRequest{ - Namespace: s.Namespace().String(), - TaskQueuePartition: &taskqueuespb.TaskQueuePartition{ - TaskQueue: tv.TaskQueue().Name, - TaskQueueType: tp, - PartitionId: &taskqueuespb.TaskQueuePartition_NormalPartitionId{NormalPartitionId: int32(i)}, - }, - BuildIds: &taskqueuepb.TaskQueueVersionSelection{Unversioned: true}, + s.Equal(1, obsoleteWorkflowTaskCount, "Expected 1 worklow task to be obsolete") + + // Wait for activity task to be backlogged + s.Eventually(func() bool { + resp, err := s.AdminClient().GetTaskQueueTasks(ctx, &adminservice.GetTaskQueueTasksRequest{ + Namespace: s.Namespace().String(), + TaskQueue: tv.TaskQueue().Name, + TaskQueueType: enumspb.TASK_QUEUE_TYPE_ACTIVITY, + BatchSize: 10, + MinPass: 1, }) - if err != nil { - return 0, 0, err + return err == nil && len(resp.GetTasks()) == 1 + }, 10*time.Second, 100*time.Millisecond) + + // Update activity options to set a new priority + _, err = s.FrontendClient().UpdateActivityOptions(ctx, &workflowservice.UpdateActivityOptionsRequest{ + Namespace: s.Namespace().String(), + Execution: &commonpb.WorkflowExecution{ + WorkflowId: tv.WorkflowID(), + RunId: startResp.GetRunId(), + }, + Activity: &workflowservice.UpdateActivityOptionsRequest_Id{Id: tv.ActivityID()}, + ActivityOptions: &activitypb.ActivityOptions{ + Priority: updatedPriority, + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"priority"}}, + }) + s.NoError(err) + + // Query workflow to verify activity has the updated priority. + descResp, err = s.FrontendClient().DescribeWorkflowExecution(ctx, &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + Execution: &commonpb.WorkflowExecution{ + WorkflowId: tv.WorkflowID(), + RunId: startResp.GetRunId(), + }, + }) + s.NoError(err) + s.NotNil(descResp.GetPendingActivities()) + s.Len(descResp.GetPendingActivities(), 1) + s.ProtoEqual(updatedPriority, descResp.GetPendingActivities()[0].GetPriority()) + s.ProtoEqual(updatedPriority, descResp.GetPendingActivities()[0].GetActivityOptions().GetPriority()) + + // Poll for activity task and verify it has the updated priority. + _, err = s.TaskPoller().PollAndHandleActivityTask( + tv, + func(task *workflowservice.PollActivityTaskQueueResponse) (*workflowservice.RespondActivityTaskCompletedRequest, error) { + s.NotNil(task) + s.Equal(tv.ActivityID(), task.ActivityId) + s.ProtoEqual(updatedPriority, task.Priority) + return &workflowservice.RespondActivityTaskCompletedRequest{}, nil + }, + taskpoller.WithContext(ctx), + ) + s.NoError(err) + + // Verify that 2 activity tasks were sent to matching; and 1 was marked obsolete + addActivityTaskCount := 0 + obsoleteActivityTaskCount := 0 + snap = capture.Snapshot() + for _, rec := range snap[metrics.ClientRequests.Name()] { + for key, val := range rec.Tags { + if key == metrics.OperationTagName && val == metrics.MatchingClientAddActivityTaskScope { + addActivityTaskCount++ + } } - for _, versionInfoInternal := range res.VersionsInfoInternal { - for _, st := range versionInfoInternal.PhysicalTaskQueueInfo.InternalTaskQueueStatus { - if st.Draining { - tasksOnDraining += st.ApproximateBacklogCount - } else { - tasksOnActive += st.ApproximateBacklogCount - } + } + s.Equal(2, addActivityTaskCount, "Expected 2 activity tasks to be dispatched to matching") + for _, rec := range snap[metrics.ClientFailures.Name()] { + for key, val := range rec.Tags { + if key == metrics.ErrorTypeTagName && val == fmt.Sprintf("%T", serviceerror.ObsoleteMatchingTask{}) { + obsoleteActivityTaskCount++ } } } - return -} - -func (s *FairnessSuite) TestFairness_Migration_FromClassic() { - // classic->fair, fair->pri. fair metadata will be created on transition. - s.testMigration(false, false) -} - -func (s *FairnessSuite) TestFairness_Migration_FromPri() { - // pri->fair, fair->pri. fair metadata will be created before transition. - s.testMigration(true, false) -} - -func (s *FairnessSuite) TestFairness_Migration_FromFair() { - // fair->pri, pri->fair. fair metadata will be created first. - s.testMigration(true, true) + s.Equal(1+obsoleteWorkflowTaskCount, obsoleteActivityTaskCount, "Expected 1 activity task to be obsolete") } From 7f733c663ce26d2e9c6ebf5fb9f98e6b4622485f Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Fri, 7 Nov 2025 12:02:57 -0800 Subject: [PATCH 2/9] common.CloneProto --- service/history/api/updateworkflowoptions/api.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/service/history/api/updateworkflowoptions/api.go b/service/history/api/updateworkflowoptions/api.go index 623c8d7219d..496e73fc033 100644 --- a/service/history/api/updateworkflowoptions/api.go +++ b/service/history/api/updateworkflowoptions/api.go @@ -8,6 +8,7 @@ import ( "go.temporal.io/api/serviceerror" workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/server/api/historyservice/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/util" @@ -122,16 +123,10 @@ func MergeAndApply( func getOptionsFromMutableState(ms historyi.MutableState) *workflowpb.WorkflowExecutionOptions { opts := &workflowpb.WorkflowExecutionOptions{} if versioningInfo := ms.GetExecutionInfo().GetVersioningInfo(); versioningInfo != nil { - override, ok := proto.Clone(versioningInfo.GetVersioningOverride()).(*workflowpb.VersioningOverride) - if !ok { - return nil - } - opts.VersioningOverride = override + opts.VersioningOverride = common.CloneProto(versioningInfo.GetVersioningOverride()) } if priority := ms.GetExecutionInfo().GetPriority(); priority != nil { - if prio, ok := proto.Clone(priority).(*commonpb.Priority); ok { - opts.Priority = prio - } + opts.Priority = common.CloneProto(priority) } return opts } From 96d94819cce3420a03e304315469c4f79ad6dc30 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Fri, 7 Nov 2025 12:03:36 -0800 Subject: [PATCH 3/9] move to top --- service/history/api/updateworkflowoptions/api.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/service/history/api/updateworkflowoptions/api.go b/service/history/api/updateworkflowoptions/api.go index 496e73fc033..5cd3ca362ee 100644 --- a/service/history/api/updateworkflowoptions/api.go +++ b/service/history/api/updateworkflowoptions/api.go @@ -165,6 +165,10 @@ func mergeWorkflowExecutionOptions( // ==== VersioningOverride + if _, ok := updateFields["versioningOverride"]; ok { + mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() + } + if _, ok := updateFields["versioningOverride.deployment"]; ok { if _, ok := updateFields["versioningOverride.behavior"]; !ok { return nil, serviceerror.NewInvalidArgument("versioning_override fields must be updated together") @@ -179,9 +183,5 @@ func mergeWorkflowExecutionOptions( mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() } - if _, ok := updateFields["versioningOverride"]; ok { - mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() - } - return mergeInto, nil } From ed125d4ed5a6729e13eee28381f6ec66d8407b04 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Tue, 11 Nov 2025 19:32:00 -0800 Subject: [PATCH 4/9] re-order --- service/matching/forwarder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/matching/forwarder.go b/service/matching/forwarder.go index c9eae36f6c6..f014f5ee9c0 100644 --- a/service/matching/forwarder.go +++ b/service/matching/forwarder.go @@ -148,8 +148,8 @@ func (fwdr *Forwarder) ForwardTask(ctx context.Context, task *internalTask) erro ScheduleToStartTimeout: expirationDuration, ForwardInfo: fwdr.getForwardInfo(task), Stamp: task.event.Data.GetStamp(), - VersionDirective: task.event.Data.GetVersionDirective(), Priority: task.event.Data.GetPriority(), + VersionDirective: task.event.Data.GetVersionDirective(), }, ) default: From a5936ed117f086427222e9f4b4b9600741dd60ce Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Tue, 11 Nov 2025 19:34:20 -0800 Subject: [PATCH 5/9] remove TODO --- tests/priority_fairness_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/priority_fairness_test.go b/tests/priority_fairness_test.go index d4696bb07c6..cef96c13ca7 100644 --- a/tests/priority_fairness_test.go +++ b/tests/priority_fairness_test.go @@ -355,7 +355,6 @@ func unfairness(vs []int) float64 { return float64(totalDelay) / float64(len(firsts)*len(firsts)) } -// TODO: rename func (s *FairnessSuite) TestFairness_UpdateWorkflowExecutionOptions_InvalidatesPendingTask() { tv := testvars.New(s.T()) From 0bf3962effe757f9c01bc3ca17124bee060b29c9 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Tue, 11 Nov 2025 19:39:24 -0800 Subject: [PATCH 6/9] if instead of cmp.Or --- service/history/api/updateworkflowoptions/api.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/service/history/api/updateworkflowoptions/api.go b/service/history/api/updateworkflowoptions/api.go index 5cd3ca362ee..b442ba23375 100644 --- a/service/history/api/updateworkflowoptions/api.go +++ b/service/history/api/updateworkflowoptions/api.go @@ -149,17 +149,23 @@ func mergeWorkflowExecutionOptions( } if _, ok := updateFields["priority.priorityKey"]; ok { - mergeInto.Priority = cmp.Or(mergeInto.Priority, &commonpb.Priority{}) + if mergeInto.Priority == nil { + mergeInto.Priority = &commonpb.Priority{} + } mergeInto.Priority.PriorityKey = mergeFrom.GetPriority().GetPriorityKey() } if _, ok := updateFields["priority.fairnessKey"]; ok { - mergeInto.Priority = cmp.Or(mergeInto.Priority, &commonpb.Priority{}) + if mergeInto.Priority == nil { + mergeInto.Priority = &commonpb.Priority{} + } mergeInto.Priority.FairnessKey = mergeFrom.Priority.GetFairnessKey() } if _, ok := updateFields["priority.fairnessWeight"]; ok { - mergeInto.Priority = cmp.Or(mergeInto.Priority, &commonpb.Priority{}) + if mergeInto.Priority == nil { + mergeInto.Priority = &commonpb.Priority{} + } mergeInto.Priority.FairnessWeight = mergeFrom.Priority.GetFairnessWeight() } From d76b42ebe7c4c785bb4322fb15b62d05008f2f51 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Tue, 11 Nov 2025 19:41:48 -0800 Subject: [PATCH 7/9] fix imports --- service/history/api/updateworkflowoptions/api.go | 1 - 1 file changed, 1 deletion(-) diff --git a/service/history/api/updateworkflowoptions/api.go b/service/history/api/updateworkflowoptions/api.go index b442ba23375..b10c5a05e28 100644 --- a/service/history/api/updateworkflowoptions/api.go +++ b/service/history/api/updateworkflowoptions/api.go @@ -1,7 +1,6 @@ package updateworkflowoptions import ( - "cmp" "context" commonpb "go.temporal.io/api/common/v1" From 00b396f0e9f680d17905eed897f55225bf8b0400 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Tue, 11 Nov 2025 19:46:25 -0800 Subject: [PATCH 8/9] clone Priority --- service/history/api/updateactivityoptions/api.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service/history/api/updateactivityoptions/api.go b/service/history/api/updateactivityoptions/api.go index 14758086304..1d92d64934c 100644 --- a/service/history/api/updateactivityoptions/api.go +++ b/service/history/api/updateactivityoptions/api.go @@ -13,6 +13,7 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/historyservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/util" @@ -148,7 +149,7 @@ func processActivityOptionsUpdate( ScheduleToStartTimeout: ai.ScheduleToStartTimeout, StartToCloseTimeout: ai.StartToCloseTimeout, HeartbeatTimeout: ai.HeartbeatTimeout, - Priority: ai.Priority, + Priority: common.CloneProto(ai.Priority), RetryPolicy: &commonpb.RetryPolicy{ BackoffCoefficient: ai.RetryBackoffCoefficient, InitialInterval: ai.RetryInitialInterval, From a494b82fb3496e5b50977a6bff1b12f8237e1dd7 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 1 Dec 2025 15:47:01 -0800 Subject: [PATCH 9/9] rebase --- go.mod | 2 +- go.sum | 4 +- service/frontend/workflow_handler.go | 3 + service/frontend/workflow_handler_test.go | 8 +- service/history/api/startworkflow/api.go | 29 +- .../api/updateactivityoptions/api_test.go | 20 +- .../history/api/updateworkflowoptions/api.go | 58 +- .../api/updateworkflowoptions/api_test.go | 155 +-- .../history/historybuilder/event_factory.go | 34 + .../history/historybuilder/history_builder.go | 26 + .../history_builder_categorization_test.go | 2 +- service/history/interfaces/mutable_state.go | 21 +- .../history/interfaces/mutable_state_mock.go | 212 ++++- service/history/ndc/events_reapplier_test.go | 1 + service/history/ndc/workflow_resetter.go | 6 +- service/history/ndc/workflow_resetter_test.go | 46 +- .../history/workflow/mutable_state_impl.go | 725 ++++++++++++-- .../workflow/mutable_state_impl_test.go | 887 ++++++++++++++++-- service/matching/forwarder.go | 2 +- service/matching/pri_forwarder.go | 3 +- tests/priority_fairness_test.go | 235 ++++- 21 files changed, 2057 insertions(+), 422 deletions(-) diff --git a/go.mod b/go.mod index b4c1c938c84..dd12769c8bd 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,7 @@ require ( go.opentelemetry.io/otel/sdk v1.34.0 go.opentelemetry.io/otel/sdk/metric v1.34.0 go.opentelemetry.io/otel/trace v1.34.0 - go.temporal.io/api v1.58.1-0.20251126231839-2fcd2247e106 + go.temporal.io/api v1.58.1-0.20251128181858-703071215042 go.temporal.io/sdk v1.35.0 go.uber.org/fx v1.24.0 go.uber.org/mock v0.6.0 diff --git a/go.sum b/go.sum index b6cadbeeb2c..ea33209e437 100644 --- a/go.sum +++ b/go.sum @@ -390,8 +390,8 @@ go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= -go.temporal.io/api v1.58.1-0.20251126231839-2fcd2247e106 h1:V2H8rfBDapmWpIsNDWZLOS95WIIWAgPXnG7gpNrWO5Y= -go.temporal.io/api v1.58.1-0.20251126231839-2fcd2247e106/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM= +go.temporal.io/api v1.58.1-0.20251128181858-703071215042 h1:44+nPe+rGhYUwA1oDi46rkXEYEVfoAxOmb0myvTm4Es= +go.temporal.io/api v1.58.1-0.20251128181858-703071215042/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM= go.temporal.io/sdk v1.35.0 h1:lRNAQ5As9rLgYa7HBvnmKyzxLcdElTuoFJ0FXM/AsLQ= go.temporal.io/sdk v1.35.0/go.mod h1:1q5MuLc2MEJ4lneZTHJzpVebW2oZnyxoIOWX3oFVebw= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index b2b86366a60..d8904cbc772 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -5950,6 +5950,9 @@ func (wh *WorkflowHandler) UpdateWorkflowExecutionOptions( if err != nil { return nil, serviceerror.NewInvalidArgumentf("error parsing UpdateMask: %s", err.Error()) } + if err := priorities.Validate(opts.GetPriority()); err != nil { + return nil, err + } namespaceID, err := wh.namespaceRegistry.GetNamespaceID(namespace.Name(request.GetNamespace())) if err != nil { diff --git a/service/frontend/workflow_handler_test.go b/service/frontend/workflow_handler_test.go index 9fce1ddaa0c..061ecda3e28 100644 --- a/service/frontend/workflow_handler_test.go +++ b/service/frontend/workflow_handler_test.go @@ -936,7 +936,7 @@ func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Priority() { _, err := wh.StartWorkflowExecution(context.Background(), request) var invalidArg *serviceerror.InvalidArgument s.ErrorAs(err, &invalidArg) - s.ErrorContains(err, "PriorityKey can't be negative") + s.ErrorContains(err, "priority key can't be negative") // NOTE: only testing a single validation scenario here; the priority validation has its own unit tests } @@ -1059,7 +1059,7 @@ func (s *WorkflowHandlerSuite) TestSignalWithStartWorkflowExecution_Priority() { _, err := wh.SignalWithStartWorkflowExecution(context.Background(), request) var invalidArg *serviceerror.InvalidArgument s.ErrorAs(err, &invalidArg) - s.ErrorContains(err, "PriorityKey can't be negative") + s.ErrorContains(err, "priority key can't be negative") // NOTE: only testing a single validation scenario here; the priority validation has its own unit tests } @@ -4030,7 +4030,7 @@ func (s *WorkflowHandlerSuite) TestUpdateWorkflowExecutionOptions_Priority() { _, err := wh.UpdateWorkflowExecutionOptions(context.Background(), request) var invalidArg *serviceerror.InvalidArgument s.ErrorAs(err, &invalidArg) - s.ErrorContains(err, "PriorityKey can't be negative") + s.ErrorContains(err, "priority key can't be negative") // NOTE: only testing a single validation scenario here; the priority validation has its own unit tests } @@ -4056,6 +4056,6 @@ func (s *WorkflowHandlerSuite) TestUpdateActivityOptions_Priority() { _, err := wh.UpdateActivityOptions(context.Background(), request) var invalidArg *serviceerror.InvalidArgument s.ErrorAs(err, &invalidArg) - s.ErrorContains(err, "PriorityKey can't be negative") + s.ErrorContains(err, "priority key can't be negative") // NOTE: only testing a single validation scenario here; the priority validation has its own unit tests } diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index d7b06478fc6..613846794c3 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -20,8 +20,8 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" - "go.temporal.io/server/common/persistence/visibility/manager" "go.temporal.io/server/common/primitives" + "go.temporal.io/server/common/softassert" "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/consts" @@ -54,7 +54,6 @@ type Starter struct { shardContext historyi.ShardContext workflowConsistencyChecker api.WorkflowConsistencyChecker tokenSerializer *tasktoken.Serializer - visibilityManager manager.VisibilityManager request *historyservice.StartWorkflowExecutionRequest namespace *namespace.Namespace createOrUpdateLeaseFn api.CreateOrUpdateLeaseFunc @@ -85,7 +84,6 @@ func NewStarter( shardContext historyi.ShardContext, workflowConsistencyChecker api.WorkflowConsistencyChecker, tokenSerializer *tasktoken.Serializer, - visibilityManager manager.VisibilityManager, request *historyservice.StartWorkflowExecutionRequest, createLeaseFn api.CreateOrUpdateLeaseFunc, ) (*Starter, error) { @@ -100,7 +98,6 @@ func NewStarter( shardContext: shardContext, workflowConsistencyChecker: workflowConsistencyChecker, tokenSerializer: tokenSerializer, - visibilityManager: visibilityManager, request: request, namespace: namespaceEntry, createOrUpdateLeaseFn: createLeaseFn, @@ -254,7 +251,11 @@ func (s *Starter) prepareNewWorkflow(workflowID string) (*creationParams, error) workflowTaskInfo := mutableState.GetStartedWorkflowTask() if s.requestEagerStart() && workflowTaskInfo == nil { - return nil, serviceerror.NewInternal("unexpected error: mutable state did not have a started workflow task") + return nil, softassert.UnexpectedInternalErr( + s.shardContext.GetLogger(), + "unexpected error: mutable state did not have a started workflow task", + nil, + ) } workflowSnapshot, eventBatches, err := mutableState.CloseTransactionAsSnapshot( historyi.TransactionPolicyActive, @@ -263,7 +264,11 @@ func (s *Starter) prepareNewWorkflow(workflowID string) (*creationParams, error) return nil, err } if len(eventBatches) != 1 { - return nil, serviceerror.NewInternal("unable to create 1st event batch") + return nil, softassert.UnexpectedInternalErr( + s.shardContext.GetLogger(), + "unable to create 1st event batch", + nil, + ) } return &creationParams{ @@ -665,7 +670,8 @@ func (s *Starter) handleUseExistingWorkflowOnConflictOptions( requestID, completionCallbacks, links, - nil, + "", // identity + nil, // priority ) return api.UpdateWorkflowWithoutWorkflowTask, err }, @@ -742,15 +748,6 @@ func (s *Starter) generateResponse( }, nil } - if err := api.ProcessOutgoingSearchAttributes( - shardCtx.GetSearchAttributesProvider(), - shardCtx.GetSearchAttributesMapperProvider(), - historyEvents, - s.namespace.Name(), - s.visibilityManager); err != nil { - return nil, err - } - clock, err := shardCtx.NewVectorClock() if err != nil { return nil, err diff --git a/service/history/api/updateactivityoptions/api_test.go b/service/history/api/updateactivityoptions/api_test.go index 0480abcd6f1..d0b7ceb64c3 100644 --- a/service/history/api/updateactivityoptions/api_test.go +++ b/service/history/api/updateactivityoptions/api_test.go @@ -171,35 +171,35 @@ func TestApplyActivityOptionsErrors(t *testing.T) { var err error err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"retry_policy.maximum_interval"}})) - assert.ErrorContains(t, err, "RetryPolicy is not provided") + require.ErrorContains(t, err, "RetryPolicy is not provided") err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"retry_policy.maximum_attempts"}})) - assert.ErrorContains(t, err, "RetryPolicy is not provided") + require.ErrorContains(t, err, "RetryPolicy is not provided") err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"retry_policy.backoff_coefficient"}})) - assert.ErrorContains(t, err, "RetryPolicy is not provided") + require.ErrorContains(t, err, "RetryPolicy is not provided") err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"retry_policy.initial_interval"}})) - assert.ErrorContains(t, err, "RetryPolicy is not provided") + require.ErrorContains(t, err, "RetryPolicy is not provided") err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"taskQueue.name"}})) - assert.ErrorContains(t, err, "TaskQueue is not provided") + require.ErrorContains(t, err, "TaskQueue is not provided") err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"priority.priority_key"}})) - assert.ErrorContains(t, err, "Priority is not provided") + require.ErrorContains(t, err, "Priority is not provided") err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"priority.fairness_key"}})) - assert.ErrorContains(t, err, "Priority is not provided") + require.ErrorContains(t, err, "Priority is not provided") err = mergeActivityOptions(&activitypb.ActivityOptions{}, &activitypb.ActivityOptions{}, util.ParseFieldMask(&fieldmaskpb.FieldMask{Paths: []string{"priority.fairness_weight"}})) - assert.ErrorContains(t, err, "Priority is not provided") + require.ErrorContains(t, err, "Priority is not provided") } @@ -260,8 +260,8 @@ func TestApplyActivityOptionsReset(t *testing.T) { assert.Nil(t, options.HeartbeatTimeout) assert.Equal(t, int32(10), options.Priority.PriorityKey) - assert.Equal(t, "", options.Priority.FairnessKey) - assert.Equal(t, float32(0), options.Priority.FairnessWeight) + assert.Empty(t, options.Priority.FairnessKey) + assert.Zero(t, options.Priority.FairnessWeight) assert.Nil(t, options.RetryPolicy.InitialInterval) assert.Nil(t, options.RetryPolicy.MaximumInterval) diff --git a/service/history/api/updateworkflowoptions/api.go b/service/history/api/updateworkflowoptions/api.go index b10c5a05e28..e3afe3c20d7 100644 --- a/service/history/api/updateworkflowoptions/api.go +++ b/service/history/api/updateworkflowoptions/api.go @@ -7,7 +7,6 @@ import ( "go.temporal.io/api/serviceerror" workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/server/api/historyservice/v1" - "go.temporal.io/server/common" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/util" @@ -106,14 +105,8 @@ func MergeAndApply( if mergedOpts.GetVersioningOverride() == nil { unsetOverride = true } - if _, err = ms.AddWorkflowExecutionOptionsUpdatedEvent( - mergedOpts.GetVersioningOverride(), - unsetOverride, - "", - nil, - nil, - mergedOpts.GetPriority(), - ); err != nil { + _, err = ms.AddWorkflowExecutionOptionsUpdatedEvent(mergedOpts.GetVersioningOverride(), unsetOverride, "", nil, nil, identity, mergedOpts.GetPriority()) + if err != nil { return nil, hasChanges, err } return mergedOpts, hasChanges, nil @@ -122,10 +115,16 @@ func MergeAndApply( func getOptionsFromMutableState(ms historyi.MutableState) *workflowpb.WorkflowExecutionOptions { opts := &workflowpb.WorkflowExecutionOptions{} if versioningInfo := ms.GetExecutionInfo().GetVersioningInfo(); versioningInfo != nil { - opts.VersioningOverride = common.CloneProto(versioningInfo.GetVersioningOverride()) + override, ok := proto.Clone(versioningInfo.GetVersioningOverride()).(*workflowpb.VersioningOverride) + if !ok { + return nil + } + opts.VersioningOverride = override } if priority := ms.GetExecutionInfo().GetPriority(); priority != nil { - opts.Priority = common.CloneProto(priority) + if cloned, ok := proto.Clone(priority).(*commonpb.Priority); ok { + opts.Priority = cloned + } } return opts } @@ -140,6 +139,23 @@ func mergeWorkflowExecutionOptions( return nil, err } updateFields := util.ParseFieldMask(updateMask) + if _, ok := updateFields["versioningOverride"]; ok { + mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() + } + + if _, ok := updateFields["versioningOverride.deployment"]; ok { + if _, ok := updateFields["versioningOverride.behavior"]; !ok { + return nil, serviceerror.NewInvalidArgument("versioning_override fields must be updated together") + } + mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() + } + + if _, ok := updateFields["versioningOverride.behavior"]; ok { + if _, ok := updateFields["versioningOverride.deployment"]; !ok { + return nil, serviceerror.NewInvalidArgument("versioning_override fields must be updated together") + } + mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() + } // ==== Priority @@ -168,25 +184,5 @@ func mergeWorkflowExecutionOptions( mergeInto.Priority.FairnessWeight = mergeFrom.Priority.GetFairnessWeight() } - // ==== VersioningOverride - - if _, ok := updateFields["versioningOverride"]; ok { - mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() - } - - if _, ok := updateFields["versioningOverride.deployment"]; ok { - if _, ok := updateFields["versioningOverride.behavior"]; !ok { - return nil, serviceerror.NewInvalidArgument("versioning_override fields must be updated together") - } - mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() - } - - if _, ok := updateFields["versioningOverride.behavior"]; ok { - if _, ok := updateFields["versioningOverride.deployment"]; !ok { - return nil, serviceerror.NewInvalidArgument("versioning_override fields must be updated together") - } - mergeInto.VersioningOverride = mergeFrom.GetVersioningOverride() - } - return mergeInto, nil } diff --git a/service/history/api/updateworkflowoptions/api_test.go b/service/history/api/updateworkflowoptions/api_test.go index b3d74a2dd77..51882ee556e 100644 --- a/service/history/api/updateworkflowoptions/api_test.go +++ b/service/history/api/updateworkflowoptions/api_test.go @@ -14,7 +14,7 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/historyservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" - chasmworkflow "go.temporal.io/server/chasm/lib/workflow" + "go.temporal.io/server/chasm" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/cluster/clustertest" "go.temporal.io/server/common/locks" @@ -125,147 +125,6 @@ func TestMergeOptions_FooMask(t *testing.T) { assert.Error(t, err) } -func TestMergeOptions_Priority(t *testing.T) { - updateMask := &fieldmaskpb.FieldMask{Paths: []string{"priority"}} - - // Merge priority into empty options - merged, err := mergeWorkflowExecutionOptions( - emptyOptions, - &workflowpb.WorkflowExecutionOptions{ - Priority: &commonpb.Priority{PriorityKey: 10}, - }, updateMask) - assert.NoError(t, err) - assert.Equal(t, int32(10), merged.Priority.PriorityKey) - - // Update priority with all fields set - merged, err = mergeWorkflowExecutionOptions( - merged, - &workflowpb.WorkflowExecutionOptions{ - Priority: &commonpb.Priority{ - PriorityKey: 20, - FairnessKey: "key1", - FairnessWeight: 3.5, - }, - }, updateMask) - assert.NoError(t, err) - assert.Equal(t, int32(20), merged.Priority.PriorityKey) - assert.Equal(t, "key1", merged.Priority.FairnessKey) - assert.Equal(t, float32(3.5), merged.Priority.FairnessWeight) - - // Replace with partial priority and clear unspecified fields - merged, err = mergeWorkflowExecutionOptions( - merged, - &workflowpb.WorkflowExecutionOptions{ - Priority: &commonpb.Priority{PriorityKey: 30}, - }, updateMask) - assert.NoError(t, err) - assert.Equal(t, int32(30), merged.Priority.PriorityKey) - assert.Equal(t, "", merged.Priority.FairnessKey) // cleared - assert.Equal(t, float32(0), merged.Priority.FairnessWeight) // cleared -} - -func TestMergeOptions_Priority_FairnessKey(t *testing.T) { - updateMask := &fieldmaskpb.FieldMask{Paths: []string{"priority.fairness_key"}} - - init := &workflowpb.WorkflowExecutionOptions{ - Priority: &commonpb.Priority{ - PriorityKey: 50, - FairnessWeight: 2.5, - }, - } - - merged, err := mergeWorkflowExecutionOptions( - init, - &workflowpb.WorkflowExecutionOptions{ - Priority: &commonpb.Priority{FairnessKey: "key1"}, - }, updateMask) - assert.NoError(t, err) - assert.Equal(t, "key1", merged.Priority.FairnessKey) - assert.Equal(t, int32(50), merged.Priority.PriorityKey) // preserved - assert.Equal(t, float32(2.5), merged.Priority.FairnessWeight) // preserved -} - -func TestMergeOptions_Priority_FairnessWeight(t *testing.T) { - updateMask := &fieldmaskpb.FieldMask{Paths: []string{"priority.fairness_weight"}} - - init := &workflowpb.WorkflowExecutionOptions{ - Priority: &commonpb.Priority{ - PriorityKey: 75, - FairnessKey: "existingKey", - }, - } - - merged, err := mergeWorkflowExecutionOptions( - init, - &workflowpb.WorkflowExecutionOptions{ - Priority: &commonpb.Priority{FairnessWeight: 5}, - }, updateMask) - assert.NoError(t, err) - assert.Equal(t, float32(5), merged.Priority.FairnessWeight) - assert.Equal(t, int32(75), merged.Priority.PriorityKey) // preserved - assert.Equal(t, "existingKey", merged.Priority.FairnessKey) // preserved -} - -func TestMergeOptions_Priority_PriorityKey(t *testing.T) { - priorityKeyMask := &fieldmaskpb.FieldMask{Paths: []string{"priority.priority_key"}} - - init := &workflowpb.WorkflowExecutionOptions{ - Priority: &commonpb.Priority{ - FairnessKey: "key1", - FairnessWeight: 3.0, - }, - } - - merged, err := mergeWorkflowExecutionOptions( - init, - &workflowpb.WorkflowExecutionOptions{ - Priority: &commonpb.Priority{PriorityKey: 100}, - }, priorityKeyMask) - assert.NoError(t, err) - assert.Equal(t, int32(100), merged.Priority.PriorityKey) - assert.Equal(t, "key1", merged.Priority.FairnessKey) // preserved - assert.Equal(t, float32(3.0), merged.Priority.FairnessWeight) // preserved -} - -func TestMergeOptions_Priority_ClearPriority(t *testing.T) { - updateMask := &fieldmaskpb.FieldMask{Paths: []string{"priority"}} - - init := &workflowpb.WorkflowExecutionOptions{ - Priority: &commonpb.Priority{ - PriorityKey: 100, - FairnessKey: "key", - FairnessWeight: 5, - }, - } - - merged, err := mergeWorkflowExecutionOptions( - init, - &workflowpb.WorkflowExecutionOptions{ - Priority: nil, - }, updateMask) - assert.NoError(t, err) - assert.Nil(t, merged.Priority) -} - -func TestMergeOptions_Priority_MultipleFields(t *testing.T) { - allFieldsMask := &fieldmaskpb.FieldMask{ - Paths: []string{"priority.priority_key", "priority.fairness_key", "priority.fairness_weight"}} - - merged, err := mergeWorkflowExecutionOptions( - emptyOptions, - &workflowpb.WorkflowExecutionOptions{ - Priority: &commonpb.Priority{ - PriorityKey: 99, - FairnessKey: "testKey", - FairnessWeight: 7.5, - }, - }, allFieldsMask) - assert.NoError(t, err) - assert.Equal(t, int32(99), merged.Priority.PriorityKey) - assert.Equal(t, "testKey", merged.Priority.FairnessKey) - assert.Equal(t, float32(7.5), merged.Priority.FairnessWeight) -} - type ( // updateWorkflowOptionsSuite contains tests for the UpdateWorkflowOptions API. updateWorkflowOptionsSuite struct { @@ -319,7 +178,7 @@ func (s *updateWorkflowOptionsSuite) SetupTest() { s.currentContext.EXPECT().LoadMutableState(gomock.Any(), s.shardContext).Return(s.currentMutableState, nil) s.workflowCache = wcache.NewMockCache(s.controller) - s.workflowCache.EXPECT().GetOrCreateChasmEntity(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), chasmworkflow.Archetype, locks.PriorityHigh). + s.workflowCache.EXPECT().GetOrCreateChasmExecution(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), chasm.WorkflowArchetypeID, locks.PriorityHigh). Return(s.currentContext, wcache.NoopReleaseFn, nil) s.workflowConsistencyChecker = api.NewWorkflowConsistencyChecker( @@ -333,6 +192,7 @@ func (s *updateWorkflowOptionsSuite) TearDownTest() { } func (s *updateWorkflowOptionsSuite) TestInvoke_Success() { + expectedOverrideOptions := &workflowpb.WorkflowExecutionOptions{ VersioningOverride: &workflowpb.VersioningOverride{ Behavior: enumspb.VERSIONING_BEHAVIOR_PINNED, @@ -340,14 +200,7 @@ func (s *updateWorkflowOptionsSuite) TestInvoke_Success() { }, } s.currentMutableState.EXPECT().IsWorkflowExecutionRunning().Return(true) - s.currentMutableState.EXPECT().AddWorkflowExecutionOptionsUpdatedEvent( - expectedOverrideOptions.VersioningOverride, - false, - "", - nil, - nil, - nil, - ).Return(&historypb.HistoryEvent{}, nil) + s.currentMutableState.EXPECT().AddWorkflowExecutionOptionsUpdatedEvent(expectedOverrideOptions.VersioningOverride, false, "", nil, nil, "", expectedOverrideOptions.Priority).Return(&historypb.HistoryEvent{}, nil) s.currentContext.EXPECT().UpdateWorkflowExecutionAsActive(gomock.Any(), s.shardContext).Return(nil) updateReq := &historyservice.UpdateWorkflowExecutionOptionsRequest{ diff --git a/service/history/historybuilder/event_factory.go b/service/history/historybuilder/event_factory.go index e7576add50b..1e5d59c322c 100644 --- a/service/history/historybuilder/event_factory.go +++ b/service/history/historybuilder/event_factory.go @@ -384,6 +384,7 @@ func (b *EventFactory) CreateWorkflowExecutionOptionsUpdatedEvent( attachRequestID string, attachCompletionCallbacks []*commonpb.Callback, links []*commonpb.Link, + identity string, priority *commonpb.Priority, ) *historypb.HistoryEvent { event := b.createHistoryEvent(enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED, b.timeSource.Now()) @@ -393,6 +394,7 @@ func (b *EventFactory) CreateWorkflowExecutionOptionsUpdatedEvent( UnsetVersioningOverride: unsetVersioningOverride, AttachedRequestId: attachRequestID, AttachedCompletionCallbacks: attachCompletionCallbacks, + Identity: identity, Priority: priority, }, } @@ -1009,6 +1011,38 @@ func (b *EventFactory) CreateChildWorkflowExecutionTimedOutEvent( return event } +func (b *EventFactory) CreateWorkflowExecutionPausedEvent( + identity string, + reason string, + requestID string, +) *historypb.HistoryEvent { + event := b.createHistoryEvent(enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_PAUSED, b.timeSource.Now()) + event.Attributes = &historypb.HistoryEvent_WorkflowExecutionPausedEventAttributes{ + WorkflowExecutionPausedEventAttributes: &historypb.WorkflowExecutionPausedEventAttributes{ + Identity: identity, + Reason: reason, + RequestId: requestID, + }, + } + return event +} + +func (b *EventFactory) CreateWorkflowExecutionUnpausedEvent( + identity string, + reason string, + requestID string, +) *historypb.HistoryEvent { + event := b.createHistoryEvent(enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UNPAUSED, b.timeSource.Now()) + event.Attributes = &historypb.HistoryEvent_WorkflowExecutionUnpausedEventAttributes{ + WorkflowExecutionUnpausedEventAttributes: &historypb.WorkflowExecutionUnpausedEventAttributes{ + Identity: identity, + Reason: reason, + RequestId: requestID, + }, + } + return event +} + func (b *EventFactory) createHistoryEvent( eventType enumspb.EventType, time time.Time, diff --git a/service/history/historybuilder/history_builder.go b/service/history/historybuilder/history_builder.go index 6935c7244b9..002dadf0116 100644 --- a/service/history/historybuilder/history_builder.go +++ b/service/history/historybuilder/history_builder.go @@ -276,6 +276,30 @@ func (b *HistoryBuilder) AddWorkflowTaskFailedEvent( return event } +func (b *HistoryBuilder) AddWorkflowExecutionPausedEvent( + identity string, + reason string, + requestID string, +) *historypb.HistoryEvent { + event := b.CreateWorkflowExecutionPausedEvent(identity, reason, requestID) + // Mark the event as 'worker may ignore' so that older SDKs can safely ignore it. + event.WorkerMayIgnore = true + event, _ = b.add(event) + return event +} + +func (b *HistoryBuilder) AddWorkflowExecutionUnpausedEvent( + identity string, + reason string, + requestID string, +) *historypb.HistoryEvent { + event := b.CreateWorkflowExecutionUnpausedEvent(identity, reason, requestID) + // Mark the event as 'worker may ignore' so that older SDKs can safely ignore it. + event.WorkerMayIgnore = true + event, _ = b.add(event) + return event +} + func (b *HistoryBuilder) AddActivityTaskScheduledEvent( workflowTaskCompletedEventID int64, command *commandpb.ScheduleActivityTaskCommandAttributes, @@ -428,6 +452,7 @@ func (b *HistoryBuilder) AddWorkflowExecutionOptionsUpdatedEvent( attachRequestID string, attachCompletionCallbacks []*commonpb.Callback, links []*commonpb.Link, + identity string, priority *commonpb.Priority, ) *historypb.HistoryEvent { event := b.EventFactory.CreateWorkflowExecutionOptionsUpdatedEvent( @@ -436,6 +461,7 @@ func (b *HistoryBuilder) AddWorkflowExecutionOptionsUpdatedEvent( attachRequestID, attachCompletionCallbacks, links, + identity, priority, ) event, _ = b.EventStore.add(event) diff --git a/service/history/historybuilder/history_builder_categorization_test.go b/service/history/historybuilder/history_builder_categorization_test.go index be945937182..3ae5619bf9f 100644 --- a/service/history/historybuilder/history_builder_categorization_test.go +++ b/service/history/historybuilder/history_builder_categorization_test.go @@ -208,7 +208,7 @@ func TestHistoryBuilder_FlushBufferToCurrentBatch(t *testing.T) { t.Errorf("expected 1 event in memBufferBatch got %d", len(hb.memBufferBatch)) } // add another event to memBufferBatch - hb.AddWorkflowExecutionOptionsUpdatedEvent(nil, false, "request-id-1", nil, nil, nil) + hb.AddWorkflowExecutionOptionsUpdatedEvent(nil, false, "request-id-1", nil, nil, "", nil) if len(hb.memBufferBatch) != 2 { t.Errorf("expected 2 event in memBufferBatch got %d", len(hb.memBufferBatch)) } diff --git a/service/history/interfaces/mutable_state.go b/service/history/interfaces/mutable_state.go index c81b82d191c..aee318e279a 100644 --- a/service/history/interfaces/mutable_state.go +++ b/service/history/interfaces/mutable_state.go @@ -23,6 +23,8 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" taskqueuespb "go.temporal.io/server/api/taskqueue/v1" workflowspb "go.temporal.io/server/api/workflow/v1" + "go.temporal.io/server/chasm" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" @@ -116,6 +118,7 @@ type ( attachRequestID string, attachCompletionCallbacks []*commonpb.Callback, links []*commonpb.Link, + identity string, priority *commonpb.Priority, ) (*historypb.HistoryEvent, error) AddWorkflowExecutionUpdateAcceptedEvent(protocolInstanceID string, acceptedRequestMessageId string, acceptedRequestSequencingEventId int64, acceptedRequest *updatepb.Request) (*historypb.HistoryEvent, error) @@ -213,6 +216,10 @@ type ( TaskQueueScheduleToStartTimeout(name string) (*taskqueuepb.TaskQueue, *durationpb.Duration) IsWorkflowExecutionRunning() bool + AddWorkflowExecutionPausedEvent(identity string, reason string, requestID string) (*historypb.HistoryEvent, error) + ApplyWorkflowExecutionPausedEvent(event *historypb.HistoryEvent) error + AddWorkflowExecutionUnpausedEvent(identity string, reason string, requestID string) (*historypb.HistoryEvent, error) + ApplyWorkflowExecutionUnpausedEvent(event *historypb.HistoryEvent) error IsResourceDuplicated(resourceDedupKey definition.DeduplicationID) bool IsWorkflowPendingOnWorkflowTaskBackoff() bool UpdateDuplicatedResource(resourceDedupKey definition.DeduplicationID) @@ -287,6 +294,8 @@ type ( AddTasks(tasks ...tasks.Task) PopTasks() map[tasks.Category][]tasks.Task + DeleteCHASMPureTasks(maxScheduledTime time.Time) + SetUpdateCondition(int64, int64) GetUpdateCondition() (int64, int64) @@ -294,6 +303,11 @@ type ( CheckSpeculativeWorkflowTaskTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) bool RemoveSpeculativeWorkflowTaskTimeoutTask() + SetWorkflowTaskScheduleToStartTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) + SetWorkflowTaskStartToCloseTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) + GetWorkflowTaskScheduleToStartTimeoutTask() *tasks.WorkflowTaskTimeoutTask + GetWorkflowTaskStartToCloseTimeoutTask() *tasks.WorkflowTaskTimeoutTask + IsDirty() bool IsTransitionHistoryEnabled() bool // StartTransaction sets up the mutable state for transacting. @@ -317,6 +331,9 @@ type ( IsWorkflow() bool ChasmTree() ChasmTree + ChasmEnabled() bool + ChasmWorkflowComponent(ctx context.Context) (*chasmworkflow.Workflow, chasm.MutableContext, error) + ChasmWorkflowComponentReadOnly(ctx context.Context) (*chasmworkflow.Workflow, chasm.Context, error) // NextTransitionCount returns the next state transition count from the state transition history. // If state transition history is empty (e.g. when disabled or fresh mutable state), returns 0. @@ -351,7 +368,9 @@ type ( // activities. // If there is a pending workflow task that is not started yet, it'll be rescheduled after // transition start. - StartDeploymentTransition(deployment *deploymentpb.Deployment) error + StartDeploymentTransition(deployment *deploymentpb.Deployment, revisionNumber int64) error + GetVersioningRevisionNumber() int64 + SetVersioningRevisionNumber(revisionNumber int64) AddReapplyCandidateEvent(event *historypb.HistoryEvent) GetReapplyCandidateEvents() []*historypb.HistoryEvent diff --git a/service/history/interfaces/mutable_state_mock.go b/service/history/interfaces/mutable_state_mock.go index fd29ce5d197..ef4f52ec631 100644 --- a/service/history/interfaces/mutable_state_mock.go +++ b/service/history/interfaces/mutable_state_mock.go @@ -31,6 +31,8 @@ import ( persistence "go.temporal.io/server/api/persistence/v1" taskqueue0 "go.temporal.io/server/api/taskqueue/v1" workflow0 "go.temporal.io/server/api/workflow/v1" + chasm "go.temporal.io/server/chasm" + workflow1 "go.temporal.io/server/chasm/lib/workflow" definition "go.temporal.io/server/common/definition" namespace "go.temporal.io/server/common/namespace" persistence0 "go.temporal.io/server/common/persistence" @@ -637,18 +639,33 @@ func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionCanceledEvent(arg0, } // AddWorkflowExecutionOptionsUpdatedEvent mocks base method. -func (m *MockMutableState) AddWorkflowExecutionOptionsUpdatedEvent(versioningOverride *workflow.VersioningOverride, unsetVersioningOverride bool, attachRequestID string, attachCompletionCallbacks []*common.Callback, links []*common.Link, priority *common.Priority) (*history.HistoryEvent, error) { +func (m *MockMutableState) AddWorkflowExecutionOptionsUpdatedEvent(versioningOverride *workflow.VersioningOverride, unsetVersioningOverride bool, attachRequestID string, attachCompletionCallbacks []*common.Callback, links []*common.Link, identity string, priority *common.Priority) (*history.HistoryEvent, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "AddWorkflowExecutionOptionsUpdatedEvent", versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, priority) + ret := m.ctrl.Call(m, "AddWorkflowExecutionOptionsUpdatedEvent", versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, identity, priority) ret0, _ := ret[0].(*history.HistoryEvent) ret1, _ := ret[1].(error) return ret0, ret1 } // AddWorkflowExecutionOptionsUpdatedEvent indicates an expected call of AddWorkflowExecutionOptionsUpdatedEvent. -func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionOptionsUpdatedEvent(versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, priority any) *gomock.Call { +func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionOptionsUpdatedEvent(versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, identity, priority any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowExecutionOptionsUpdatedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowExecutionOptionsUpdatedEvent), versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, priority) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowExecutionOptionsUpdatedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowExecutionOptionsUpdatedEvent), versioningOverride, unsetVersioningOverride, attachRequestID, attachCompletionCallbacks, links, identity, priority) +} + +// AddWorkflowExecutionPausedEvent mocks base method. +func (m *MockMutableState) AddWorkflowExecutionPausedEvent(identity, reason, requestID string) (*history.HistoryEvent, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddWorkflowExecutionPausedEvent", identity, reason, requestID) + ret0, _ := ret[0].(*history.HistoryEvent) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddWorkflowExecutionPausedEvent indicates an expected call of AddWorkflowExecutionPausedEvent. +func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionPausedEvent(identity, reason, requestID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowExecutionPausedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowExecutionPausedEvent), identity, reason, requestID) } // AddWorkflowExecutionSignaled mocks base method. @@ -726,6 +743,21 @@ func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionTerminatedEvent(firs return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowExecutionTerminatedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowExecutionTerminatedEvent), firstEventID, reason, details, identity, deleteAfterTerminate, links) } +// AddWorkflowExecutionUnpausedEvent mocks base method. +func (m *MockMutableState) AddWorkflowExecutionUnpausedEvent(identity, reason, requestID string) (*history.HistoryEvent, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddWorkflowExecutionUnpausedEvent", identity, reason, requestID) + ret0, _ := ret[0].(*history.HistoryEvent) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddWorkflowExecutionUnpausedEvent indicates an expected call of AddWorkflowExecutionUnpausedEvent. +func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionUnpausedEvent(identity, reason, requestID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowExecutionUnpausedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowExecutionUnpausedEvent), identity, reason, requestID) +} + // AddWorkflowExecutionUpdateAcceptedEvent mocks base method. func (m *MockMutableState) AddWorkflowExecutionUpdateAcceptedEvent(protocolInstanceID, acceptedRequestMessageId string, acceptedRequestSequencingEventId int64, acceptedRequest *update.Request) (*history.HistoryEvent, error) { m.ctrl.T.Helper() @@ -1386,6 +1418,20 @@ func (mr *MockMutableStateMockRecorder) ApplyWorkflowExecutionOptionsUpdatedEven return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyWorkflowExecutionOptionsUpdatedEvent", reflect.TypeOf((*MockMutableState)(nil).ApplyWorkflowExecutionOptionsUpdatedEvent), event) } +// ApplyWorkflowExecutionPausedEvent mocks base method. +func (m *MockMutableState) ApplyWorkflowExecutionPausedEvent(event *history.HistoryEvent) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ApplyWorkflowExecutionPausedEvent", event) + ret0, _ := ret[0].(error) + return ret0 +} + +// ApplyWorkflowExecutionPausedEvent indicates an expected call of ApplyWorkflowExecutionPausedEvent. +func (mr *MockMutableStateMockRecorder) ApplyWorkflowExecutionPausedEvent(event any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyWorkflowExecutionPausedEvent", reflect.TypeOf((*MockMutableState)(nil).ApplyWorkflowExecutionPausedEvent), event) +} + // ApplyWorkflowExecutionSignaled mocks base method. func (m *MockMutableState) ApplyWorkflowExecutionSignaled(arg0 *history.HistoryEvent) error { m.ctrl.T.Helper() @@ -1442,6 +1488,20 @@ func (mr *MockMutableStateMockRecorder) ApplyWorkflowExecutionTimedoutEvent(arg0 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyWorkflowExecutionTimedoutEvent", reflect.TypeOf((*MockMutableState)(nil).ApplyWorkflowExecutionTimedoutEvent), arg0, arg1) } +// ApplyWorkflowExecutionUnpausedEvent mocks base method. +func (m *MockMutableState) ApplyWorkflowExecutionUnpausedEvent(event *history.HistoryEvent) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ApplyWorkflowExecutionUnpausedEvent", event) + ret0, _ := ret[0].(error) + return ret0 +} + +// ApplyWorkflowExecutionUnpausedEvent indicates an expected call of ApplyWorkflowExecutionUnpausedEvent. +func (mr *MockMutableStateMockRecorder) ApplyWorkflowExecutionUnpausedEvent(event any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyWorkflowExecutionUnpausedEvent", reflect.TypeOf((*MockMutableState)(nil).ApplyWorkflowExecutionUnpausedEvent), event) +} + // ApplyWorkflowExecutionUpdateAcceptedEvent mocks base method. func (m *MockMutableState) ApplyWorkflowExecutionUpdateAcceptedEvent(arg0 *history.HistoryEvent) error { m.ctrl.T.Helper() @@ -1580,6 +1640,20 @@ func (mr *MockMutableStateMockRecorder) AttachRequestID(requestID, eventType, ev return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AttachRequestID", reflect.TypeOf((*MockMutableState)(nil).AttachRequestID), requestID, eventType, eventID) } +// ChasmEnabled mocks base method. +func (m *MockMutableState) ChasmEnabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ChasmEnabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// ChasmEnabled indicates an expected call of ChasmEnabled. +func (mr *MockMutableStateMockRecorder) ChasmEnabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChasmEnabled", reflect.TypeOf((*MockMutableState)(nil).ChasmEnabled)) +} + // ChasmTree mocks base method. func (m *MockMutableState) ChasmTree() ChasmTree { m.ctrl.T.Helper() @@ -1594,6 +1668,38 @@ func (mr *MockMutableStateMockRecorder) ChasmTree() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChasmTree", reflect.TypeOf((*MockMutableState)(nil).ChasmTree)) } +// ChasmWorkflowComponent mocks base method. +func (m *MockMutableState) ChasmWorkflowComponent(ctx context.Context) (*workflow1.Workflow, chasm.MutableContext, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ChasmWorkflowComponent", ctx) + ret0, _ := ret[0].(*workflow1.Workflow) + ret1, _ := ret[1].(chasm.MutableContext) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ChasmWorkflowComponent indicates an expected call of ChasmWorkflowComponent. +func (mr *MockMutableStateMockRecorder) ChasmWorkflowComponent(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChasmWorkflowComponent", reflect.TypeOf((*MockMutableState)(nil).ChasmWorkflowComponent), ctx) +} + +// ChasmWorkflowComponentReadOnly mocks base method. +func (m *MockMutableState) ChasmWorkflowComponentReadOnly(ctx context.Context) (*workflow1.Workflow, chasm.Context, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ChasmWorkflowComponentReadOnly", ctx) + ret0, _ := ret[0].(*workflow1.Workflow) + ret1, _ := ret[1].(chasm.Context) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ChasmWorkflowComponentReadOnly indicates an expected call of ChasmWorkflowComponentReadOnly. +func (mr *MockMutableStateMockRecorder) ChasmWorkflowComponentReadOnly(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChasmWorkflowComponentReadOnly", reflect.TypeOf((*MockMutableState)(nil).ChasmWorkflowComponentReadOnly), ctx) +} + // CheckResettable mocks base method. func (m *MockMutableState) CheckResettable() error { m.ctrl.T.Helper() @@ -1736,6 +1842,18 @@ func (mr *MockMutableStateMockRecorder) CurrentVersionedTransition() *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CurrentVersionedTransition", reflect.TypeOf((*MockMutableState)(nil).CurrentVersionedTransition)) } +// DeleteCHASMPureTasks mocks base method. +func (m *MockMutableState) DeleteCHASMPureTasks(maxScheduledTime time.Time) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "DeleteCHASMPureTasks", maxScheduledTime) +} + +// DeleteCHASMPureTasks indicates an expected call of DeleteCHASMPureTasks. +func (mr *MockMutableStateMockRecorder) DeleteCHASMPureTasks(maxScheduledTime any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCHASMPureTasks", reflect.TypeOf((*MockMutableState)(nil).DeleteCHASMPureTasks), maxScheduledTime) +} + // DeleteSignalRequested mocks base method. func (m *MockMutableState) DeleteSignalRequested(requestID string) { m.ctrl.T.Helper() @@ -2572,6 +2690,20 @@ func (mr *MockMutableStateMockRecorder) GetUserTimerInfoByEventID(arg0 any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserTimerInfoByEventID", reflect.TypeOf((*MockMutableState)(nil).GetUserTimerInfoByEventID), arg0) } +// GetVersioningRevisionNumber mocks base method. +func (m *MockMutableState) GetVersioningRevisionNumber() int64 { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVersioningRevisionNumber") + ret0, _ := ret[0].(int64) + return ret0 +} + +// GetVersioningRevisionNumber indicates an expected call of GetVersioningRevisionNumber. +func (mr *MockMutableStateMockRecorder) GetVersioningRevisionNumber() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVersioningRevisionNumber", reflect.TypeOf((*MockMutableState)(nil).GetVersioningRevisionNumber)) +} + // GetWorkflowCloseTime mocks base method. func (m *MockMutableState) GetWorkflowCloseTime(ctx context.Context) (time.Time, error) { m.ctrl.T.Helper() @@ -2645,6 +2777,34 @@ func (mr *MockMutableStateMockRecorder) GetWorkflowTaskByID(scheduledEventID any return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkflowTaskByID", reflect.TypeOf((*MockMutableState)(nil).GetWorkflowTaskByID), scheduledEventID) } +// GetWorkflowTaskScheduleToStartTimeoutTask mocks base method. +func (m *MockMutableState) GetWorkflowTaskScheduleToStartTimeoutTask() *tasks.WorkflowTaskTimeoutTask { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetWorkflowTaskScheduleToStartTimeoutTask") + ret0, _ := ret[0].(*tasks.WorkflowTaskTimeoutTask) + return ret0 +} + +// GetWorkflowTaskScheduleToStartTimeoutTask indicates an expected call of GetWorkflowTaskScheduleToStartTimeoutTask. +func (mr *MockMutableStateMockRecorder) GetWorkflowTaskScheduleToStartTimeoutTask() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkflowTaskScheduleToStartTimeoutTask", reflect.TypeOf((*MockMutableState)(nil).GetWorkflowTaskScheduleToStartTimeoutTask)) +} + +// GetWorkflowTaskStartToCloseTimeoutTask mocks base method. +func (m *MockMutableState) GetWorkflowTaskStartToCloseTimeoutTask() *tasks.WorkflowTaskTimeoutTask { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetWorkflowTaskStartToCloseTimeoutTask") + ret0, _ := ret[0].(*tasks.WorkflowTaskTimeoutTask) + return ret0 +} + +// GetWorkflowTaskStartToCloseTimeoutTask indicates an expected call of GetWorkflowTaskStartToCloseTimeoutTask. +func (mr *MockMutableStateMockRecorder) GetWorkflowTaskStartToCloseTimeoutTask() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkflowTaskStartToCloseTimeoutTask", reflect.TypeOf((*MockMutableState)(nil).GetWorkflowTaskStartToCloseTimeoutTask)) +} + // GetWorkflowType mocks base method. func (m *MockMutableState) GetWorkflowType() *common.WorkflowType { m.ctrl.T.Helper() @@ -3246,6 +3406,42 @@ func (mr *MockMutableStateMockRecorder) SetUpdateCondition(arg0, arg1 any) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUpdateCondition", reflect.TypeOf((*MockMutableState)(nil).SetUpdateCondition), arg0, arg1) } +// SetVersioningRevisionNumber mocks base method. +func (m *MockMutableState) SetVersioningRevisionNumber(revisionNumber int64) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetVersioningRevisionNumber", revisionNumber) +} + +// SetVersioningRevisionNumber indicates an expected call of SetVersioningRevisionNumber. +func (mr *MockMutableStateMockRecorder) SetVersioningRevisionNumber(revisionNumber any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetVersioningRevisionNumber", reflect.TypeOf((*MockMutableState)(nil).SetVersioningRevisionNumber), revisionNumber) +} + +// SetWorkflowTaskScheduleToStartTimeoutTask mocks base method. +func (m *MockMutableState) SetWorkflowTaskScheduleToStartTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetWorkflowTaskScheduleToStartTimeoutTask", task) +} + +// SetWorkflowTaskScheduleToStartTimeoutTask indicates an expected call of SetWorkflowTaskScheduleToStartTimeoutTask. +func (mr *MockMutableStateMockRecorder) SetWorkflowTaskScheduleToStartTimeoutTask(task any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetWorkflowTaskScheduleToStartTimeoutTask", reflect.TypeOf((*MockMutableState)(nil).SetWorkflowTaskScheduleToStartTimeoutTask), task) +} + +// SetWorkflowTaskStartToCloseTimeoutTask mocks base method. +func (m *MockMutableState) SetWorkflowTaskStartToCloseTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetWorkflowTaskStartToCloseTimeoutTask", task) +} + +// SetWorkflowTaskStartToCloseTimeoutTask indicates an expected call of SetWorkflowTaskStartToCloseTimeoutTask. +func (mr *MockMutableStateMockRecorder) SetWorkflowTaskStartToCloseTimeoutTask(task any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetWorkflowTaskStartToCloseTimeoutTask", reflect.TypeOf((*MockMutableState)(nil).SetWorkflowTaskStartToCloseTimeoutTask), task) +} + // ShouldResetActivityTimerTaskMask mocks base method. func (m *MockMutableState) ShouldResetActivityTimerTaskMask(current, incoming *persistence.ActivityInfo) bool { m.ctrl.T.Helper() @@ -3261,17 +3457,17 @@ func (mr *MockMutableStateMockRecorder) ShouldResetActivityTimerTaskMask(current } // StartDeploymentTransition mocks base method. -func (m *MockMutableState) StartDeploymentTransition(arg0 *deployment.Deployment) error { +func (m *MockMutableState) StartDeploymentTransition(arg0 *deployment.Deployment, revisionNumber int64) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StartDeploymentTransition", arg0) + ret := m.ctrl.Call(m, "StartDeploymentTransition", arg0, revisionNumber) ret0, _ := ret[0].(error) return ret0 } // StartDeploymentTransition indicates an expected call of StartDeploymentTransition. -func (mr *MockMutableStateMockRecorder) StartDeploymentTransition(arg0 any) *gomock.Call { +func (mr *MockMutableStateMockRecorder) StartDeploymentTransition(arg0, revisionNumber any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartDeploymentTransition", reflect.TypeOf((*MockMutableState)(nil).StartDeploymentTransition), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartDeploymentTransition", reflect.TypeOf((*MockMutableState)(nil).StartDeploymentTransition), arg0, revisionNumber) } // StartTransaction mocks base method. diff --git a/service/history/ndc/events_reapplier_test.go b/service/history/ndc/events_reapplier_test.go index b8928d44a32..c2e013fb40c 100644 --- a/service/history/ndc/events_reapplier_test.go +++ b/service/history/ndc/events_reapplier_test.go @@ -111,6 +111,7 @@ func (s *nDCEventReapplicationSuite) TestReapplyEvents_AppliedEvent_WorkflowExec attr.GetAttachedRequestId(), attr.GetAttachedCompletionCallbacks(), event.Links, + attr.GetIdentity(), attr.GetPriority(), ).Return(event, nil) msCurrent.EXPECT().HSM().Return(s.hsmNode).AnyTimes() diff --git a/service/history/ndc/workflow_resetter.go b/service/history/ndc/workflow_resetter.go index c7537b5b328..b52b0cdf6f4 100644 --- a/service/history/ndc/workflow_resetter.go +++ b/service/history/ndc/workflow_resetter.go @@ -953,7 +953,8 @@ func reapplyEvents( requestID, callbacks, event.Links, - attr.Priority, + attr.GetIdentity(), + attr.GetPriority(), ); err != nil { return reappliedEvents, err } @@ -1167,7 +1168,8 @@ func (r *workflowResetterImpl) performPostResetOperations(ctx context.Context, r for _, operation := range postResetOperations { switch op := operation.GetVariant().(type) { case *workflowpb.PostResetOperation_UpdateWorkflowOptions_: - _, _, err := updateworkflowoptions.MergeAndApply(resetMS, op.UpdateWorkflowOptions.GetWorkflowExecutionOptions(), op.UpdateWorkflowOptions.GetUpdateMask()) + // TODO(carlydf): Put the reset requester in the event so that with state-based replication this code will run on the passive side. + _, _, err := updateworkflowoptions.MergeAndApply(resetMS, op.UpdateWorkflowOptions.GetWorkflowExecutionOptions(), op.UpdateWorkflowOptions.GetUpdateMask(), "") if err != nil { return err } diff --git a/service/history/ndc/workflow_resetter_test.go b/service/history/ndc/workflow_resetter_test.go index 74987e5f803..4fec71a62e4 100644 --- a/service/history/ndc/workflow_resetter_test.go +++ b/service/history/ndc/workflow_resetter_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/pborman/uuid" + "github.com/google/uuid" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" commonpb "go.temporal.io/api/common/v1" @@ -21,7 +21,7 @@ import ( historyspb "go.temporal.io/server/api/history/v1" "go.temporal.io/server/api/historyservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" - chasmworkflow "go.temporal.io/server/chasm/lib/workflow" + "go.temporal.io/server/chasm" "go.temporal.io/server/common" "go.temporal.io/server/common/collection" "go.temporal.io/server/common/definition" @@ -113,9 +113,9 @@ func (s *workflowResetterSuite) SetupTest() { s.namespaceID = tests.NamespaceID s.workflowID = "some random workflow ID" - s.baseRunID = uuid.New() - s.currentRunID = uuid.New() - s.resetRunID = uuid.New() + s.baseRunID = uuid.NewString() + s.currentRunID = uuid.NewString() + s.resetRunID = uuid.NewString() } func (s *workflowResetterSuite) TearDownTest() { @@ -196,6 +196,7 @@ func (s *workflowResetterSuite) TestPersistToDB_CurrentTerminated() { s.mockTransaction.EXPECT().UpdateWorkflowExecution( gomock.Any(), persistence.UpdateWorkflowModeUpdateCurrent, + chasm.WorkflowArchetypeID, int64(0), currentMutation, currentEventsSeq, @@ -260,6 +261,7 @@ func (s *workflowResetterSuite) TestPersistToDB_CurrentNotTerminated() { s.mockTransaction.EXPECT().UpdateWorkflowExecution( gomock.Any(), persistence.UpdateWorkflowModeUpdateCurrent, + chasm.WorkflowArchetypeID, int64(0), currentMutation, currentEventsSeq, @@ -283,7 +285,7 @@ func (s *workflowResetterSuite) TestReplayResetWorkflow() { baseRebuildLastEventVersion := int64(12) resetBranchToken := []byte("some random reset branch token") - resetRequestID := uuid.New() + resetRequestID := uuid.NewString() resetHistorySize := int64(4411) resetMutableState := historyi.NewMockMutableState(s.controller) @@ -333,10 +335,10 @@ func (s *workflowResetterSuite) TestReplayResetWorkflow() { } func (s *workflowResetterSuite) TestFailWorkflowTask_NoWorkflowTask() { - baseRunID := uuid.New() + baseRunID := uuid.NewString() baseRebuildLastEventID := int64(1234) baseRebuildLastEventVersion := int64(5678) - resetRunID := uuid.New() + resetRunID := uuid.NewString() resetReason := "some random reset reason" mutableState := historyi.NewMockMutableState(s.controller) @@ -354,17 +356,17 @@ func (s *workflowResetterSuite) TestFailWorkflowTask_NoWorkflowTask() { } func (s *workflowResetterSuite) TestFailWorkflowTask_WorkflowTaskScheduled() { - baseRunID := uuid.New() + baseRunID := uuid.NewString() baseRebuildLastEventID := int64(1234) baseRebuildLastEventVersion := int64(5678) - resetRunID := uuid.New() + resetRunID := uuid.NewString() resetReason := "some random reset reason" mutableState := historyi.NewMockMutableState(s.controller) workflowTaskSchedule := &historyi.WorkflowTaskInfo{ ScheduledEventID: baseRebuildLastEventID - 12, StartedEventID: common.EmptyEventID, - RequestID: uuid.New(), + RequestID: uuid.NewString(), TaskQueue: &taskqueuepb.TaskQueue{ Name: "random task queue name", Kind: enumspb.TASK_QUEUE_KIND_NORMAL, @@ -411,17 +413,17 @@ func (s *workflowResetterSuite) TestFailWorkflowTask_WorkflowTaskScheduled() { } func (s *workflowResetterSuite) TestFailWorkflowTask_WorkflowTaskStarted() { - baseRunID := uuid.New() + baseRunID := uuid.NewString() baseRebuildLastEventID := int64(1234) baseRebuildLastEventVersion := int64(5678) - resetRunID := uuid.New() + resetRunID := uuid.NewString() resetReason := "some random reset reason" mutableState := historyi.NewMockMutableState(s.controller) workflowTask := &historyi.WorkflowTaskInfo{ ScheduledEventID: baseRebuildLastEventID - 12, StartedEventID: baseRebuildLastEventID - 10, - RequestID: uuid.New(), + RequestID: uuid.NewString(), TaskQueue: &taskqueuepb.TaskQueue{ Name: "random task queue name", Kind: enumspb.TASK_QUEUE_KIND_NORMAL, @@ -627,7 +629,7 @@ func (s *workflowResetterSuite) TestReapplyContinueAsNewWorkflowEvents_WithConti baseNextEventID := int64(456) baseBranchToken := []byte("some random base branch token") - newRunID := uuid.New() + newRunID := uuid.NewString() newFirstEventID := common.FirstEventID newNextEventID := int64(6) newBranchToken := []byte("some random new branch token") @@ -712,10 +714,10 @@ func (s *workflowResetterSuite) TestReapplyContinueAsNewWorkflowEvents_WithConti resetContext.EXPECT().Lock(gomock.Any(), locks.PriorityHigh).Return(nil) resetContext.EXPECT().Unlock() resetContext.EXPECT().IsDirty().Return(false).AnyTimes() - resetContext.EXPECT().SetArchetype(chasmworkflow.Archetype).Times(1) resetMutableState := historyi.NewMockMutableState(s.controller) resetContextCacheKey := wcache.Key{ WorkflowKey: definition.NewWorkflowKey(s.namespaceID.String(), s.workflowID, newRunID), + ArchetypeID: chasm.WorkflowArchetypeID, ShardUUID: s.mockShard.GetOwner(), } resetContext.EXPECT().LoadMutableState(gomock.Any(), s.mockShard).Return(resetMutableState, nil) @@ -756,7 +758,7 @@ func (s *workflowResetterSuite) TestReapplyWorkflowEvents() { nextEventID := int64(6) branchToken := []byte("some random branch token") - newRunID := uuid.New() + newRunID := uuid.NewString() event1 := &historypb.HistoryEvent{ EventId: 1, EventType: enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, @@ -826,8 +828,8 @@ func (s *workflowResetterSuite) TestReapplyEvents_WithPendingChildren() { testInitiatedEventID := int64(123) testChildWFType := &commonpb.WorkflowType{Name: "TEST-CHILD-WF-TYPE"} testChildWFExecution := &commonpb.WorkflowExecution{ - WorkflowId: uuid.New(), - RunId: uuid.New(), + WorkflowId: uuid.NewString(), + RunId: uuid.NewString(), } testStartEventHeader := &commonpb.Header{} @@ -1024,6 +1026,7 @@ func (s *workflowResetterSuite) TestReapplyEvents_WithNoPendingChildren() { } func (s *workflowResetterSuite) TestReapplyEvents() { + event1 := &historypb.HistoryEvent{ EventId: 101, EventType: enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED, @@ -1189,6 +1192,7 @@ func (s *workflowResetterSuite) TestReapplyEvents() { attr.GetAttachedRequestId(), attr.GetAttachedCompletionCallbacks(), event.Links, + attr.GetIdentity(), attr.GetPriority(), ).Return(&historypb.HistoryEvent{}, nil) case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED: @@ -1467,7 +1471,7 @@ func (s *workflowResetterSuite) TestWorkflowRestartAfterExecutionTimeout() { resetReason := "some random reset reason" resetBranchToken := []byte("some random reset branch token") - resetRequestID := uuid.New() + resetRequestID := uuid.NewString() resetHistorySize := int64(4411) resetMutableState := historyi.NewMockMutableState(s.controller) executionInfos := make(map[int64]*persistencespb.ChildExecutionInfo) @@ -1475,7 +1479,7 @@ func (s *workflowResetterSuite) TestWorkflowRestartAfterExecutionTimeout() { workflowTaskSchedule := &historyi.WorkflowTaskInfo{ ScheduledEventID: baseRebuildLastEventID - 12, StartedEventID: common.EmptyEventID, - RequestID: uuid.New(), + RequestID: uuid.NewString(), TaskQueue: &taskqueuepb.TaskQueue{ Name: "random task queue name", Kind: enumspb.TASK_QUEUE_KIND_NORMAL, diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 09125fbc2bf..a9dd46e472c 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -11,8 +11,8 @@ import ( "strings" "time" + "github.com/google/uuid" "github.com/nexus-rpc/sdk-go/nexus" - "github.com/pborman/uuid" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" deploymentpb "go.temporal.io/api/deployment/v1" @@ -48,13 +48,16 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/payload" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/transitionhistory" "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/searchattribute" + "go.temporal.io/server/common/searchattribute/sadefs" serviceerrors "go.temporal.io/server/common/serviceerror" + "go.temporal.io/server/common/softassert" "go.temporal.io/server/common/util" "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/components/callbacks" @@ -213,8 +216,33 @@ type ( InsertTasks map[tasks.Category][]tasks.Task + // BestEffortDeleteTasks holds keys of history tasks to be deleted after a successful + // persistence update. This deletion is done on best effort basis. Persistence layer can ignore it without + // any errors. + BestEffortDeleteTasks map[tasks.Category][]tasks.Key + speculativeWorkflowTaskTimeoutTask *tasks.WorkflowTaskTimeoutTask + // In-memory storage for workflow task timeout tasks. These are set when timeout tasks are + // generated and used to delete them when the workflow task completes. Not persisted to storage. + wftScheduleToStartTimeoutTask *tasks.WorkflowTaskTimeoutTask + wftStartToCloseTimeoutTask *tasks.WorkflowTaskTimeoutTask + + // In-memory storage for CHASM pure tasks. These are set when CHASM pure tasks are generated and used to + // delete them when then are no longer needed. (i.e. when the task's scheduled time is after that of the + // earliest valid CHASM pure task's). + // + // Those pure tasks are mostly reverse ordered by their scheduled time (the VisibilityTimestamp field). + // Since a physical pure task is only generated when there's no other pure task with an earlier scheduled time, + // simply appending new pure tasks to the end of the slice maintains the order. + // + // NOTE: shard context may move those tasks' scheduled time to the future if they are earlier than the timer queue's + // max read level (otherwise those tasks won't be loaded), which may potentially break the reverse order. + // That is fine, however, as in the worst case we just delete fewer tasks than we could have, but we will never delete + // tasks that are still needed (all tasks deleted are those having an earlier scheduled time than what's needed). + // Task deletion is just a best-effort optimization after all, so not complicating the logic to account for that here. + chasmPureTasks []*tasks.ChasmTaskPure + // Do not rely on this, this is only updated on // Load() and closeTransactionXXX methods. So when // a transaction is in progress, this value will be @@ -299,6 +327,7 @@ func NewMutableState( namespaceEntry: namespaceEntry, appliedEvents: make(map[string]struct{}), InsertTasks: make(map[tasks.Category][]tasks.Task), + BestEffortDeleteTasks: make(map[tasks.Category][]tasks.Key), transitionHistoryEnabled: shard.GetConfig().EnableTransitionHistory(), visibilityUpdated: false, executionStateUpdated: false, @@ -503,7 +532,7 @@ func NewMutableStateFromDB( if shard.GetConfig().EnableChasm() { var err error - mutableState.chasmTree, err = chasm.NewTree( + mutableState.chasmTree, err = chasm.NewTreeFromDB( dbRecord.ChasmNodes, shard.ChasmRegistry(), shard.GetTimeSource(), @@ -591,8 +620,7 @@ func (ms *MutableStateImpl) mustInitHSM() { } func (ms *MutableStateImpl) IsWorkflow() bool { - archetype := ms.chasmTree.Archetype() - return archetype == chasmworkflow.Archetype || archetype == "" + return ms.chasmTree.ArchetypeID() == chasm.WorkflowArchetypeID } func (ms *MutableStateImpl) HSM() *hsm.Node { @@ -603,13 +631,77 @@ func (ms *MutableStateImpl) ChasmTree() historyi.ChasmTree { return ms.chasmTree } +// ChasmEnabled returns true if the mutable state has a real chasm tree. +// The chasmTree is initialized with a noopChasmTree which is then overwritten with an actual chasm tree if chasm is +// enabled when the mutable state is created. Once the EnableChasm dynamic config is removed and the tree is always +// initialized, this helper can be removed. +func (ms *MutableStateImpl) ChasmEnabled() bool { + _, isNoop := ms.chasmTree.(*noopChasmTree) + return !isNoop +} + +// chasmCallbacksEnabled returns true if CHASM callbacks are enabled for this workflow. +func (ms *MutableStateImpl) chasmCallbacksEnabled() bool { + if !ms.ChasmEnabled() { + return false + } + + // Check the callback library's EnableCallbacks config via history config + return ms.shard.GetConfig().EnableCHASMCallbacks(ms.GetNamespaceEntry().Name().String()) +} + +// ChasmWorkflowComponent gets the root workflow component from the CHASM tree. +// Returns the workflow component (which is *chasmworkflow.Workflow) and the CHASM mutable context. +// This method is for write operations. Callers can type assert to *chasmworkflow.Workflow if needed. +func (ms *MutableStateImpl) ChasmWorkflowComponent(ctx context.Context) (*chasmworkflow.Workflow, chasm.MutableContext, error) { + chasmCtx := chasm.NewMutableContext(ctx, ms.chasmTree.(*chasm.Node)) + rootComponent, err := ms.chasmTree.ComponentByPath(chasmCtx, nil) + if err != nil { + return nil, nil, err + } + wf, ok := rootComponent.(*chasmworkflow.Workflow) + if !ok { + return nil, nil, serviceerror.NewInternalf("expected workflow component, but got %T", rootComponent) + } + return wf, chasmCtx, nil +} + +func (ms *MutableStateImpl) ensureChasmWorkflowComponent(ctx context.Context) { + // Initialize chasm tree once for new workflows. + // Using context.Background() because this is done outside an actual request context and the + // chasmworkflow.NewWorkflow does not actually use it currently. + root, ok := ms.chasmTree.(*chasm.Node) + softassert.That(ms.logger, ok, "chasmTree cast failed") + + if root.ArchetypeID() == chasm.UnspecifiedArchetypeID { + mutableContext := chasm.NewMutableContext(ctx, root) + root.SetRootComponent(chasmworkflow.NewWorkflow(mutableContext, chasm.NewMSPointer(ms))) + } +} + +// ChasmWorkflowComponentReadOnly gets the root workflow component from the CHASM tree. +// Returns both the workflow component and a read-only CHASM context. +// This method is for read-only operations. +func (ms *MutableStateImpl) ChasmWorkflowComponentReadOnly(ctx context.Context) (*chasmworkflow.Workflow, chasm.Context, error) { + chasmCtx := chasm.NewContext(ctx, ms.chasmTree.(*chasm.Node)) + rootComponent, err := ms.chasmTree.ComponentByPath(chasmCtx, nil) + if err != nil { + return nil, nil, err + } + wf, ok := rootComponent.(*chasmworkflow.Workflow) + if !ok { + return nil, nil, serviceerror.NewInternalf("expected workflow component, but got %T", rootComponent) + } + return wf, chasmCtx, nil +} + // GetNexusCompletion converts a workflow completion event into a [nexus.OperationCompletion]. // Completions may be sent to arbitrary third parties, we intentionally do not include any termination reasons, and // expose only failure messages. func (ms *MutableStateImpl) GetNexusCompletion( ctx context.Context, requestID string, -) (nexus.OperationCompletion, error) { +) (nexusrpc.OperationCompletion, error) { ce, err := ms.GetCompletionEvent(ctx) if err != nil { return nil, err @@ -653,9 +745,10 @@ func (ms *MutableStateImpl) GetNexusCompletion( // Nexus does not support it. p = payloads[0] } - completion, err := nexus.NewOperationCompletionSuccessful(p, nexus.OperationCompletionSuccessfulOptions{ + completion, err := nexusrpc.NewOperationCompletionSuccessful(p, nexusrpc.OperationCompletionSuccessfulOptions{ Serializer: commonnexus.PayloadSerializer, StartTime: ms.executionState.GetStartTime().AsTime(), + CloseTime: ce.GetEventTime().AsTime(), Links: []nexus.Link{startLink}, }) if err != nil { @@ -667,10 +760,11 @@ func (ms *MutableStateImpl) GetNexusCompletion( if err != nil { return nil, err } - return nexus.NewOperationCompletionUnsuccessful( + return nexusrpc.NewOperationCompletionUnsuccessful( &nexus.OperationError{State: nexus.OperationStateFailed, Cause: &nexus.FailureError{Failure: f}}, - nexus.OperationCompletionUnsuccessfulOptions{ + nexusrpc.OperationCompletionUnsuccessfulOptions{ StartTime: ms.executionState.GetStartTime().AsTime(), + CloseTime: ce.GetEventTime().AsTime(), Links: []nexus.Link{startLink}, }) case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_CANCELED: @@ -685,13 +779,14 @@ func (ms *MutableStateImpl) GetNexusCompletion( if err != nil { return nil, err } - return nexus.NewOperationCompletionUnsuccessful( + return nexusrpc.NewOperationCompletionUnsuccessful( &nexus.OperationError{ State: nexus.OperationStateCanceled, Cause: &nexus.FailureError{Failure: f}, }, - nexus.OperationCompletionUnsuccessfulOptions{ + nexusrpc.OperationCompletionUnsuccessfulOptions{ StartTime: ms.executionState.GetStartTime().AsTime(), + CloseTime: ce.GetEventTime().AsTime(), Links: []nexus.Link{startLink}, }) case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_TERMINATED: @@ -704,10 +799,11 @@ func (ms *MutableStateImpl) GetNexusCompletion( if err != nil { return nil, err } - return nexus.NewOperationCompletionUnsuccessful( + return nexusrpc.NewOperationCompletionUnsuccessful( &nexus.OperationError{State: nexus.OperationStateFailed, Cause: &nexus.FailureError{Failure: f}}, - nexus.OperationCompletionUnsuccessfulOptions{ + nexusrpc.OperationCompletionUnsuccessfulOptions{ StartTime: ms.executionState.GetStartTime().AsTime(), + CloseTime: ce.GetEventTime().AsTime(), Links: []nexus.Link{startLink}, }) case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_TIMED_OUT: @@ -723,13 +819,14 @@ func (ms *MutableStateImpl) GetNexusCompletion( if err != nil { return nil, err } - return nexus.NewOperationCompletionUnsuccessful( + return nexusrpc.NewOperationCompletionUnsuccessful( &nexus.OperationError{ State: nexus.OperationStateFailed, Cause: &nexus.FailureError{Failure: f}, }, - nexus.OperationCompletionUnsuccessfulOptions{ + nexusrpc.OperationCompletionUnsuccessfulOptions{ StartTime: ms.executionState.GetStartTime().AsTime(), + CloseTime: ce.GetEventTime().AsTime(), Links: []nexus.Link{startLink}, }) } @@ -806,8 +903,21 @@ func (ms *MutableStateImpl) SetHistoryTree( ) error { // NOTE: Unfortunately execution timeout and run timeout are not yet initialized into ms.executionInfo at this point. // TODO: Consider explicitly initializing mutable state with these timeout parameters instead of passing them in. - workflowKey := ms.GetWorkflowKey() + + archetypeID := ms.ChasmTree().ArchetypeID() + if archetypeID != chasm.WorkflowArchetypeID { + return softassert.UnexpectedInternalErr( + ms.logger, + "Backfilling history not supported for non-workflow archetype", + nil, + tag.ArchetypeID(archetypeID), + tag.WorkflowNamespaceID(workflowKey.NamespaceID), + tag.WorkflowID(workflowKey.WorkflowID), + tag.WorkflowRunID(workflowKey.RunID), + ) + } + var retentionDuration *durationpb.Duration if duration := ms.namespaceEntry.Retention(); duration > 0 { retentionDuration = durationpb.New(duration) @@ -2102,14 +2212,25 @@ func (ms *MutableStateImpl) IsTransientWorkflowTask() bool { func (ms *MutableStateImpl) ClearTransientWorkflowTask() error { if !ms.HasStartedWorkflowTask() { - return serviceerror.NewInternal("cannot clear transient workflow task when task is missing") + return softassert.UnexpectedInternalErr( + ms.logger, + "cannot clear transient workflow task when task is missing", + nil, + ) } if !ms.IsTransientWorkflowTask() { - return serviceerror.NewInternal("cannot clear transient workflow task when task is not transient") + return softassert.UnexpectedInternalErr( + ms.logger, + "cannot clear transient workflow task when task is not transient", + nil, + ) } - // this is transient workflow task if ms.HasBufferedEvents() { - return serviceerror.NewInternal("cannot clear transient workflow task when there are buffered events") + return softassert.UnexpectedInternalErr( + ms.logger, + "cannot clear transient workflow task when there are buffered events", + nil, + ) } // no buffered event emptyWorkflowTaskInfo := &historyi.WorkflowTaskInfo{ @@ -2273,6 +2394,7 @@ func (ms *MutableStateImpl) HasRequestID( } func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( + ctx context.Context, parentExecutionInfo *workflowspb.ParentExecutionInfo, execution *commonpb.WorkflowExecution, previousExecutionState historyi.MutableState, @@ -2313,7 +2435,7 @@ func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( // for other fields as well. runTimeout := command.GetWorkflowRunTimeout() - completionCallbacks, err := getCompletionCallbacksAsProtoSlice(previousExecutionState) + completionCallbacks, err := getCompletionCallbacksAsProtoSlice(ctx, previousExecutionState) if err != nil { return nil, err } @@ -2330,9 +2452,9 @@ func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( inheritedPinnedVersion = worker_versioning.ExternalWorkerDeploymentVersionFromDeployment(previousExecutionState.GetEffectiveDeployment()) newTQ := command.GetTaskQueue().GetName() if newTQ != previousExecutionInfo.GetTaskQueue() { - newTQInPinnedVersion, err = IsWFTaskQueueInVersionDetector(context.Background(), ms.GetNamespaceEntry().ID().String(), newTQ, inheritedPinnedVersion) + newTQInPinnedVersion, err = IsWFTaskQueueInVersionDetector(ctx, ms.GetNamespaceEntry().ID().String(), newTQ, inheritedPinnedVersion) if err != nil { - return nil, errors.New(fmt.Sprintf("error determining child task queue presence in inherited version: %s", err.Error())) + return nil, fmt.Errorf("error determining child task queue presence in inherited version: %w", err) } if !newTQInPinnedVersion { inheritedPinnedVersion = nil @@ -2350,8 +2472,35 @@ func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( } } + // New run initiated by ContinueAsNew of an AUTO_UPGRADE workflow execution will inherit the previous run's + // deployment version and revision number iff the new run's Task Queue belongs to source deployment version. + var sourceDeploymentVersion *deploymentpb.WorkerDeploymentVersion + var sourceDeploymentRevisionNumber int64 + if previousExecutionState.GetEffectiveVersioningBehavior() == enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE { + sourceDeploymentVersion = worker_versioning.ExternalWorkerDeploymentVersionFromDeployment(previousExecutionState.GetEffectiveDeployment()) + sourceDeploymentRevisionNumber = previousExecutionState.GetVersioningRevisionNumber() + + newTQ := command.GetTaskQueue().GetName() + if newTQ != previousExecutionInfo.GetTaskQueue() { + // Cross-TQ CAN: check if new TQ is in parent's deployment + TQInSourceDeploymentVersion, err := IsWFTaskQueueInVersionDetector( + ctx, + ms.GetNamespaceEntry().ID().String(), + newTQ, + sourceDeploymentVersion, + ) + if err != nil { + return nil, fmt.Errorf("error determining CAN task queue presence in auto upgrade deployment: %w", err) + } + if !TQInSourceDeploymentVersion { + sourceDeploymentVersion = nil + sourceDeploymentRevisionNumber = 0 + } + } + } + createRequest := &workflowservice.StartWorkflowExecutionRequest{ - RequestId: uuid.New(), + RequestId: uuid.NewString(), Namespace: ms.namespaceEntry.Name().String(), WorkflowId: execution.WorkflowId, TaskQueue: tq, @@ -2414,6 +2563,14 @@ func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( req.WorkflowExecutionExpirationTime = timestamppb.New(workflowTimeoutTime) } + // Add InheritedAutoUpgradeInfo if InheritedPinnedVersion is not set and source deployment version and revision number are set. + if sourceDeploymentVersion != nil && sourceDeploymentRevisionNumber != 0 && inheritedPinnedVersion == nil { + req.InheritedAutoUpgradeInfo = &deploymentpb.InheritedAutoUpgradeInfo{ + SourceDeploymentVersion: sourceDeploymentVersion, + SourceDeploymentRevisionNumber: sourceDeploymentRevisionNumber, + } + } + event, err := ms.AddWorkflowExecutionStartedEventWithOptions( execution, req, @@ -2744,19 +2901,163 @@ func (ms *MutableStateImpl) ApplyWorkflowExecutionStartedEvent( ms.approximateSize += ms.executionInfo.Size() ms.approximateSize += ms.executionState.Size() + // Populate the versioningInfo if the inheritedAutoUpgradeInfo is present. + if event.GetInheritedAutoUpgradeInfo() != nil { + ms.SetVersioningRevisionNumber(event.GetInheritedAutoUpgradeInfo().GetSourceDeploymentRevisionNumber()) + // TODO (Shivam): Remove this once you make SetDeploymentVersion and SetVersioningBehavior methods with nil checks + if ms.executionInfo.VersioningInfo == nil { + ms.executionInfo.VersioningInfo = &workflowpb.WorkflowExecutionVersioningInfo{} + } + ms.executionInfo.VersioningInfo.DeploymentVersion = event.GetInheritedAutoUpgradeInfo().GetSourceDeploymentVersion() + // Assume AutoUpgrade behavior for the first workflow task. + ms.executionInfo.VersioningInfo.Behavior = enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE + } + ms.writeEventToCache(startEvent) return nil } +func (ms *MutableStateImpl) AddWorkflowExecutionPausedEvent( + identity string, + reason string, + requestID string, +) (*historypb.HistoryEvent, error) { + opTag := tag.WorkflowActionWorkflowPaused + if err := ms.checkMutability(opTag); err != nil { + return nil, err + } + event := ms.hBuilder.AddWorkflowExecutionPausedEvent(identity, reason, requestID) + if err := ms.ApplyWorkflowExecutionPausedEvent(event); err != nil { + return nil, err + } + return event, nil +} + +// ApplyWorkflowExecutionPausedEvent applies the paused event to the mutable state. It updates the workflow execution status to paused and sets the pause info. +func (ms *MutableStateImpl) ApplyWorkflowExecutionPausedEvent(event *historypb.HistoryEvent) error { + // Update workflow status. + if _, err := ms.UpdateWorkflowStateStatus(ms.executionState.GetState(), enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED); err != nil { + return err + } + // Set pause info in mutable state. + ms.executionInfo.PauseInfo = &persistencespb.WorkflowPauseInfo{ + PauseTime: timestamppb.New(event.GetEventTime().AsTime()), + Identity: event.GetWorkflowExecutionPausedEventAttributes().GetIdentity(), + Reason: event.GetWorkflowExecutionPausedEventAttributes().GetReason(), + RequestId: event.GetWorkflowExecutionPausedEventAttributes().GetRequestId(), + } + + // Update approximate size of the mutable state. This will be decreased when the pause info is removed (when the workflow is unpaused) + ms.approximateSize += ms.executionInfo.PauseInfo.Size() + + // Invalidate all the pending activities. Do not mark individual activities as paused. + for _, ai := range ms.GetPendingActivityInfos() { + if err := ms.UpdateActivity(ai.ScheduledEventId, func(activityInfo *persistencespb.ActivityInfo, _ historyi.MutableState) error { + activityInfo.Stamp = activityInfo.Stamp + 1 + return nil + }); err != nil { + return err + } + } + + // Invalidate pending workflow task by incrementing the persisted stamp. + // This ensures subsequent task dispatch detects the change. + if ms.HasPendingWorkflowTask() { + ms.executionInfo.WorkflowTaskStamp += 1 + ms.workflowTaskManager.UpdateWorkflowTask(ms.GetPendingWorkflowTask()) + } + return nil +} + +func (ms *MutableStateImpl) AddWorkflowExecutionUnpausedEvent( + identity string, + reason string, + requestID string, +) (*historypb.HistoryEvent, error) { + opTag := tag.WorkflowActionWorkflowUnpaused + if err := ms.checkMutability(opTag); err != nil { + return nil, err + } + event := ms.hBuilder.AddWorkflowExecutionUnpausedEvent(identity, reason, requestID) + if err := ms.ApplyWorkflowExecutionUnpausedEvent(event); err != nil { + return nil, err + } + return event, nil +} + +// ApplyWorkflowExecutionUnpausedEvent applies the unpaused event to the mutable state. It updates the workflow execution status to running and clears the pause info. +func (ms *MutableStateImpl) ApplyWorkflowExecutionUnpausedEvent(event *historypb.HistoryEvent) error { + // Update workflow status. + if _, err := ms.UpdateWorkflowStateStatus(ms.executionState.GetState(), enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING); err != nil { + return err + } + + // save pauseInfoSize before clearing so that we can adjust approximate size later before returning success + pauseInfoSize := 0 + if ms.executionInfo.PauseInfo != nil { + pauseInfoSize = ms.GetExecutionInfo().GetPauseInfo().Size() + // Clear pause info in mutable state. + ms.executionInfo.PauseInfo = nil + } + + // Reschedule any pending activities + // Note: workflow task is scheduled in the unpause API. So no need to schedule it here. + for _, ai := range ms.GetPendingActivityInfos() { + // Bump activity stamp to force replication so that the passive cluster can recreate the activity task. + if err := ms.UpdateActivity(ai.ScheduledEventId, func(activityInfo *persistencespb.ActivityInfo, _ historyi.MutableState) error { + activityInfo.Stamp = activityInfo.Stamp + 1 + return nil + }); err != nil { + return err + } + + // Check activity scheduled time and generate activity retry task if scheduled time is in the future. + if ai.GetScheduledTime().AsTime().After(ms.timeSource.Now().UTC()) { + if err := ms.taskGenerator.GenerateActivityRetryTasks(ai); err != nil { + return err + } + } else { + // Generate activity task to resend the activity to matching immediately. + if err := ms.taskGenerator.GenerateActivityTasks(ai.ScheduledEventId); err != nil { + return err + } + } + } + + // Update approximate size of the mutable state. + ms.approximateSize -= pauseInfoSize + return nil +} + func (ms *MutableStateImpl) addCompletionCallbacks( event *historypb.HistoryEvent, requestID string, completionCallbacks []*commonpb.Callback, +) error { + if len(completionCallbacks) == 0 { + return nil + } + if ms.chasmCallbacksEnabled() { + // Initialize chasm tree once for new workflows. + // Using context.Background() because this is done outside an actual request context and the + // chasmworkflow.NewWorkflow does not actually use it currently. + ms.ensureChasmWorkflowComponent(context.Background()) + return ms.addCompletionCallbacksChasm(event, requestID, completionCallbacks) + } + + return ms.addCompletionCallbacksHsm(event, requestID, completionCallbacks) +} + +// addCompletionCallbacksHsm creates completion callbacks using the HSM implementation. +func (ms *MutableStateImpl) addCompletionCallbacksHsm( + event *historypb.HistoryEvent, + requestID string, + completionCallbacks []*commonpb.Callback, ) error { coll := callbacks.MachineCollection(ms.HSM()) maxCallbacksPerWorkflow := ms.config.MaxCallbacksPerWorkflow(ms.GetNamespaceEntry().Name().String()) if len(completionCallbacks)+coll.Size() > maxCallbacksPerWorkflow { - return serviceerror.NewInvalidArgumentf( + return serviceerror.NewFailedPreconditionf( "cannot attach more than %d callbacks to a workflow (%d callbacks already attached)", maxCallbacksPerWorkflow, coll.Size(), @@ -2774,11 +3075,8 @@ func (ms *MutableStateImpl) addCompletionCallbacks( Header: variant.Nexus.GetHeader(), }, } - case *commonpb.Callback_Internal_: - err := proto.Unmarshal(cb.GetInternal().GetData(), persistenceCB) - if err != nil { - return err - } + default: + return serviceerror.NewInvalidArgumentf("unknown callback variant: %T", cb.Variant) } machine := callbacks.NewCallback(requestID, event.EventTime, callbacks.NewWorkflowClosedTrigger(), persistenceCB) id := "" @@ -2798,6 +3096,21 @@ func (ms *MutableStateImpl) addCompletionCallbacks( return nil } +// addCompletionCallbacksChasm creates completion callbacks using the CHASM implementation. +func (ms *MutableStateImpl) addCompletionCallbacksChasm( + event *historypb.HistoryEvent, + requestID string, + completionCallbacks []*commonpb.Callback, +) error { + wf, ctx, err := ms.ChasmWorkflowComponent(context.Background()) + if err != nil { + return err + } + + maxCallbacksPerWorkflow := ms.config.MaxCHASMCallbacksPerWorkflow(ms.GetNamespaceEntry().Name().String()) + return wf.AddCompletionCallbacks(ctx, event.EventTime, requestID, completionCallbacks, maxCallbacksPerWorkflow) +} + // AddFirstWorkflowTaskScheduled adds the first workflow task scheduled event unless it should be delayed as indicated // by the startEvent's FirstWorkflowTaskBackoff. // Returns the workflow task's scheduled event ID if a task was scheduled, 0 otherwise. @@ -2922,10 +3235,10 @@ func (ms *MutableStateImpl) updateBinaryChecksumSearchAttribute() error { if exeInfo.SearchAttributes == nil { exeInfo.SearchAttributes = make(map[string]*commonpb.Payload, 1) } - if proto.Equal(exeInfo.SearchAttributes[searchattribute.BinaryChecksums], checksumsPayload) { + if proto.Equal(exeInfo.SearchAttributes[sadefs.BinaryChecksums], checksumsPayload) { return nil // unchanged } - ms.updateSearchAttributes(map[string]*commonpb.Payload{searchattribute.BinaryChecksums: checksumsPayload}) + ms.updateSearchAttributes(map[string]*commonpb.Payload{sadefs.BinaryChecksums: checksumsPayload}) return ms.taskGenerator.GenerateUpsertVisibilityTask() } @@ -3025,7 +3338,7 @@ func (ms *MutableStateImpl) ApplyBuildIdRedirect( ms.GetExecutionInfo().BuildIdRedirectCounter = redirectCounter // Re-scheduling pending workflow and activity tasks. - err = ms.reschedulePendingWorkflowTask(false) + err = ms.reschedulePendingWorkflowTask() if err != nil { return err } @@ -3039,6 +3352,16 @@ func (ms *MutableStateImpl) ApplyBuildIdRedirect( // TODO: skip task generation also when activity is in backoff period continue } + + // need to update stamp so the passive side regenerate the task + err := ms.UpdateActivity(ai.ScheduledEventId, func(info *persistencespb.ActivityInfo, state historyi.MutableState) error { + info.Stamp++ + return nil + }) + if err != nil { + return err + } + // we only need to resend the activities to matching, no need to update timer tasks. err = ms.taskGenerator.GenerateActivityTasks(ai.ScheduledEventId) if err != nil { @@ -3093,7 +3416,7 @@ func (ms *MutableStateImpl) loadBuildIds() ([]string, error) { if searchAttributes == nil { return []string{}, nil } - saPayload, found := searchAttributes[searchattribute.BuildIds] + saPayload, found := searchAttributes[sadefs.BuildIds] if !found { return []string{}, nil } @@ -3201,7 +3524,7 @@ func (ms *MutableStateImpl) saveBuildIds(buildIds []string, maxSearchAttributeVa return err } if len(buildIds) == 0 || len(saPayload.GetData()) <= maxSearchAttributeValueSize { - ms.updateSearchAttributes(map[string]*commonpb.Payload{searchattribute.BuildIds: saPayload}) + ms.updateSearchAttributes(map[string]*commonpb.Payload{sadefs.BuildIds: saPayload}) break } if len(buildIds) == 1 { @@ -3226,21 +3549,21 @@ func (ms *MutableStateImpl) saveDeploymentSearchAttributes(deployment, version, return err } if len(deploymentPayload.GetData()) <= maxSearchAttributeValueSize { // we know the string won't really be over, but still check - saPayloads[searchattribute.TemporalWorkerDeployment] = deploymentPayload + saPayloads[sadefs.TemporalWorkerDeployment] = deploymentPayload } versionPayload, err := searchattribute.EncodeValue(version, enumspb.INDEXED_VALUE_TYPE_KEYWORD) if err != nil { return err } if len(versionPayload.GetData()) <= maxSearchAttributeValueSize { // we know the string won't really be over, but still check - saPayloads[searchattribute.TemporalWorkerDeploymentVersion] = versionPayload + saPayloads[sadefs.TemporalWorkerDeploymentVersion] = versionPayload } behaviorPayload, err := searchattribute.EncodeValue(behavior, enumspb.INDEXED_VALUE_TYPE_KEYWORD) if err != nil { return err } if len(behaviorPayload.GetData()) <= maxSearchAttributeValueSize { // we know the string won't really be over, but still check - saPayloads[searchattribute.TemporalWorkflowVersioningBehavior] = behaviorPayload + saPayloads[sadefs.TemporalWorkflowVersioningBehavior] = behaviorPayload } ms.updateSearchAttributes(saPayloads) return nil @@ -3252,15 +3575,15 @@ func (ms *MutableStateImpl) addBuildIdAndDeploymentInfoToSearchAttributesWithNoV if err != nil { return false, err } - existingDeployment, err := ms.loadSearchAttributeString(searchattribute.TemporalWorkerDeployment) + existingDeployment, err := ms.loadSearchAttributeString(sadefs.TemporalWorkerDeployment) if err != nil { return false, err } - existingVersion, err := ms.loadSearchAttributeString(searchattribute.TemporalWorkerDeploymentVersion) + existingVersion, err := ms.loadSearchAttributeString(sadefs.TemporalWorkerDeploymentVersion) if err != nil { return false, err } - existingBehavior, err := ms.loadSearchAttributeString(searchattribute.TemporalWorkflowVersioningBehavior) + existingBehavior, err := ms.loadSearchAttributeString(sadefs.TemporalWorkflowVersioningBehavior) if err != nil { return false, err } @@ -3966,6 +4289,7 @@ func (ms *MutableStateImpl) AddCompletedWorkflowEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( event.GetEventTime().AsTime(), false, + false, // skipCloseTransferTask ); err != nil { return nil, err } @@ -4009,6 +4333,7 @@ func (ms *MutableStateImpl) AddFailWorkflowEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( event.GetEventTime().AsTime(), false, + false, // skipCloseTransferTask ); err != nil { return nil, err } @@ -4056,6 +4381,7 @@ func (ms *MutableStateImpl) AddTimeoutWorkflowEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( event.GetEventTime().AsTime(), false, + false, // skipCloseTransferTask ); err != nil { return nil, err } @@ -4138,6 +4464,7 @@ func (ms *MutableStateImpl) AddWorkflowExecutionCanceledEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( event.GetEventTime().AsTime(), false, + false, // skipCloseTransferTask ); err != nil { return nil, err } @@ -4670,6 +4997,7 @@ func (ms *MutableStateImpl) AddWorkflowExecutionTerminatedEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( event.GetEventTime().AsTime(), deleteAfterTerminate, + false, // skipCloseTransferTask ); err != nil { return nil, err } @@ -4855,6 +5183,7 @@ func (ms *MutableStateImpl) AddWorkflowExecutionOptionsUpdatedEvent( attachRequestID string, attachCompletionCallbacks []*commonpb.Callback, links []*commonpb.Link, + identity string, priority *commonpb.Priority, ) (*historypb.HistoryEvent, error) { if err := ms.checkMutability(tag.WorkflowActionWorkflowOptionsUpdated); err != nil { @@ -4866,6 +5195,7 @@ func (ms *MutableStateImpl) AddWorkflowExecutionOptionsUpdatedEvent( attachRequestID, attachCompletionCallbacks, links, + identity, priority, ) prevEffectiveVersioningBehavior := ms.GetEffectiveVersioningBehavior() @@ -4905,38 +5235,42 @@ func (ms *MutableStateImpl) ApplyWorkflowExecutionOptionsUpdatedEvent(event *his return err } - // Update priority. - if attributes.GetPriority() != nil { - ms.executionInfo.Priority = attributes.GetPriority() - requestReschedulePendingWorkflowTask = true - } - // Update attached request ID. if attributes.GetAttachedRequestId() != "" { ms.AttachRequestID(attributes.GetAttachedRequestId(), event.EventType, event.EventId) } // Update completion callbacks. - if len(attributes.GetAttachedCompletionCallbacks()) > 0 { - if err := ms.addCompletionCallbacks( - event, - attributes.GetAttachedRequestId(), - attributes.GetAttachedCompletionCallbacks(), - ); err != nil { - return err + if err := ms.addCompletionCallbacks( + event, + attributes.GetAttachedRequestId(), + attributes.GetAttachedCompletionCallbacks(), + ); err != nil { + return err + } + + // Update priority. + if attributes.GetPriority() != nil { + if !proto.Equal(ms.executionInfo.Priority, attributes.GetPriority()) { + requestReschedulePendingWorkflowTask = true } + ms.executionInfo.Priority = attributes.GetPriority() } // Finally, reschedule the pending workflow task if so requested. if requestReschedulePendingWorkflowTask { - return ms.reschedulePendingWorkflowTask(true) + return ms.reschedulePendingWorkflowTask() } return nil } func (ms *MutableStateImpl) updateVersioningOverride( override *workflowpb.VersioningOverride, -) (requestReschedulePendingWorkflowTask bool, err error) { +) (bool, error) { + previousEffectiveDeployment := ms.GetEffectiveDeployment() + previousEffectiveVersioningBehavior := ms.GetEffectiveVersioningBehavior() + var requestReschedulePendingWorkflowTask bool + if override != nil { if ms.GetExecutionInfo().GetVersioningInfo() == nil { ms.GetExecutionInfo().VersioningInfo = &workflowpb.WorkflowExecutionVersioningInfo{} @@ -4998,8 +5332,6 @@ func (ms *MutableStateImpl) updateVersioningOverride( ms.GetExecutionInfo().WorkerDeploymentName = "" } - previousEffectiveDeployment := ms.GetEffectiveDeployment() - previousEffectiveVersioningBehavior := ms.GetEffectiveVersioningBehavior() if !proto.Equal(ms.GetEffectiveDeployment(), previousEffectiveDeployment) || ms.GetEffectiveVersioningBehavior() != previousEffectiveVersioningBehavior { // TODO (carly) part 2: if safe mode, do replay test on new deployment if deployment changed, if fail, revert changes and abort @@ -5042,13 +5374,12 @@ func (ms *MutableStateImpl) updateVersioningOverride( // For v3 versioned workflows (ms.GetEffectiveVersioningBehavior() != UNSPECIFIED), this will update the reachability // search attribute based on the execution_info.deployment and/or override deployment if one exists. limit := ms.config.SearchAttributesSizeOfValueLimit(ms.namespaceEntry.Name().String()) - if err = ms.updateBuildIdsAndDeploymentSearchAttributes(nil, limit); err != nil { - return + if err := ms.updateBuildIdsAndDeploymentSearchAttributes(nil, limit); err != nil { + return requestReschedulePendingWorkflowTask, err } } - err = ms.reschedulePendingActivities() - return + return requestReschedulePendingWorkflowTask, ms.reschedulePendingActivities() } func (ms *MutableStateImpl) ApplyWorkflowExecutionTerminatedEvent( @@ -5136,7 +5467,7 @@ func (ms *MutableStateImpl) AddContinueAsNewEvent( } var err error - newRunID := uuid.New() + newRunID := uuid.NewString() newExecution := commonpb.WorkflowExecution{ WorkflowId: ms.executionInfo.WorkflowId, RunId: newRunID, @@ -5196,6 +5527,7 @@ func (ms *MutableStateImpl) AddContinueAsNewEvent( } if _, err = newMutableState.addWorkflowExecutionStartedEventForContinueAsNew( + ctx, parentInfo, &newExecution, ms, @@ -5226,6 +5558,7 @@ func (ms *MutableStateImpl) AddContinueAsNewEvent( if err := ms.taskGenerator.GenerateWorkflowCloseTasks( continueAsNewEvent.GetEventTime().AsTime(), false, + false, // skipCloseTransferTask ); err != nil { return nil, nil, err } @@ -5719,6 +6052,9 @@ func (ms *MutableStateImpl) RetryActivity( activityInfo.RequestId = "" activityInfo.RetryLastFailure = ms.truncateRetryableActivityFailure(activityFailure) activityInfo.Attempt++ + if ms.config.EnableActivityRetryStampIncrement() { + activityInfo.Stamp++ + } return nil }); err != nil { return enumspb.RETRY_STATE_INTERNAL_SERVER_ERROR, err @@ -5792,12 +6128,14 @@ func (ms *MutableStateImpl) updateActivityInfoForRetries( _ = ms.UpdateActivity(ai.ScheduledEventId, func(activityInfo *persistencespb.ActivityInfo, mutableState historyi.MutableState) error { mutableStateImpl, ok := mutableState.(*MutableStateImpl) if ok { + isActivityRetryStampIncrementEnabled := ms.config.EnableActivityRetryStampIncrement() ai = UpdateActivityInfoForRetries( activityInfo, mutableStateImpl.GetCurrentVersion(), nextAttempt, mutableStateImpl.truncateRetryableActivityFailure(activityFailure), timestamppb.New(nextScheduledTime), + isActivityRetryStampIncrementEnabled, ) } return nil @@ -5880,14 +6218,123 @@ func (ms *MutableStateImpl) updatePauseInfoSearchAttribute() error { exeInfo.SearchAttributes = make(map[string]*commonpb.Payload, 1) } - if proto.Equal(exeInfo.SearchAttributes[searchattribute.TemporalPauseInfo], pauseInfoPayload) { + if proto.Equal(exeInfo.SearchAttributes[sadefs.TemporalPauseInfo], pauseInfoPayload) { return nil // unchanged } - ms.updateSearchAttributes(map[string]*commonpb.Payload{searchattribute.TemporalPauseInfo: pauseInfoPayload}) + ms.updateSearchAttributes(map[string]*commonpb.Payload{sadefs.TemporalPauseInfo: pauseInfoPayload}) + return ms.taskGenerator.GenerateUpsertVisibilityTask() +} + +func (ms *MutableStateImpl) UpdateReportedProblemsSearchAttribute() error { + var reportedProblems []string + switch wftFailure := ms.executionInfo.LastWorkflowTaskFailure.(type) { + case *persistencespb.WorkflowExecutionInfo_LastWorkflowTaskFailureCause: + reportedProblems = []string{ + "category=WorkflowTaskFailed", + fmt.Sprintf("cause=WorkflowTaskFailedCause%s", wftFailure.LastWorkflowTaskFailureCause.String()), + } + case *persistencespb.WorkflowExecutionInfo_LastWorkflowTaskTimedOutType: + reportedProblems = []string{ + "category=WorkflowTaskTimedOut", + fmt.Sprintf("cause=WorkflowTaskTimedOutCause%s", wftFailure.LastWorkflowTaskTimedOutType.String()), + } + } + + reportedProblemsPayload, err := searchattribute.EncodeValue(reportedProblems, enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST) + if err != nil { + return err + } + + exeInfo := ms.executionInfo + if exeInfo.SearchAttributes == nil { + exeInfo.SearchAttributes = make(map[string]*commonpb.Payload, 1) + } + + decodedA, err := searchattribute.DecodeValue(exeInfo.SearchAttributes[sadefs.TemporalReportedProblems], enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, false) + if err != nil { + return err + } + + existingProblems, ok := decodedA.([]string) + if !ok && decodedA != nil { + softassert.Fail(ms.logger, "TemporalReportedProblems payload decoded to unexpected type for logging") + return errors.New("TemporalReportedProblems payload decoded to unexpected type for logging") + } + + if slices.Equal(existingProblems, reportedProblems) { + return nil + } + + // Log the search attribute change + ms.logReportedProblemsChange(existingProblems, reportedProblems) + + ms.updateSearchAttributes(map[string]*commonpb.Payload{sadefs.TemporalReportedProblems: reportedProblemsPayload}) return ms.taskGenerator.GenerateUpsertVisibilityTask() } +func (ms *MutableStateImpl) RemoveReportedProblemsSearchAttribute() error { + if ms.executionInfo.SearchAttributes == nil { + return nil + } + + temporalReportedProblems := ms.executionInfo.SearchAttributes[sadefs.TemporalReportedProblems] + if temporalReportedProblems == nil { + return nil + } + + // Log the removal of the search attribute + ms.logReportedProblemsChange(ms.decodeReportedProblems(temporalReportedProblems), nil) + + ms.executionInfo.LastWorkflowTaskFailure = nil + + // Just remove the search attribute entirely for now + ms.updateSearchAttributes(map[string]*commonpb.Payload{sadefs.TemporalReportedProblems: nil}) + return ms.taskGenerator.GenerateUpsertVisibilityTask() +} + +// logReportedProblemsChange logs changes to the TemporalReportedProblems search attribute +func (ms *MutableStateImpl) logReportedProblemsChange(oldPayload, newPayload []string) { + if oldPayload == nil && newPayload != nil { + // Adding search attribute + ms.logger.Info("TemporalReportedProblems search attribute added", + tag.NewStringsTag("reported-problems", newPayload)) + } else if oldPayload != nil && newPayload == nil { + // Removing search attribute + ms.logger.Info("TemporalReportedProblems search attribute removed", + tag.NewStringsTag("previous-reported-problems", oldPayload)) + } else if oldPayload != nil && newPayload != nil { + // Updating search attribute + ms.logger.Info("TemporalReportedProblems search attribute updated", + tag.NewStringsTag("previous-reported-problems", oldPayload), + tag.NewStringsTag("reported-problems", newPayload)) + } +} + +// decodeReportedProblems safely decodes a keyword list payload to []string +func (ms *MutableStateImpl) decodeReportedProblems(p *commonpb.Payload) []string { + if p == nil { + return nil + } + + decoded, err := searchattribute.DecodeValue(p, enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, false) + if err != nil { + ms.logger.Error("Failed to decode TemporalReportedProblems payload for logging", + tag.Error(err)) + softassert.Fail(ms.logger, "Failed to decode TemporalReportedProblems payload for logging") + return []string{} + } + + problems, ok := decoded.([]string) + if !ok { + ms.logger.Error("TemporalReportedProblems payload decoded to unexpected type for logging") + softassert.Fail(ms.logger, "TemporalReportedProblems payload decoded to unexpected type for logging") + return []string{} + } + + return problems +} + func (ms *MutableStateImpl) truncateRetryableActivityFailure( activityFailure *failurepb.Failure, ) *failurepb.Failure { @@ -5927,6 +6374,22 @@ func (ms *MutableStateImpl) processCloseCallbacks() error { return nil } + // Process CHASM callbacks if CHASM is enabled. Note that we check ChasmEnabled() rather than + // chasmCallbacksEnabled() to ensure that callbacks created when both HSM and CHASM callbacks + // were enabled can still be triggered even if the EnableCHASMCallbacks dynamic config is later + // turned off. Once created in CHASM, callbacks should always be processed as long as CHASM is enabled. + if ms.ChasmEnabled() { + if err := ms.processCloseCallbacksChasm(); err != nil { + return err + } + } + + // Always process HSM callbacks as well (a workflow can have both) + return ms.processCloseCallbacksHsm() +} + +// processCloseCallbacksHsm triggers "WorkflowClosed" callbacks using the HSM implementation. +func (ms *MutableStateImpl) processCloseCallbacksHsm() error { coll := callbacks.MachineCollection(ms.HSM()) for _, node := range coll.List() { cb, err := coll.Data(node.Key.ID) @@ -5947,8 +6410,15 @@ func (ms *MutableStateImpl) processCloseCallbacks() error { return nil } -// TODO mutable state should generate corresponding transfer / timer tasks according to -// updates accumulated, while currently all transfer / timer tasks are managed manually +// processCloseCallbacksChasm triggers "WorkflowClosed" callbacks using the CHASM implementation. +func (ms *MutableStateImpl) processCloseCallbacksChasm() error { + wf, ctx, err := ms.ChasmWorkflowComponent(context.Background()) + if err != nil { + return err + } + + return wf.ProcessCloseCallbacks(ctx) +} func (ms *MutableStateImpl) AddTasks( newTasks ...tasks.Task, @@ -5961,14 +6431,47 @@ func (ms *MutableStateImpl) AddTasks( ms.logger.Info("Dropped long duration scheduled task.", tasks.Tags(task)...) continue } + + if chasmPureTask, ok := task.(*tasks.ChasmTaskPure); ok { + ms.chasmPureTasks = append(ms.chasmPureTasks, chasmPureTask) + maxPureTasks := ms.config.ChasmMaxInMemoryPureTasks() + if len(ms.chasmPureTasks) > maxPureTasks { + // Since tasks are reverse ordered by their scheduled time, tasks in the beginning are those + // - Generated a long time ago + // - Scheduled time is far in the future + // both types of tasks are likely to already be persisted in DB and best-effort deletion won't help, + // so drop them from the in-memory list first. + ms.chasmPureTasks = ms.chasmPureTasks[len(ms.chasmPureTasks)-maxPureTasks:] + } + } + ms.InsertTasks[category] = append(ms.InsertTasks[category], task) } } func (ms *MutableStateImpl) PopTasks() map[tasks.Category][]tasks.Task { - insterTasks := ms.InsertTasks + insertTasks := ms.InsertTasks ms.InsertTasks = make(map[tasks.Category][]tasks.Task) - return insterTasks + return insertTasks +} + +func (ms *MutableStateImpl) DeleteCHASMPureTasks(maxScheduledTime time.Time) { + for lastTaskIdx := len(ms.chasmPureTasks) - 1; lastTaskIdx >= 0; lastTaskIdx-- { + task := ms.chasmPureTasks[lastTaskIdx] + if !task.GetVisibilityTime().Before(maxScheduledTime) { + ms.chasmPureTasks = ms.chasmPureTasks[:lastTaskIdx+1] + return + } + + ms.BestEffortDeleteTasks[tasks.CategoryTimer] = append( + ms.BestEffortDeleteTasks[tasks.CategoryTimer], + task.GetKey(), + ) + } + + // If we reach here, all tasks have visibility time before maxScheduledTime + // and need to be deleted. + ms.chasmPureTasks = ms.chasmPureTasks[:0] } func (ms *MutableStateImpl) SetUpdateCondition( @@ -6009,6 +6512,22 @@ func (ms *MutableStateImpl) RemoveSpeculativeWorkflowTaskTimeoutTask() { } } +func (ms *MutableStateImpl) SetWorkflowTaskScheduleToStartTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) { + ms.wftScheduleToStartTimeoutTask = task +} + +func (ms *MutableStateImpl) SetWorkflowTaskStartToCloseTimeoutTask(task *tasks.WorkflowTaskTimeoutTask) { + ms.wftStartToCloseTimeoutTask = task +} + +func (ms *MutableStateImpl) GetWorkflowTaskScheduleToStartTimeoutTask() *tasks.WorkflowTaskTimeoutTask { + return ms.wftScheduleToStartTimeoutTask +} + +func (ms *MutableStateImpl) GetWorkflowTaskStartToCloseTimeoutTask() *tasks.WorkflowTaskTimeoutTask { + return ms.wftStartToCloseTimeoutTask +} + func (ms *MutableStateImpl) GetWorkflowStateStatus() (enumsspb.WorkflowExecutionState, enumspb.WorkflowExecutionStatus) { return ms.executionState.State, ms.executionState.Status } @@ -6134,7 +6653,8 @@ func (ms *MutableStateImpl) CloseTransactionAsMutation( NewBufferedEvents: result.bufferEvents, ClearBufferedEvents: result.clearBuffer, - Tasks: ms.InsertTasks, + Tasks: ms.InsertTasks, + BestEffortDeleteTasks: ms.BestEffortDeleteTasks, Condition: ms.nextEventIDInDB, DBRecordVersion: ms.dbRecordVersion, @@ -6158,7 +6678,11 @@ func (ms *MutableStateImpl) CloseTransactionAsSnapshot( if len(result.bufferEvents) > 0 { // TODO do we need the functionality to generate snapshot with buffered events? - return nil, nil, serviceerror.NewInternal("cannot generate workflow snapshot with buffered events") + return nil, nil, softassert.UnexpectedInternalErr( + ms.logger, + "cannot generate workflow snapshot with buffered events", + nil, + ) } workflowSnapshot := &persistence.WorkflowSnapshot{ @@ -6767,6 +7291,12 @@ func (ms *MutableStateImpl) closeTransactionPrepareReplicationTasks( replicationTasks = append(replicationTasks, ms.syncActivityToReplicationTask(transactionPolicy)...) replicationTasks = append(replicationTasks, ms.dirtyHSMToReplicationTask(transactionPolicy, eventBatches, clearBufferEvents)...) + archetypeID := ms.ChasmTree().ArchetypeID() + isWorkflow := archetypeID == chasm.WorkflowArchetypeID + if !isWorkflow && len(replicationTasks) != 0 { + return softassert.UnexpectedInternalErr(ms.logger, "chasm execution generated workflow replication tasks", nil) + } + if ms.transitionHistoryEnabled { switch transactionPolicy { case historyi.TransactionPolicyActive: @@ -6809,6 +7339,7 @@ func (ms *MutableStateImpl) closeTransactionPrepareReplicationTasks( syncVersionedTransitionTask := &tasks.SyncVersionedTransitionTask{ WorkflowKey: workflowKey, VisibilityTimestamp: now, + ArchetypeID: archetypeID, Priority: enumsspb.TASK_PRIORITY_HIGH, VersionedTransition: currentVersionedTransition, FirstEventID: firstEventID, @@ -6833,16 +7364,22 @@ func (ms *MutableStateImpl) closeTransactionPrepareReplicationTasks( default: panic(fmt.Sprintf("unknown transaction policy: %v", transactionPolicy)) } - } else { + } else if isWorkflow { ms.InsertTasks[tasks.CategoryReplication] = append( ms.InsertTasks[tasks.CategoryReplication], replicationTasks..., ) + } else { + return softassert.UnexpectedInternalErr(ms.logger, "state-based replication not enabled for chasm execution", nil) } if transactionPolicy == historyi.TransactionPolicyPassive && len(ms.InsertTasks[tasks.CategoryReplication]) > 0 { - return serviceerror.NewInternal("should not generate replication task when close transaction as passive") + return softassert.UnexpectedInternalErr( + ms.logger, + "should not generate replication task when close transaction as passive", + nil, + ) } return nil @@ -6897,6 +7434,7 @@ func (ms *MutableStateImpl) cleanupTransaction() error { ) ms.InsertTasks = make(map[tasks.Category][]tasks.Task) + ms.BestEffortDeleteTasks = make(map[tasks.Category][]tasks.Key) // Clear outputs for the next transaction. ms.stateMachineNode.ClearTransactionState() @@ -7885,6 +8423,7 @@ func (ms *MutableStateImpl) syncExecutionInfo(current *persistencespb.WorkflowEx RequestID: incoming.WorkflowTaskRequestId, WorkflowTaskTimeout: incoming.WorkflowTaskTimeout.AsDuration(), Attempt: incoming.WorkflowTaskAttempt, + Stamp: incoming.WorkflowTaskStamp, StartedTime: incoming.WorkflowTaskStartedTime.AsTime(), ScheduledTime: incoming.WorkflowTaskScheduledTime.AsTime(), @@ -8168,7 +8707,7 @@ func (ms *MutableStateImpl) GetEffectiveVersioningBehavior() enumspb.VersioningB // If there is a pending workflow task that is not started yet, it'll be rescheduled after // transition start. // This method must be called with a version different from the effective version. -func (ms *MutableStateImpl) StartDeploymentTransition(deployment *deploymentpb.Deployment) error { +func (ms *MutableStateImpl) StartDeploymentTransition(deployment *deploymentpb.Deployment, revisionNumber int64) error { wfBehavior := ms.GetEffectiveVersioningBehavior() if wfBehavior == enumspb.VERSIONING_BEHAVIOR_PINNED { // WF is pinned so we reject the transition. @@ -8202,7 +8741,7 @@ func (ms *MutableStateImpl) StartDeploymentTransition(deployment *deploymentpb.D // - reschedule the pending WFT so the old one is invalided ms.ClearStickyTaskQueue() - err := ms.reschedulePendingWorkflowTask(false) + err := ms.reschedulePendingWorkflowTask() if err != nil { return err } @@ -8216,9 +8755,22 @@ func (ms *MutableStateImpl) StartDeploymentTransition(deployment *deploymentpb.D ), ).Record(1) + ms.SetVersioningRevisionNumber(revisionNumber) + return nil } +func (ms *MutableStateImpl) GetVersioningRevisionNumber() int64 { + return ms.GetExecutionInfo().GetVersioningInfo().GetRevisionNumber() +} + +func (ms *MutableStateImpl) SetVersioningRevisionNumber(revisionNumber int64) { + if ms.GetExecutionInfo().GetVersioningInfo() == nil { + ms.GetExecutionInfo().VersioningInfo = &workflowpb.WorkflowExecutionVersioningInfo{} + } + ms.GetExecutionInfo().GetVersioningInfo().RevisionNumber = revisionNumber +} + // reschedulePendingActivities reschedules all the activities that are not started, so they are // scheduled against the right queue in matching. func (ms *MutableStateImpl) reschedulePendingActivities() error { @@ -8228,8 +8780,17 @@ func (ms *MutableStateImpl) reschedulePendingActivities() error { // activity already started continue } + + // need to update stamp so the passive side regenerate the task + err := ms.UpdateActivity(ai.ScheduledEventId, func(info *persistencespb.ActivityInfo, state historyi.MutableState) error { + info.Stamp++ + return nil + }) + if err != nil { + return err + } // we only need to resend the activities to matching, no need to update timer tasks. - err := ms.taskGenerator.GenerateActivityTasks(ai.ScheduledEventId) + err = ms.taskGenerator.GenerateActivityTasks(ai.ScheduledEventId) if err != nil { return err } @@ -8240,7 +8801,7 @@ func (ms *MutableStateImpl) reschedulePendingActivities() error { // reschedulePendingWorkflowTask reschedules the pending WFT if it is not started yet. // The currently scheduled WFT will be rejected when attempting to start because its stamp changed. -func (ms *MutableStateImpl) reschedulePendingWorkflowTask(invalidatePendingTasks bool) error { +func (ms *MutableStateImpl) reschedulePendingWorkflowTask() error { // If the WFT is started but not finished, we let it run its course // - once it's completed, failed or timed out a new one will be scheduled. if !ms.HasPendingWorkflowTask() || ms.HasStartedWorkflowTask() { @@ -8255,15 +8816,13 @@ func (ms *MutableStateImpl) reschedulePendingWorkflowTask(invalidatePendingTasks ms.logInfo("start transition did not reschedule pending speculative task") return nil } + // Reset the attempt; forcing a non-transient workflow task to be scheduled. + ms.executionInfo.WorkflowTaskAttempt = 1 - // If requested, increase the stamp ("version") to invalidate the pending non-speculative WFT. + // Increase the stamp ("version") to invalidate the pending non-speculative WFT. // We don't invalidate speculative WFTs because they are very latency sensitive. - if invalidatePendingTasks { - ms.executionInfo.WorkflowTaskStamp += 1 - } - - // Reset the attempt; forcing a non-transient workflow task to be scheduled. - ms.executionInfo.Attempt = 1 + ms.executionInfo.WorkflowTaskStamp += 1 + ms.workflowTaskUpdated = true return ms.taskGenerator.GenerateScheduleWorkflowTaskTasks(pendingTask.ScheduledEventID) } diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index 108198badb7..d6b6336271a 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/pborman/uuid" + "github.com/google/uuid" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/uber-go/tally/v4" @@ -32,7 +32,6 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" taskqueuespb "go.temporal.io/server/api/taskqueue/v1" "go.temporal.io/server/chasm" - chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/definition" @@ -45,6 +44,7 @@ import ( "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/searchattribute" + "go.temporal.io/server/common/searchattribute/sadefs" serviceerror2 "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/common/testing/testvars" @@ -223,7 +223,7 @@ func (s *mutableStateSuite) SetupSubTest() { func (s *mutableStateSuite) TestTransientWorkflowTaskCompletionFirstBatchApplied_ApplyWorkflowTaskCompleted() { version := int64(12) workflowID := "some random workflow ID" - runID := uuid.New() + runID := uuid.NewString() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -257,7 +257,7 @@ func (s *mutableStateSuite) TestTransientWorkflowTaskCompletionFirstBatchApplied func (s *mutableStateSuite) TestTransientWorkflowTaskCompletionFirstBatchApplied_FailoverWorkflowTaskTimeout() { version := int64(12) workflowID := "some random workflow ID" - runID := uuid.New() + runID := uuid.NewString() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -282,7 +282,7 @@ func (s *mutableStateSuite) TestTransientWorkflowTaskCompletionFirstBatchApplied func (s *mutableStateSuite) TestTransientWorkflowTaskCompletionFirstBatchApplied_FailoverWorkflowTaskFailed() { version := int64(12) workflowID := "some random workflow ID" - runID := uuid.New() + runID := uuid.NewString() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -475,11 +475,161 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_UnexpectedSticky() { s.Equal(int64(0), s.mutableState.GetExecutionInfo().GetBuildIdRedirectCounter()) } +func (s *mutableStateSuite) TestPopulateDeleteTasks_WithWorkflowTaskTimeouts() { + // Test that workflow task timeout task references are added to BestEffortDeleteTasks when present. + version := int64(1) + workflowID := "wf-timeout-delete" + runID := uuid.NewString() + s.mutableState = TestGlobalMutableState( + s.mockShard, + s.mockEventsCache, + s.logger, + version, + workflowID, + runID, + ) + + // Create mock timeout tasks that meet the criteria + now := time.Now().UTC() + mockScheduleToStartTask := &tasks.WorkflowTaskTimeoutTask{ + WorkflowKey: definition.NewWorkflowKey( + s.mutableState.GetExecutionInfo().NamespaceId, + workflowID, + runID, + ), + VisibilityTimestamp: now.Add(10 * time.Second), // < 120s + TaskID: 123, + TimeoutType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START, + InMemory: false, // Persisted task + } + + mockStartToCloseTask := &tasks.WorkflowTaskTimeoutTask{ + WorkflowKey: definition.NewWorkflowKey( + s.mutableState.GetExecutionInfo().NamespaceId, + workflowID, + runID, + ), + VisibilityTimestamp: now.Add(30 * time.Second), // < 120s + TaskID: 456, + TimeoutType: enumspb.TIMEOUT_TYPE_START_TO_CLOSE, + InMemory: false, // Persisted task + } + + // Schedule and start a workflow task + wft, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) + s.NoError(err) + + sticky := &taskqueuepb.TaskQueue{Name: "sticky-tq", Kind: enumspb.TASK_QUEUE_KIND_STICKY} + _, wft, err = s.mutableState.AddWorkflowTaskStartedEvent( + wft.ScheduledEventID, + "", + sticky, + "", + nil, + nil, + nil, + false, + ) + s.NoError(err) + + // Set timeout tasks directly in mutable state (simulating what task_generator does) + s.mutableState.SetWorkflowTaskScheduleToStartTimeoutTask(mockScheduleToStartTask) + s.mutableState.SetWorkflowTaskStartToCloseTimeoutTask(mockStartToCloseTask) + // Call UpdateWorkflowTask to persist the workflow task info to ExecutionInfo + s.mutableState.workflowTaskManager.UpdateWorkflowTask(wft) + + // Complete the workflow task + _, err = s.mutableState.AddWorkflowTaskCompletedEvent( + wft, + &workflowservice.RespondWorkflowTaskCompletedRequest{}, + workflowTaskCompletionLimits, + ) + s.NoError(err) + + // Verify that BestEffortDeleteTasks contains the timeout task keys + del := s.mutableState.BestEffortDeleteTasks + s.Contains(del, tasks.CategoryTimer) + s.Equal(2, len(del[tasks.CategoryTimer]), "Should have both ScheduleToStart and StartToClose timeout tasks") + s.Contains(del[tasks.CategoryTimer], mockScheduleToStartTask.GetKey()) + s.Contains(del[tasks.CategoryTimer], mockStartToCloseTask.GetKey()) +} + +func (s *mutableStateSuite) TestPopulateDeleteTasks_LongTimeout_NotIncluded() { + // Test that timeout tasks with very long timeouts (> 120s) are NOT added to BestEffortDeleteTasks. + version := int64(1) + workflowID := "wf-long-timeout" + runID := uuid.NewString() + s.mutableState = TestGlobalMutableState( + s.mockShard, + s.mockEventsCache, + s.logger, + version, + workflowID, + runID, + ) + + // Schedule a workflow task - this sets the scheduled time + wft, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) + s.NoError(err) + + sticky := &taskqueuepb.TaskQueue{Name: "sticky-tq", Kind: enumspb.TASK_QUEUE_KIND_STICKY} + _, wft, err = s.mutableState.AddWorkflowTaskStartedEvent( + wft.ScheduledEventID, + "", + sticky, + "", + nil, + nil, + nil, + false, + ) + s.NoError(err) + + // Get the actual scheduled time from wft (this is what will be used in the calculation) + scheduledTime := wft.ScheduledTime + if scheduledTime.IsZero() { + // If scheduled time is not set in wft, use current time + scheduledTime = time.Now().UTC() + } + + // Create a timeout task with timeout > 120s relative to the actual scheduled time + mockLongTimeoutTask := &tasks.WorkflowTaskTimeoutTask{ + WorkflowKey: definition.NewWorkflowKey( + s.mutableState.GetExecutionInfo().NamespaceId, + workflowID, + runID, + ), + VisibilityTimestamp: scheduledTime.Add(200 * time.Second), // > 120s from scheduled time + TaskID: 123, + TimeoutType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START, + InMemory: false, // Persisted task + } + + // Set the long timeout task directly in mutable state + s.mutableState.SetWorkflowTaskScheduleToStartTimeoutTask(mockLongTimeoutTask) + // Clear the StartToClose task so it doesn't interfere with the test + s.mutableState.SetWorkflowTaskStartToCloseTimeoutTask(nil) + + // Complete the workflow task + _, err = s.mutableState.AddWorkflowTaskCompletedEvent( + wft, + &workflowservice.RespondWorkflowTaskCompletedRequest{}, + workflowTaskCompletionLimits, + ) + s.NoError(err) + + // Verify that BestEffortDeleteTasks does NOT contain the long timeout task + del := s.mutableState.BestEffortDeleteTasks + if timerTasks, exists := del[tasks.CategoryTimer]; exists { + s.Equal(0, len(timerTasks), "Tasks with timeout > 120s should not be added to BestEffortDeleteTasks") + } +} + // creates a mutable state with first WFT completed on Build ID "b1" func (s *mutableStateSuite) createVersionedMutableStateWithCompletedWFT(tq *taskqueuepb.TaskQueue) { version := int64(12) workflowID := "some random workflow ID" - runID := uuid.New() + runID := uuid.NewString() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -752,7 +902,7 @@ func (s *mutableStateSuite) createMutableStateWithVersioningBehavior( ) { version := int64(12) workflowID := "some random workflow ID" - runID := uuid.New() + runID := uuid.NewString() s.mutableState = TestGlobalMutableState( s.mockShard, @@ -769,11 +919,9 @@ func (s *mutableStateSuite) createMutableStateWithVersioningBehavior( s.NoError(err) s.verifyEffectiveDeployment(nil, enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED) - err = s.mutableState.StartDeploymentTransition(deployment) + err = s.mutableState.StartDeploymentTransition(deployment, 0) s.NoError(err) s.verifyEffectiveDeployment(deployment, enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE) - s.EqualValues(1, s.mutableState.executionInfo.Attempt, - "workflow task attempt must be reset to 1 since pending tasks are rescheduled") _, wft, err = s.mutableState.AddWorkflowTaskStartedEvent( wft.ScheduledEventID, @@ -821,7 +969,7 @@ func (s *mutableStateSuite) TestUnpinnedTransition() { s.NoError(err) s.verifyEffectiveDeployment(deployment1, behavior) - err = s.mutableState.StartDeploymentTransition(deployment2) + err = s.mutableState.StartDeploymentTransition(deployment2, 0) s.NoError(err) s.verifyEffectiveDeployment(deployment2, behavior) @@ -860,7 +1008,7 @@ func (s *mutableStateSuite) TestUnpinnedTransitionFailed() { s.NoError(err) s.verifyEffectiveDeployment(deployment1, behavior) - err = s.mutableState.StartDeploymentTransition(deployment2) + err = s.mutableState.StartDeploymentTransition(deployment2, 0) s.NoError(err) s.verifyEffectiveDeployment(deployment2, behavior) @@ -902,7 +1050,7 @@ func (s *mutableStateSuite) TestUnpinnedTransitionTimeout() { s.NoError(err) s.verifyEffectiveDeployment(deployment1, behavior) - err = s.mutableState.StartDeploymentTransition(deployment2) + err = s.mutableState.StartDeploymentTransition(deployment2, 0) s.NoError(err) s.verifyEffectiveDeployment(deployment2, behavior) @@ -942,6 +1090,7 @@ func (s *mutableStateSuite) verifyWorkflowOptionsUpdatedEventAttr( s.Equal(expectedOverride.GetPinnedVersion(), actualOverride.GetPinnedVersion()) //nolint:staticcheck // SA1019: worker versioning v0.31 s.Equal(actualAttr.GetUnsetVersioningOverride(), expectedAttr.GetUnsetVersioningOverride()) + s.Equal(actualAttr.GetIdentity(), expectedAttr.GetIdentity()) } func (s *mutableStateSuite) verifyOverrides( @@ -972,10 +1121,11 @@ func (s *mutableStateSuite) TestOverride_UnpinnedBase_SetPinnedAndUnsetWithEmpty tq := &taskqueuepb.TaskQueue{Name: "tq"} baseBehavior := enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE overrideBehavior := enumspb.VERSIONING_BEHAVIOR_PINNED + id := uuid.NewString() s.createMutableStateWithVersioningBehavior(baseBehavior, deployment1, tq) // set pinned override - event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions2.GetVersioningOverride(), false, "", nil, nil, nil) + event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions2.GetVersioningOverride(), false, "", nil, nil, id, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment2, overrideBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -983,12 +1133,14 @@ func (s *mutableStateSuite) TestOverride_UnpinnedBase_SetPinnedAndUnsetWithEmpty &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: pinnedOptions2.GetVersioningOverride(), UnsetVersioningOverride: false, + Identity: id, }, ) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, deployment2) // unset pinned override with boolean - event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, nil) + id = uuid.NewString() + event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, id, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment1, baseBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -996,6 +1148,7 @@ func (s *mutableStateSuite) TestOverride_UnpinnedBase_SetPinnedAndUnsetWithEmpty &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: nil, UnsetVersioningOverride: true, + Identity: id, }, ) s.verifyOverrides(baseBehavior, enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED, deployment1, nil) @@ -1005,10 +1158,11 @@ func (s *mutableStateSuite) TestOverride_PinnedBase_SetUnpinnedAndUnsetWithEmpty tq := &taskqueuepb.TaskQueue{Name: "tq"} baseBehavior := enumspb.VERSIONING_BEHAVIOR_PINNED overrideBehavior := enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE + id := uuid.NewString() s.createMutableStateWithVersioningBehavior(baseBehavior, deployment1, tq) // set unpinned override - event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(unpinnedOptions.GetVersioningOverride(), false, "", nil, nil, nil) + event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(unpinnedOptions.GetVersioningOverride(), false, "", nil, nil, id, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment1, overrideBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1016,12 +1170,14 @@ func (s *mutableStateSuite) TestOverride_PinnedBase_SetUnpinnedAndUnsetWithEmpty &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: unpinnedOptions.GetVersioningOverride(), UnsetVersioningOverride: false, + Identity: id, }, ) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, nil) // unset pinned override with empty - event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, nil) + id = uuid.NewString() + event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, id, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment1, baseBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1029,6 +1185,7 @@ func (s *mutableStateSuite) TestOverride_PinnedBase_SetUnpinnedAndUnsetWithEmpty &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: nil, UnsetVersioningOverride: true, + Identity: id, }, ) s.verifyOverrides(baseBehavior, enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED, deployment1, nil) @@ -1038,9 +1195,10 @@ func (s *mutableStateSuite) TestOverride_RedirectFails() { tq := &taskqueuepb.TaskQueue{Name: "tq"} baseBehavior := enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE overrideBehavior := enumspb.VERSIONING_BEHAVIOR_PINNED + id := uuid.NewString() s.createMutableStateWithVersioningBehavior(baseBehavior, deployment1, tq) - event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions3.GetVersioningOverride(), false, "", nil, nil, nil) + event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions3.GetVersioningOverride(), false, "", nil, nil, id, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment3, overrideBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1048,12 +1206,13 @@ func (s *mutableStateSuite) TestOverride_RedirectFails() { &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: pinnedOptions3.GetVersioningOverride(), UnsetVersioningOverride: false, + Identity: id, }, ) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, deployment3) // assert that transition fails - err = s.mutableState.StartDeploymentTransition(deployment2) + err = s.mutableState.StartDeploymentTransition(deployment2, 0) s.ErrorIs(err, ErrPinnedWorkflowCannotTransition) s.verifyEffectiveDeployment(deployment3, overrideBehavior) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, deployment3) @@ -1063,9 +1222,10 @@ func (s *mutableStateSuite) TestOverride_BaseDeploymentUpdatedOnCompletion() { tq := &taskqueuepb.TaskQueue{Name: "tq"} baseBehavior := enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE overrideBehavior := enumspb.VERSIONING_BEHAVIOR_PINNED + id := uuid.NewString() s.createMutableStateWithVersioningBehavior(baseBehavior, deployment1, tq) - event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions3.GetVersioningOverride(), false, "", nil, nil, nil) + event, err := s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(pinnedOptions3.GetVersioningOverride(), false, "", nil, nil, id, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment3, overrideBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1073,12 +1233,13 @@ func (s *mutableStateSuite) TestOverride_BaseDeploymentUpdatedOnCompletion() { &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: pinnedOptions3.GetVersioningOverride(), UnsetVersioningOverride: false, + Identity: id, }, ) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, deployment3) // assert that redirect fails - should be its own test - err = s.mutableState.StartDeploymentTransition(deployment2) + err = s.mutableState.StartDeploymentTransition(deployment2, 0) s.ErrorIs(err, ErrPinnedWorkflowCannotTransition) s.verifyEffectiveDeployment(deployment3, overrideBehavior) s.verifyOverrides(baseBehavior, overrideBehavior, deployment1, deployment3) // base deployment still deployment1 here -- good @@ -1115,7 +1276,8 @@ func (s *mutableStateSuite) TestOverride_BaseDeploymentUpdatedOnCompletion() { s.verifyOverrides(baseBehavior, overrideBehavior, deployment2, deployment3) // now we unset the override and check that the base deployment/behavior is in effect - event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, nil) + id = uuid.NewString() + event, err = s.mutableState.AddWorkflowExecutionOptionsUpdatedEvent(nil, true, "", nil, nil, id, nil) s.NoError(err) s.verifyEffectiveDeployment(deployment2, baseBehavior) s.verifyWorkflowOptionsUpdatedEventAttr( @@ -1123,6 +1285,7 @@ func (s *mutableStateSuite) TestOverride_BaseDeploymentUpdatedOnCompletion() { &historypb.WorkflowExecutionOptionsUpdatedEventAttributes{ VersioningOverride: nil, UnsetVersioningOverride: true, + Identity: id, }, ) s.verifyOverrides(baseBehavior, enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED, deployment2, nil) @@ -1257,6 +1420,435 @@ func (s *mutableStateSuite) TestChecksumShouldInvalidate() { s.False(s.mutableState.shouldInvalidateCheckum()) } +func (s *mutableStateSuite) TestUpdateWorkflowStateStatus_Table() { + s.SetupSubTest() + cases := []struct { + name string + currentState enumsspb.WorkflowExecutionState + currentStatus enumspb.WorkflowExecutionStatus + toState enumsspb.WorkflowExecutionState + toStatus enumspb.WorkflowExecutionStatus + wantErr bool + }{ + { + name: "created-> {running, running}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + wantErr: false, + }, + { + name: "created-> {running, paused}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, + wantErr: false, + }, + { + name: "created-> {running, completed}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + wantErr: true, + }, + // CREATED -> CREATED (allowed for RUNNING/PAUSED) + { + name: "created-> {created, running}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + wantErr: false, + }, + { + name: "created-> {created, paused}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, + wantErr: true, + }, + { + name: "created-> {created, completed} (invalid)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + wantErr: true, + }, + // CREATED -> COMPLETED (allowed only for TERMINATED/TIMED_OUT/CONTINUED_AS_NEW) + { + name: "created-> {completed, terminated}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, + wantErr: false, + }, + { + name: "created-> {completed, timed_out}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TIMED_OUT, + wantErr: false, + }, + { + name: "created-> {completed, continued_as_new}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_CONTINUED_AS_NEW, + wantErr: false, + }, + // CREATED -> ZOMBIE (allowed for RUNNING/PAUSED) + { + name: "created-> {zombie, running}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + wantErr: false, + }, + { + name: "created-> {zombie, paused}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, + wantErr: false, + }, + // RUNNING state transitions + { + name: "running-> {created, running} (invalid)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + wantErr: true, + }, + { + name: "running-> {running, paused}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, + wantErr: false, + }, + { + name: "running-> {running, terminated} (invalid)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, + wantErr: true, + }, + { + name: "running-> {completed, completed}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + wantErr: false, + }, + { + name: "running-> {completed, paused} (invalid)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, + wantErr: true, + }, + { + name: "running-> {zombie, running}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + wantErr: false, + }, + { + name: "running-> {zombie, paused}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, + wantErr: false, + }, + { + name: "running-> {zombie, terminated} (invalid)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, + wantErr: true, + }, + // COMPLETED state transitions + { + name: "completed-> {completed, sameStatus} (no-op)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + currentStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + wantErr: false, + }, + { + name: "completed-> {created, running} (invalid)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + currentStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + wantErr: true, + }, + { + name: "completed-> {running, running} (invalid)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + currentStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + wantErr: true, + }, + { + name: "completed-> {zombie, running} (invalid)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + currentStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + wantErr: true, + }, + { + name: "completed-> {completed, differentStatus} (invalid)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + currentStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_FAILED, + wantErr: true, + }, + // ZOMBIE state transitions + { + name: "zombie-> {created, running}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + wantErr: false, + }, + { + name: "zombie-> {created, paused}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, + wantErr: true, + }, + { + name: "zombie-> {running, paused}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, + wantErr: false, + }, + { + name: "zombie-> {completed, terminated}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, + wantErr: false, + }, + { + name: "zombie-> {completed, paused} (invalid)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, + wantErr: true, + }, + { + name: "zombie-> {zombie, running}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + wantErr: false, + }, + { + name: "zombie-> {zombie, terminated} (invalid)", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, + wantErr: true, + }, + // VOID state (no validation) + { + name: "void-> {running, running}", + currentState: enumsspb.WORKFLOW_EXECUTION_STATE_VOID, + toState: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + toStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + wantErr: false, + }, + } + + for _, c := range cases { + s.Run(c.name, func() { + s.SetupSubTest() + s.mutableState.executionState.State = c.currentState + // default current status to RUNNING unless specified + curStatus := c.currentStatus + if curStatus == 0 { + curStatus = enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING + } + s.mutableState.executionState.Status = curStatus + _, err := s.mutableState.UpdateWorkflowStateStatus(c.toState, c.toStatus) + if c.wantErr { + s.Error(err) + } else { + s.NoError(err) + } + if !c.wantErr { // if the transition was successful, verify the state and status are updated. + s.Equal(c.toState, s.mutableState.executionState.State) + s.Equal(c.toStatus, s.mutableState.executionState.Status) + } + }) + } +} + +func (s *mutableStateSuite) TestAddWorkflowExecutionPausedEvent() { + s.SetupSubTest() + s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() + + tq := &taskqueuepb.TaskQueue{Name: "tq"} + s.createVersionedMutableStateWithCompletedWFT(tq) + + // Complete another WFT to obtain a valid completed event id for scheduling an activity. + wft, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) + s.NoError(err) + _, wft, err = s.mutableState.AddWorkflowTaskStartedEvent( + wft.ScheduledEventID, + "", + tq, + "", + worker_versioning.StampFromBuildId("b1"), + nil, + nil, + false, + ) + s.NoError(err) + completedEvent, err := s.mutableState.AddWorkflowTaskCompletedEvent( + wft, + &workflowservice.RespondWorkflowTaskCompletedRequest{}, + workflowTaskCompletionLimits, + ) + s.NoError(err) + + // Schedule an activity (pending) using the completed WFT event id. + _, activityInfo, err := s.mutableState.AddActivityTaskScheduledEvent( + completedEvent.GetEventId(), + &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: "act-1", + ActivityType: &commonpb.ActivityType{Name: "activity-type"}, + TaskQueue: tq, + }, + false, + ) + s.NoError(err) + prevActivityStamp := activityInfo.Stamp + + // Create a pending workflow task. + pendingWFT, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) + s.NoError(err) + prevWFTStamp := pendingWFT.Stamp + + // Pause and assert stamps incremented. + pausedEvent, err := s.mutableState.AddWorkflowExecutionPausedEvent("tester", "reason", uuid.NewString()) + s.NoError(err) + + updatedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) + s.True(ok) + s.Greater(updatedActivityInfo.Stamp, prevActivityStamp) + + wftInfo := s.mutableState.GetPendingWorkflowTask() + s.NotNil(wftInfo) + s.Greater(wftInfo.Stamp, prevWFTStamp) + + // assert the event is marked as 'worker may ignore' so that older SDKs can safely ignore it. + s.True(pausedEvent.GetWorkerMayIgnore()) +} + +func (s *mutableStateSuite) TestAddWorkflowExecutionUnpausedEvent() { + s.SetupSubTest() + s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() + + tq := &taskqueuepb.TaskQueue{Name: "tq"} + s.createVersionedMutableStateWithCompletedWFT(tq) + + // Complete another WFT to obtain a valid completed event id for scheduling an activity. + wft, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) + s.NoError(err) + _, wft, err = s.mutableState.AddWorkflowTaskStartedEvent( + wft.ScheduledEventID, + "", + tq, + "", + worker_versioning.StampFromBuildId("b1"), + nil, + nil, + false, + ) + s.NoError(err) + completedEvent, err := s.mutableState.AddWorkflowTaskCompletedEvent( + wft, + &workflowservice.RespondWorkflowTaskCompletedRequest{}, + workflowTaskCompletionLimits, + ) + s.NoError(err) + + // Schedule an activity (pending) using the completed WFT event id. + _, activityInfo, err := s.mutableState.AddActivityTaskScheduledEvent( + completedEvent.GetEventId(), + &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: "act-1", + ActivityType: &commonpb.ActivityType{Name: "activity-type"}, + TaskQueue: tq, + }, + false, + ) + s.NoError(err) + // Create a pending workflow task. + pendingWFT, err := s.mutableState.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) + s.NoError(err) + + // Pause first to simulate paused workflow state. + _, err = s.mutableState.AddWorkflowExecutionPausedEvent("tester", "reason", uuid.NewString()) + s.NoError(err) + + // Capture stamps after pause. + pausedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) + s.True(ok) + pausedActivityStamp := pausedActivityInfo.Stamp + pausedWFT := s.mutableState.GetPendingWorkflowTask() + s.NotNil(pausedWFT) + pausedWFTStamp := pausedWFT.Stamp + + // Unpause and verify. + unpausedEvent, err := s.mutableState.AddWorkflowExecutionUnpausedEvent("tester", "reason", uuid.NewString()) + s.NoError(err) + + // PauseInfo should be cleared and status should be RUNNING. + s.Nil(s.mutableState.executionInfo.PauseInfo) + s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, s.mutableState.executionState.Status) + + // Stamps should be incremented again (only for activities) on unpause. + updatedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) + s.True(ok) + s.Greater(updatedActivityInfo.Stamp, pausedActivityStamp) + + currentWFT := s.mutableState.GetPendingWorkflowTask() + s.NotNil(currentWFT) + s.Equal(currentWFT.Stamp, pausedWFTStamp) // workflow task stamp should not change between pause and unpause. + + // assert the event is marked as 'worker may ignore' so that older SDKs can safely ignore it. + s.True(unpausedEvent.GetWorkerMayIgnore()) + + // Ensure the pending workflow task we created earlier still exists (no unexpected removal). + s.Equal(pendingWFT.ScheduledEventID, currentWFT.ScheduledEventID) +} + +func (s *mutableStateSuite) TestPauseWorkflowExecution_FailStateValidation() { + s.SetupSubTest() + s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() + + // Simulate a completed workflow where transitioning status to PAUSED is invalid. + s.mutableState.executionState.State = enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED + s.mutableState.executionState.Status = enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED + prevStatus := s.mutableState.executionState.Status + + _, err := s.mutableState.AddWorkflowExecutionPausedEvent("tester", "test_reason", uuid.NewString()) + s.Error(err) + // Status should remain unchanged and PauseInfo should not be set when validation fails. + s.Equal(prevStatus, s.mutableState.executionState.Status) + s.Nil(s.mutableState.executionInfo.PauseInfo) +} + func (s *mutableStateSuite) TestContinueAsNewMinBackoff() { // set ContinueAsNew min interval to 5s s.mockConfig.WorkflowIdReuseMinimalInterval = func(namespace string) time.Duration { @@ -1313,7 +1905,7 @@ func (s *mutableStateSuite) TestContinueAsNewMinBackoff() { } func (s *mutableStateSuite) TestEventReapplied() { - runID := uuid.New() + runID := uuid.NewString() eventID := int64(1) version := int64(2) dedupResource := definition.NewEventReappliedID(runID, eventID, version) @@ -1327,7 +1919,7 @@ func (s *mutableStateSuite) TestEventReapplied() { func (s *mutableStateSuite) TestTransientWorkflowTaskSchedule_CurrentVersionChanged() { version := int64(2000) workflowID := "some random workflow ID" - runID := uuid.New() + runID := uuid.NewString() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -1362,7 +1954,7 @@ func (s *mutableStateSuite) TestTransientWorkflowTaskSchedule_CurrentVersionChan func (s *mutableStateSuite) TestTransientWorkflowTaskStart_CurrentVersionChanged() { version := int64(2000) workflowID := "some random workflow ID" - runID := uuid.New() + runID := uuid.NewString() s.mutableState = TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -1396,7 +1988,7 @@ func (s *mutableStateSuite) TestTransientWorkflowTaskStart_CurrentVersionChanged _, _, err = s.mutableState.AddWorkflowTaskStartedEvent( s.mutableState.GetNextEventID(), - uuid.New(), + uuid.NewString(), &taskqueuepb.TaskQueue{Name: f.TaskQueue(enumspb.TASK_QUEUE_TYPE_WORKFLOW).NormalPartition(5).RpcName()}, "random identity", nil, @@ -1432,7 +2024,7 @@ func (s *mutableStateSuite) TestNewMutableStateInChain() { s.logger, 1000, tests.WorkflowID, - uuid.New(), + uuid.NewString(), ) currentMutableState.GetExecutionInfo().WorkflowExecutionTimerTaskStatus = taskStatus @@ -1442,7 +2034,7 @@ func (s *mutableStateSuite) TestNewMutableStateInChain() { s.logger, tests.GlobalNamespaceEntry, tests.WorkflowID, - uuid.New(), + uuid.NewString(), s.mockShard.GetTimeSource().Now(), currentMutableState, ) @@ -1455,7 +2047,7 @@ func (s *mutableStateSuite) TestNewMutableStateInChain() { func (s *mutableStateSuite) TestSanitizedMutableState() { txnID := int64(2000) - runID := uuid.New() + runID := uuid.NewString() mutableState := TestGlobalMutableState( s.mockShard, s.mockEventsCache, @@ -1556,7 +2148,7 @@ func (s *mutableStateSuite) prepareTransientWorkflowTaskCompletionFirstBatchAppl EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, Attributes: &historypb.HistoryEvent_WorkflowTaskStartedEventAttributes{WorkflowTaskStartedEventAttributes: &historypb.WorkflowTaskStartedEventAttributes{ ScheduledEventId: workflowTaskScheduleEvent.GetEventId(), - RequestId: uuid.New(), + RequestId: uuid.NewString(), }}, } eventID++ @@ -1586,7 +2178,7 @@ func (s *mutableStateSuite) prepareTransientWorkflowTaskCompletionFirstBatchAppl err := s.mutableState.ApplyWorkflowExecutionStartedEvent( nil, execution, - uuid.New(), + uuid.NewString(), workflowStartEvent, ) s.Nil(err) @@ -1644,7 +2236,7 @@ func (s *mutableStateSuite) prepareTransientWorkflowTaskCompletionFirstBatchAppl EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, Attributes: &historypb.HistoryEvent_WorkflowTaskStartedEventAttributes{WorkflowTaskStartedEventAttributes: &historypb.WorkflowTaskStartedEventAttributes{ ScheduledEventId: workflowTaskScheduleEvent.GetEventId(), - RequestId: uuid.New(), + RequestId: uuid.NewString(), }}, } eventID++ @@ -1740,7 +2332,7 @@ func (s *mutableStateSuite) buildWorkflowMutableState() *persistencespb.Workflow TransitionCount: 1024, }, }, - FirstExecutionRunId: uuid.New(), + FirstExecutionRunId: uuid.NewString(), WorkflowExecutionTimerTaskStatus: TimerTaskStatusCreated, } @@ -1782,7 +2374,7 @@ func (s *mutableStateSuite) buildWorkflowMutableState() *persistencespb.Workflow InitiatedEventId: 80, InitiatedEventBatchId: 20, StartedEventId: common.EmptyEventID, - CreateRequestId: uuid.New(), + CreateRequestId: uuid.NewString(), Namespace: tests.Namespace.String(), WorkflowTypeName: "code.uber.internal/test/foobar", }, @@ -1792,7 +2384,7 @@ func (s *mutableStateSuite) buildWorkflowMutableState() *persistencespb.Workflow 70: { Version: failoverVersion, InitiatedEventBatchId: 20, - CancelRequestId: uuid.New(), + CancelRequestId: uuid.NewString(), InitiatedEventId: 70, }, } @@ -1802,7 +2394,7 @@ func (s *mutableStateSuite) buildWorkflowMutableState() *persistencespb.Workflow Version: failoverVersion, InitiatedEventId: 75, InitiatedEventBatchId: 17, - RequestId: uuid.New(), + RequestId: uuid.NewString(), }, } @@ -2079,7 +2671,7 @@ func (s *mutableStateSuite) TestTotalEntitiesCount() { _, _, err = s.mutableState.AddStartChildWorkflowExecutionInitiatedEvent( workflowTaskCompletedEventID, &commandpb.StartChildWorkflowExecutionCommandAttributes{}, - namespace.ID(uuid.New()), + namespace.ID(uuid.NewString()), ) s.NoError(err) @@ -2091,22 +2683,22 @@ func (s *mutableStateSuite) TestTotalEntitiesCount() { _, _, err = s.mutableState.AddRequestCancelExternalWorkflowExecutionInitiatedEvent( workflowTaskCompletedEventID, - uuid.New(), + uuid.NewString(), &commandpb.RequestCancelExternalWorkflowExecutionCommandAttributes{}, - namespace.ID(uuid.New()), + namespace.ID(uuid.NewString()), ) s.NoError(err) _, _, err = s.mutableState.AddSignalExternalWorkflowExecutionInitiatedEvent( workflowTaskCompletedEventID, - uuid.New(), + uuid.NewString(), &commandpb.SignalExternalWorkflowExecutionCommandAttributes{ Execution: &commonpb.WorkflowExecution{ WorkflowId: tests.WorkflowID, RunId: tests.RunID, }, }, - namespace.ID(uuid.New()), + namespace.ID(uuid.NewString()), ) s.NoError(err) @@ -2234,6 +2826,7 @@ func (s *mutableStateSuite) TestRetryWorkflowTask_WithNextRetryDelay() { } func (s *mutableStateSuite) TestRetryActivity_TruncateRetryableFailure() { s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() + s.mockConfig.EnableActivityRetryStampIncrement = dynamicconfig.GetBoolPropertyFn(true) // scheduling, starting & completing workflow task is omitted here @@ -2255,7 +2848,7 @@ func (s *mutableStateSuite) TestRetryActivity_TruncateRetryableFailure() { _, err = s.mutableState.AddActivityTaskStartedEvent( activityInfo, activityInfo.ScheduledEventId, - uuid.New(), + uuid.NewString(), "worker-identity", nil, nil, @@ -2284,16 +2877,63 @@ func (s *mutableStateSuite) TestRetryActivity_TruncateRetryableFailure() { } s.Greater(activityFailure.Size(), failureSizeErrorLimit) + prevStamp := activityInfo.Stamp + retryState, err := s.mutableState.RetryActivity(activityInfo, activityFailure) s.NoError(err) s.Equal(enumspb.RETRY_STATE_IN_PROGRESS, retryState) activityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) s.True(ok) + s.Greater(activityInfo.Stamp, prevStamp) + s.Equal(int32(2), activityInfo.Attempt) s.LessOrEqual(activityInfo.RetryLastFailure.Size(), failureSizeErrorLimit) s.Equal(activityFailure.GetMessage(), activityInfo.RetryLastFailure.Cause.GetMessage()) } +func (s *mutableStateSuite) TestRetryActivity_PausedIncrementsStamp() { + s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() + s.mockConfig.EnableActivityRetryStampIncrement = dynamicconfig.GetBoolPropertyFn(true) + + workflowTaskCompletedEventID := int64(4) + _, activityInfo, err := s.mutableState.AddActivityTaskScheduledEvent( + workflowTaskCompletedEventID, + &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: "6", + ActivityType: &commonpb.ActivityType{Name: "activity-type"}, + TaskQueue: &taskqueuepb.TaskQueue{Name: "task-queue"}, + RetryPolicy: &commonpb.RetryPolicy{ + InitialInterval: timestamp.DurationFromSeconds(1), + }, + }, + false, + ) + s.NoError(err) + + _, err = s.mutableState.AddActivityTaskStartedEvent( + activityInfo, + activityInfo.ScheduledEventId, + uuid.NewString(), + "worker-identity", + nil, + nil, + nil, + ) + s.NoError(err) + + activityInfo.Paused = true + prevStamp := activityInfo.Stamp + + retryState, err := s.mutableState.RetryActivity(activityInfo, &failurepb.Failure{Message: "activity failure"}) + s.NoError(err) + s.Equal(enumspb.RETRY_STATE_IN_PROGRESS, retryState) + + updatedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) + s.True(ok) + s.Greater(updatedActivityInfo.Stamp, prevStamp) + s.Equal(int32(2), updatedActivityInfo.Attempt) +} + func (s *mutableStateSuite) TestupdateBuildIdsAndDeploymentSearchAttributes() { versioned := func(buildId string) *commonpb.WorkerVersionStamp { return &commonpb.WorkerVersionStamp{BuildId: buildId, UseVersioning: true} @@ -2413,9 +3053,9 @@ func (s *mutableStateSuite) TestAddResetPointFromCompletion() { } func (s *mutableStateSuite) TestRolloverAutoResetPointsWithExpiringTime() { - runId1 := uuid.New() - runId2 := uuid.New() - runId3 := uuid.New() + runID1 := uuid.NewString() + runID2 := uuid.NewString() + runID3 := uuid.NewString() retention := 3 * time.Hour base := time.Now() @@ -2427,40 +3067,40 @@ func (s *mutableStateSuite) TestRolloverAutoResetPointsWithExpiringTime() { points := []*workflowpb.ResetPointInfo{ { BuildId: "buildid1", - RunId: runId1, + RunId: runID1, FirstWorkflowTaskCompletedId: 32, ExpireTime: t1, }, { BuildId: "buildid2", - RunId: runId1, + RunId: runID1, FirstWorkflowTaskCompletedId: 63, ExpireTime: t1, }, { BuildId: "buildid3", - RunId: runId2, + RunId: runID2, FirstWorkflowTaskCompletedId: 94, ExpireTime: t2, }, { BuildId: "buildid4", - RunId: runId3, + RunId: runID3, FirstWorkflowTaskCompletedId: 125, }, } - newPoints := rolloverAutoResetPointsWithExpiringTime(&workflowpb.ResetPoints{Points: points}, runId3, now.AsTime(), retention) + newPoints := rolloverAutoResetPointsWithExpiringTime(&workflowpb.ResetPoints{Points: points}, runID3, now.AsTime(), retention) expected := []*workflowpb.ResetPointInfo{ { BuildId: "buildid3", - RunId: runId2, + RunId: runID2, FirstWorkflowTaskCompletedId: 94, ExpireTime: t2, }, { BuildId: "buildid4", - RunId: runId3, + RunId: runID3, FirstWorkflowTaskCompletedId: 125, ExpireTime: t3, }, @@ -2602,7 +3242,7 @@ func (s *mutableStateSuite) TestCloseTransactionUpdateTransition() { }, txFunc: func(ms historyi.MutableState) (*persistencespb.WorkflowExecutionInfo, error) { mockChasmTree := historyi.NewMockChasmTree(s.controller) - mockChasmTree.EXPECT().Archetype().Return(chasm.Archetype("mock-archetype")).AnyTimes() + mockChasmTree.EXPECT().ArchetypeID().Return(chasm.ArchetypeID(1234)).AnyTimes() gomock.InOrder( mockChasmTree.EXPECT().IsStateDirty().Return(true).AnyTimes(), mockChasmTree.EXPECT().CloseTransaction().Return(chasm.NodesMutation{ @@ -2821,7 +3461,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackLastUpdateVersionedTransiti completedEvent := completWorkflowTaskFn(ms) initiatedEvent, _, err := ms.AddRequestCancelExternalWorkflowExecutionInitiatedEvent( completedEvent.GetEventId(), - uuid.New(), + uuid.NewString(), &commandpb.RequestCancelExternalWorkflowExecutionCommandAttributes{}, ms.GetNamespaceEntry().ID(), ) @@ -2848,7 +3488,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackLastUpdateVersionedTransiti completedEvent := completWorkflowTaskFn(ms) initiatedEvent, _, err := ms.AddSignalExternalWorkflowExecutionInitiatedEvent( completedEvent.GetEventId(), - uuid.New(), + uuid.NewString(), &commandpb.SignalExternalWorkflowExecutionCommandAttributes{ Execution: &commonpb.WorkflowExecution{ WorkflowId: "target-workflow-id", @@ -2877,7 +3517,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackLastUpdateVersionedTransiti { name: "SignalRequestedID", testFn: func(ms historyi.MutableState) { - ms.AddSignalRequested(uuid.New()) + ms.AddSignalRequested(uuid.NewString()) _, _, err := ms.CloseTransactionAsMutation(historyi.TransactionPolicyActive) s.NoError(err) @@ -3196,7 +3836,7 @@ func (s *mutableStateSuite) TestCloseTransactionHandleUnknownVersionedTransition } func (s *mutableStateSuite) getBuildIdsFromMutableState() []string { - payload, found := s.mutableState.executionInfo.SearchAttributes[searchattribute.BuildIds] + payload, found := s.mutableState.executionInfo.SearchAttributes[sadefs.BuildIds] if !found { return []string{} } @@ -3473,7 +4113,7 @@ func (s *mutableStateSuite) TestCloseTransactionPrepareReplicationTasks_HistoryT EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, Attributes: &historypb.HistoryEvent_WorkflowTaskStartedEventAttributes{WorkflowTaskStartedEventAttributes: &historypb.WorkflowTaskStartedEventAttributes{ ScheduledEventId: firstEventID, - RequestId: uuid.New(), + RequestId: uuid.NewString(), }}, }, }, @@ -3573,7 +4213,7 @@ func (s *mutableStateSuite) TestCloseTransactionPrepareReplicationTasks_SyncVers EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, Attributes: &historypb.HistoryEvent_WorkflowTaskStartedEventAttributes{WorkflowTaskStartedEventAttributes: &historypb.WorkflowTaskStartedEventAttributes{ ScheduledEventId: firstEventID, - RequestId: uuid.New(), + RequestId: uuid.NewString(), }}, }, }, @@ -3614,6 +4254,7 @@ func (s *mutableStateSuite) TestCloseTransactionPrepareReplicationTasks_SyncVers } expectedTask := &tasks.SyncVersionedTransitionTask{ WorkflowKey: s.mutableState.GetWorkflowKey(), + ArchetypeID: chasm.WorkflowArchetypeID, VisibilityTimestamp: now, Priority: enumsspb.TASK_PRIORITY_HIGH, VersionedTransition: transitionHistory[0], @@ -3625,6 +4266,7 @@ func (s *mutableStateSuite) TestCloseTransactionPrepareReplicationTasks_SyncVers s.True(ok) s.Equal(expectedTask.WorkflowKey, actualTask.WorkflowKey) s.Equal(expectedTask.VersionedTransition, actualTask.VersionedTransition) + s.Equal(expectedTask.ArchetypeID, actualTask.ArchetypeID) s.Equal(3, len(actualTask.TaskEquivalents)) s.Equal(historyTasks[0], actualTask.TaskEquivalents[0]) s.Equal(historyTasks[1], actualTask.TaskEquivalents[1]) @@ -3947,8 +4589,8 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones() { break } childExecution := &commonpb.WorkflowExecution{ - WorkflowId: uuid.New(), - RunId: uuid.New(), + WorkflowId: uuid.NewString(), + RunId: uuid.NewString(), } _, err := mutableState.AddChildWorkflowExecutionStartedEvent( childExecution, @@ -3982,8 +4624,8 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones() { initiatedEventId, s.namespaceEntry.Name(), s.namespaceEntry.ID(), - uuid.New(), - uuid.New(), + uuid.NewString(), + uuid.NewString(), enumspb.CANCEL_EXTERNAL_WORKFLOW_EXECUTION_FAILED_CAUSE_EXTERNAL_WORKFLOW_EXECUTION_NOT_FOUND, ) return &persistencespb.StateMachineTombstone{ @@ -4004,8 +4646,8 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones() { initiatedEventId, s.namespaceEntry.Name(), s.namespaceEntry.ID(), - uuid.New(), - uuid.New(), + uuid.NewString(), + uuid.NewString(), "", enumspb.SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_FAILED_CAUSE_EXTERNAL_WORKFLOW_EXECUTION_NOT_FOUND, ) @@ -4027,7 +4669,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones() { } mockChasmTree := historyi.NewMockChasmTree(s.controller) - mockChasmTree.EXPECT().Archetype().Return(chasm.Archetype("mock-archetype")).AnyTimes() + mockChasmTree.EXPECT().ArchetypeID().Return(chasm.ArchetypeID(1234)).AnyTimes() gomock.InOrder( mockChasmTree.EXPECT().IsStateDirty().Return(true).AnyTimes(), mockChasmTree.EXPECT().CloseTransaction().Return(chasm.NodesMutation{ @@ -4106,7 +4748,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones_CapIfLargerThanL Version: s.namespaceEntry.FailoverVersion(), InitiatedEventId: int64(76 + i), InitiatedEventBatchId: 17, - RequestId: uuid.New(), + RequestId: uuid.NewString(), } } @@ -4118,8 +4760,8 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones_CapIfLargerThanL initiatedEventId, s.namespaceEntry.Name(), s.namespaceEntry.ID(), - uuid.New(), - uuid.New(), + uuid.NewString(), + uuid.NewString(), "", enumspb.SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_FAILED_CAUSE_EXTERNAL_WORKFLOW_EXECUTION_NOT_FOUND, ) @@ -4162,7 +4804,7 @@ func (s *mutableStateSuite) TestCloseTransactionTrackTombstones_OnlyTrackFirstEm s.Equal(int64(1), tombstoneBatches[0].VersionedTransition.TransitionCount) } -func (s *mutableStateSuite) TestCloseTransactionGenerateCHASMRetentionTask() { +func (s *mutableStateSuite) TestCloseTransactionGenerateCHASMRetentionTask_Workflow() { dbState := s.buildWorkflowMutableState() mutableState, err := NewMutableStateFromDB(s.mockShard, s.mockEventsCache, s.logger, s.namespaceEntry, dbState, 123) @@ -4180,20 +4822,38 @@ func (s *mutableStateSuite) TestCloseTransactionGenerateCHASMRetentionTask() { // Is workflow, should not generate retention task mockChasmTree.EXPECT().IsStateDirty().Return(true).AnyTimes() - mockChasmTree.EXPECT().Archetype().Return(chasmworkflow.Archetype).Times(1) + mockChasmTree.EXPECT().ArchetypeID().Return(chasm.WorkflowArchetypeID).AnyTimes() mockChasmTree.EXPECT().CloseTransaction().Return(chasm.NodesMutation{}, nil).AnyTimes() mutation, _, err := mutableState.CloseTransactionAsMutation(historyi.TransactionPolicyActive) s.NoError(err) s.Empty(mutation.Tasks[tasks.CategoryTimer]) +} + +func (s *mutableStateSuite) TestCloseTransactionGenerateCHASMRetentionTask_NonWorkflow() { + dbState := s.buildWorkflowMutableState() - // Now make the mutable state non-workflow. - mockChasmTree.EXPECT().Archetype().Return(chasm.Archetype("test-archetype")).Times(2) // One time for each CloseTransactionAsMutation call + mutableState, err := NewMutableStateFromDB(s.mockShard, s.mockEventsCache, s.logger, s.namespaceEntry, dbState, 123) + s.NoError(err) + + // First close transaction once to get rid of unrelated tasks like UserTimer and ActivityTimeout + _, err = mutableState.StartTransaction(s.namespaceEntry) + s.NoError(err) + _, _, err = mutableState.CloseTransactionAsMutation(historyi.TransactionPolicyActive) + s.NoError(err) + + // Switch to a mock CHASM tree + mockChasmTree := historyi.NewMockChasmTree(s.controller) + mutableState.chasmTree = mockChasmTree + + mockChasmTree.EXPECT().IsStateDirty().Return(true).AnyTimes() + mockChasmTree.EXPECT().ArchetypeID().Return(chasm.WorkflowArchetypeID + 101).AnyTimes() + mockChasmTree.EXPECT().CloseTransaction().Return(chasm.NodesMutation{}, nil).AnyTimes() _, err = mutableState.UpdateWorkflowStateStatus( enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, ) s.NoError(err) - mutation, _, err = mutableState.CloseTransactionAsMutation(historyi.TransactionPolicyActive) + mutation, _, err := mutableState.CloseTransactionAsMutation(historyi.TransactionPolicyActive) s.NoError(err) s.Len(mutation.Tasks[tasks.CategoryTimer], 1) s.Equal(enumsspb.TASK_TYPE_DELETE_HISTORY_EVENT, mutation.Tasks[tasks.CategoryTimer][0].GetType()) @@ -4236,8 +4896,8 @@ func (s *mutableStateSuite) addChangesForStateReplication(state *persistencespb. state.ActivityInfos[90].TimerTaskStatus = TimerTaskStatusCreated state.TimerInfos["25"].ExpiryTime = timestamp.TimeNowPtrUtcAddDuration(time.Hour) state.ChildExecutionInfos[80].StartedEventId = 84 - state.RequestCancelInfos[70].CancelRequestId = uuid.New() - state.SignalInfos[75].RequestId = uuid.New() + state.RequestCancelInfos[70].CancelRequestId = uuid.NewString() + state.SignalInfos[75].RequestId = uuid.NewString() // These infos will be deleted during ApplySnapshot state.ActivityInfos[89] = &persistencespb.ActivityInfo{} @@ -4380,6 +5040,7 @@ func (s *mutableStateSuite) verifyExecutionInfo(current, target, origin *persist s.Equal(target.StickyTaskQueue, current.StickyTaskQueue, "StickyTaskQueue mismatch") s.True(proto.Equal(target.StickyScheduleToStartTimeout, current.StickyScheduleToStartTimeout), "StickyScheduleToStartTimeout mismatch") s.Equal(target.Attempt, current.Attempt, "Attempt mismatch") + s.Equal(target.WorkflowTaskStamp, current.WorkflowTaskStamp, "WorkflowTaskStamp mismatch") s.True(proto.Equal(target.RetryInitialInterval, current.RetryInitialInterval), "RetryInitialInterval mismatch") s.True(proto.Equal(target.RetryMaximumInterval, current.RetryMaximumInterval), "RetryMaximumInterval mismatch") s.Equal(target.RetryMaximumAttempts, current.RetryMaximumAttempts, "RetryMaximumAttempts mismatch") @@ -5163,3 +5824,77 @@ func (s *mutableStateSuite) TestHasRequestID_EmptyExecutionState() { s.False(s.mutableState.HasRequestID(requestID), "Should return false for request ID: %s", requestID) } } + +func (s *mutableStateSuite) TestAddTasks_CHASMPureTask() { + s.mockConfig.ChasmMaxInMemoryPureTasks = dynamicconfig.GetIntPropertyFn(5) + totalTasks := 2 * s.mockConfig.ChasmMaxInMemoryPureTasks() + + visTimestamp := s.mockShard.GetTimeSource().Now() + for i := 0; i < totalTasks; i++ { + task := &tasks.ChasmTaskPure{ + VisibilityTimestamp: visTimestamp, + } + s.mutableState.AddTasks(task) + s.LessOrEqual(len(s.mutableState.chasmPureTasks), s.mockConfig.ChasmMaxInMemoryPureTasks()) + + visTimestamp = visTimestamp.Add(-time.Minute) + } + + s.mockConfig.ChasmMaxInMemoryPureTasks = dynamicconfig.GetIntPropertyFn(2) + s.mutableState.AddTasks(&tasks.ChasmTaskPure{ + VisibilityTimestamp: visTimestamp, + }) + s.Len(s.mutableState.chasmPureTasks, 2) +} + +func (s *mutableStateSuite) TestDeleteCHASMPureTasks() { + now := s.mockShard.GetTimeSource().Now() + + testCases := []struct { + name string + maxScheduledTime time.Time + expectedRemaining int + }{ + { + name: "none", + maxScheduledTime: now, + expectedRemaining: 3, + }, + { + name: "paritial", + maxScheduledTime: now.Add(2 * time.Minute), + expectedRemaining: 2, + }, + { + name: "all", + maxScheduledTime: now.Add(5 * time.Minute), + expectedRemaining: 0, + }, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + s.mutableState.chasmPureTasks = []*tasks.ChasmTaskPure{ + { + VisibilityTimestamp: now.Add(3 * time.Minute), + }, + { + VisibilityTimestamp: now.Add(2 * time.Minute), + }, + { + VisibilityTimestamp: now.Add(time.Minute), + }, + } + s.mutableState.BestEffortDeleteTasks = make(map[tasks.Category][]tasks.Key) + + s.mutableState.DeleteCHASMPureTasks(tc.maxScheduledTime) + + s.Len(s.mutableState.chasmPureTasks, tc.expectedRemaining) + for _, task := range s.mutableState.chasmPureTasks { + s.False(task.VisibilityTimestamp.Before(tc.maxScheduledTime)) + } + + s.Len(s.mutableState.BestEffortDeleteTasks[tasks.CategoryTimer], 3-tc.expectedRemaining) + }) + } +} diff --git a/service/matching/forwarder.go b/service/matching/forwarder.go index f014f5ee9c0..c74edb363e7 100644 --- a/service/matching/forwarder.go +++ b/service/matching/forwarder.go @@ -147,9 +147,9 @@ func (fwdr *Forwarder) ForwardTask(ctx context.Context, task *internalTask) erro Clock: task.event.Data.GetClock(), ScheduleToStartTimeout: expirationDuration, ForwardInfo: fwdr.getForwardInfo(task), + VersionDirective: task.event.Data.GetVersionDirective(), Stamp: task.event.Data.GetStamp(), Priority: task.event.Data.GetPriority(), - VersionDirective: task.event.Data.GetVersionDirective(), }, ) default: diff --git a/service/matching/pri_forwarder.go b/service/matching/pri_forwarder.go index 7d433b7c853..186a1c005e1 100644 --- a/service/matching/pri_forwarder.go +++ b/service/matching/pri_forwarder.go @@ -90,7 +90,6 @@ func (f *priForwarder) ForwardTask(ctx context.Context, task *internalTask) erro VersionDirective: task.event.Data.GetVersionDirective(), Stamp: task.event.Data.GetStamp(), Priority: task.event.Data.GetPriority(), - Stamp: task.event.Data.GetStamp(), }, ) case enumspb.TASK_QUEUE_TYPE_ACTIVITY: @@ -106,8 +105,8 @@ func (f *priForwarder) ForwardTask(ctx context.Context, task *internalTask) erro Clock: task.event.Data.GetClock(), ScheduleToStartTimeout: expirationDuration, ForwardInfo: f.getForwardInfo(task), - Stamp: task.event.Data.GetStamp(), VersionDirective: task.event.Data.GetVersionDirective(), + Stamp: task.event.Data.GetStamp(), Priority: task.event.Data.GetPriority(), }, ) diff --git a/tests/priority_fairness_test.go b/tests/priority_fairness_test.go index cef96c13ca7..e274914dfb3 100644 --- a/tests/priority_fairness_test.go +++ b/tests/priority_fairness_test.go @@ -9,14 +9,17 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" activitypb "go.temporal.io/api/activity/v1" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/adminservice/v1" + taskqueuespb "go.temporal.io/server/api/taskqueue/v1" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/payloads" @@ -70,7 +73,7 @@ func (s *PrioritySuite) TestPriority_Activity_Basic() { _, err := s.TaskPoller().PollAndHandleWorkflowTask( tv, func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.Equal(3, len(task.History.Events)) + s.Len(task.History.Events, 3) var wfidx int _, err := fmt.Sscanf(task.WorkflowExecution.WorkflowId, "wf%d", &wfidx) @@ -79,8 +82,13 @@ func (s *PrioritySuite) TestPriority_Activity_Basic() { var commands []*commandpb.Command for i, pri := range rand.Perm(Levels) { - input, err := payloads.Encode(wfidx, pri+1) + pri += 1 // 1-based + input, err := payloads.Encode(wfidx, pri) s.NoError(err) + priMsg := &commonpb.Priority{PriorityKey: int32(pri)} + if pri == (Levels+1)/2 { + priMsg = nil // nil should be treated as default (3) + } commands = append(commands, &commandpb.Command{ CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ @@ -89,10 +97,8 @@ func (s *PrioritySuite) TestPriority_Activity_Basic() { ActivityType: tv.ActivityType(), TaskQueue: tv.TaskQueue(), ScheduleToCloseTimeout: durationpb.New(time.Minute), - Priority: &commonpb.Priority{ - PriorityKey: int32(pri + 1), - }, - Input: input, + Priority: priMsg, + Input: input, }, }, }) @@ -126,7 +132,7 @@ func (s *PrioritySuite) TestPriority_Activity_Basic() { w := wrongorderness(runs) s.T().Log("wrongorderness:", w) - s.Less(w, 0.15) + s.Less(w, 0.1) } func (s *PrioritySuite) TestSubqueue_Migration() { @@ -154,7 +160,7 @@ func (s *PrioritySuite) TestSubqueue_Migration() { _, err := s.TaskPoller().PollAndHandleWorkflowTask( tv, func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.Equal(3, len(task.History.Events)) + s.Len(task.History.Events, 3) var commands []*commandpb.Command @@ -234,6 +240,7 @@ func wrongorderness(vs []int) float64 { type FairnessSuite struct { testcore.FunctionalTestBase + partitions int } func TestFairnessSuite(t *testing.T) { @@ -242,14 +249,16 @@ func TestFairnessSuite(t *testing.T) { } func (s *FairnessSuite) SetupSuite() { + s.partitions = 1 dynamicConfigOverrides := map[dynamicconfig.Key]any{ + dynamicconfig.MatchingUseNewMatcher.Key(): true, dynamicconfig.MatchingEnableFairness.Key(): true, dynamicconfig.MatchingGetTasksBatchSize.Key(): 20, dynamicconfig.MatchingGetTasksReloadAt.Key(): 5, dynamicconfig.NumPendingActivitiesLimitError.Key(): 1000, - // TODO: disable this later? - dynamicconfig.MatchingNumTaskqueueReadPartitions.Key(): 1, - dynamicconfig.MatchingNumTaskqueueWritePartitions.Key(): 1, + // TODO: disable this and use default later? + dynamicconfig.MatchingNumTaskqueueReadPartitions.Key(): s.partitions, + dynamicconfig.MatchingNumTaskqueueWritePartitions.Key(): s.partitions, } s.FunctionalTestBase.SetupSuiteWithCluster(testcore.WithDynamicConfigOverrides(dynamicConfigOverrides)) } @@ -281,7 +290,7 @@ func (s *FairnessSuite) TestFairness_Activity_Basic() { _, err := s.TaskPoller().PollAndHandleWorkflowTask( tv, func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { - s.Equal(3, len(task.History.Events)) + s.Len(task.History.Events, 3) var wfidx int _, err := fmt.Sscanf(task.WorkflowExecution.WorkflowId, "wf%d", &wfidx) @@ -355,6 +364,208 @@ func unfairness(vs []int) float64 { return float64(totalDelay) / float64(len(firsts)*len(firsts)) } +func (s *FairnessSuite) testMigration(newMatcher, fairness bool) { + tv := testvars.New(s.T()) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + s.OverrideDynamicConfig(dynamicconfig.MatchingEnableMigration, true) + + forTest := func(v any) any { + return []dynamicconfig.ConstrainedValue{ + // test tqs (both wf and activity) + dynamicconfig.ConstrainedValue{ + Constraints: dynamicconfig.Constraints{ + Namespace: s.Namespace().String(), + TaskQueueName: tv.TaskQueue().Name, + }, + Value: v, + }, + // default (match values in SetupSuite to avoid flapping) + dynamicconfig.ConstrainedValue{Value: true}, + } + } + setConfig := func(stage string, newNewMatcher, newFairness bool) { + newMatcher, fairness = newNewMatcher, newFairness + s.T().Log("setting config: "+stage, "newMatcher", newMatcher, "fairness", fairness) + s.OverrideDynamicConfig(dynamicconfig.MatchingUseNewMatcher, forTest(newMatcher)) + s.OverrideDynamicConfig(dynamicconfig.MatchingEnableFairness, forTest(fairness)) + } + waitForTasks := func(tp enumspb.TaskQueueType, onDraining, onActive int64) { + s.T().Helper() + s.EventuallyWithT(func(c *assert.CollectT) { + tasksOnDraining, tasksOnActive, err := s.countTasksByDrainingActive(ctx, tv, tp) + require.NoError(c, err) + require.Equal(c, onDraining, tasksOnDraining) + require.Equal(c, onActive, tasksOnActive) + }, 15*time.Second, 250*time.Millisecond) + } + + setConfig("initial", newMatcher, fairness) + + // start 20 workflows. 20 tasks will be queued on wft queue. + s.T().Log("starting workflows") + for range 20 { + _, err := s.FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), + WorkflowId: uuid.NewString(), + WorkflowType: tv.WorkflowType(), + TaskQueue: tv.TaskQueue(), + }) + s.NoError(err) + } + waitForTasks(enumspb.TASK_QUEUE_TYPE_WORKFLOW, 0, 20) + + processWft := func() { + s.EventuallyWithT(func(c *assert.CollectT) { + _, err := s.TaskPoller().PollAndHandleWorkflowTask( + tv, + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.Len(task.History.Events, 3) + + var commands []*commandpb.Command + + for i := range 2 { + input, err := payloads.Encode(i) + s.NoError(err) + commands = append(commands, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ + ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: fmt.Sprintf("act%d", i), + ActivityType: tv.ActivityType(), + TaskQueue: tv.TaskQueue(), + ScheduleToCloseTimeout: durationpb.New(time.Minute), + Input: input, + }, + }, + }) + } + + return &workflowservice.RespondWorkflowTaskCompletedRequest{Commands: commands}, nil + }, + taskpoller.WithContext(ctx), + ) + assert.NoError(c, err) + }, 5*time.Second, time.Millisecond) + } + + // process half the workflow tasks and create two activities each. + // 10 tasks will be left on old workflow queue, 20 tasks will be queued on current activity queue. + s.T().Log("processing first half of wfts") + for range 10 { + processWft() + } + waitForTasks(enumspb.TASK_QUEUE_TYPE_WORKFLOW, 0, 10) + waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 0, 20) + + // switch fairness. queues will be reloaded. wft queue should drain old queue. + setConfig("switching fairness", true, !fairness) + + waitForTasks(enumspb.TASK_QUEUE_TYPE_WORKFLOW, 10, 0) + waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 20, 0) + + // process the other half of workflow tasks. these should come from the draining queue now. + // 20 tasks will be queued on new activity queue (still 20 on old). + s.T().Log("processing last half of wfts") + for range 5 { + processWft() + } + waitForTasks(enumspb.TASK_QUEUE_TYPE_WORKFLOW, 5, 0) + waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 20, 10) + for range 5 { + processWft() + } + waitForTasks(enumspb.TASK_QUEUE_TYPE_WORKFLOW, 0, 0) + waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 20, 20) + s.T().Log("wfts done") + + // process activities 1/3 at a time + processActivity := func() { + s.EventuallyWithT(func(c *assert.CollectT) { + _, err := s.TaskPoller().PollAndHandleActivityTask( + tv, + func(task *workflowservice.PollActivityTaskQueueResponse) (*workflowservice.RespondActivityTaskCompletedRequest, error) { + nothing, err := payloads.Encode() + s.NoError(err) + return &workflowservice.RespondActivityTaskCompletedRequest{Result: nothing}, nil + }, + taskpoller.WithContext(ctx), + ) + assert.NoError(c, err) + }, 5*time.Second, time.Millisecond) + } + + s.T().Log("processing first 1/3 activities") + for range 13 { + processActivity() + } + waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 7, 20) + + setConfig("switching fairness again", true, !fairness) + waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 20, 7) + + s.T().Log("processing next 1/3 activities") + for range 14 { + processActivity() + } + waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 6, 7) + + setConfig("switching fairness last time", true, !fairness) + waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 7, 6) + + s.T().Log("processing last 1/3 activities") + for range 13 { + processActivity() + } + waitForTasks(enumspb.TASK_QUEUE_TYPE_ACTIVITY, 0, 0) +} + +func (s *FairnessSuite) countTasksByDrainingActive(ctx context.Context, tv *testvars.TestVars, tp enumspb.TaskQueueType) ( + tasksOnDraining, tasksOnActive int64, retErr error, +) { + for i := range s.partitions { + res, err := s.AdminClient().DescribeTaskQueuePartition(ctx, &adminservice.DescribeTaskQueuePartitionRequest{ + Namespace: s.Namespace().String(), + TaskQueuePartition: &taskqueuespb.TaskQueuePartition{ + TaskQueue: tv.TaskQueue().Name, + TaskQueueType: tp, + PartitionId: &taskqueuespb.TaskQueuePartition_NormalPartitionId{NormalPartitionId: int32(i)}, + }, + BuildIds: &taskqueuepb.TaskQueueVersionSelection{Unversioned: true}, + }) + if err != nil { + return 0, 0, err + } + for _, versionInfoInternal := range res.VersionsInfoInternal { + for _, st := range versionInfoInternal.PhysicalTaskQueueInfo.InternalTaskQueueStatus { + if st.Draining { + tasksOnDraining += st.ApproximateBacklogCount + } else { + tasksOnActive += st.ApproximateBacklogCount + } + } + } + } + return +} + +func (s *FairnessSuite) TestFairness_Migration_FromClassic() { + // classic->fair, fair->pri. fair metadata will be created on transition. + s.testMigration(false, false) +} + +func (s *FairnessSuite) TestFairness_Migration_FromPri() { + // pri->fair, fair->pri. fair metadata will be created before transition. + s.testMigration(true, false) +} + +func (s *FairnessSuite) TestFairness_Migration_FromFair() { + // fair->pri, pri->fair. fair metadata will be created first. + s.testMigration(true, true) +} + func (s *FairnessSuite) TestFairness_UpdateWorkflowExecutionOptions_InvalidatesPendingTask() { tv := testvars.New(s.T())