Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions stovepipe/controller/issuetrack/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ go_test(
"fixture_test.go",
"history_test.go",
"issuetrack_test.go",
"request_issue_summary_test.go",
],
embed = [":go_default_library"],
deps = [
Expand Down
8 changes: 8 additions & 0 deletions stovepipe/controller/issuetrack/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,11 @@ Each validation uses the single job registered at admission; analysis does not p
Provision the non-null `request.build_id` column with an empty default before deploying. This is an additive association; existing terminal requests can still use their winning build ID. No new tables, secondary indexes, or cross-key transactions are required.

Pause admission and drain pre-upgrade validations, completing or explicitly abandoning the existing tracking checkpoint before enabling request-scoped registration. Upgrade normal and recovery consumers together and use the single-head tracking subscription before resuming admission. Old per-analysis notifications are not supported. Do not mix old publishers or consumers with the request-scoped flow. To revert, pause admission and finish or safely abandon request-scoped jobs before restoring the old consumers; the added column can remain.

## Per-request issue summaries

The existing issue log is the source for both queue-level lifecycle projections and request-level summaries. Opening and closing events produce New and Closed request results. A reported event records an already-open issue encountered by a later build and produces Existing; it does not change issue status, transition time, or the open-issue lookup. Each event retains its opening build reference, and the materializer loads the source build to identify the request. Request-wide completion and abandonment events remain in request history.

The cursor cannot advance until the accepted event and all its projections are durable. Retry repairs missing projections from the retained event, preserving the original timestamp and opening. A closure wins over an opening or report from the same request, with conditional writes preventing an older replay from changing it back. Recovery repairs lifecycle history and any accepted report before abandonment; merely fencing an older row does not attribute a report to the abandoned request.

Existing lifecycle events can lack the additive opening reference. Their lifecycle projections continue using the supplied opening, but those records do not create request summaries from information absent from retained history. Requests begun before recording was deployed have no coverage marker and are not backfilled. Deploy the opening-reference column before updated writers; recording coverage assumes every worker uses the updated contract.
8 changes: 6 additions & 2 deletions stovepipe/controller/issuetrack/delivery_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ func TestFinalizationOpensKeepsAndClosesIssues(t *testing.T) {
require.Equal(t, int32(8), f.states[issueKey{"b", "unit"}].Version)
require.Equal(t, "build", f.states[issueKey{"c", "unit"}].FirstSeenBuildID)
require.Equal(t, int32(8), f.states[issueKey{"closed", ""}].Version)
require.Len(t, f.logs, 2)
require.Len(t, f.logs, 3)
require.Contains(t, f.logs, logKey{issueKey{"b", "unit"}, "build", entity.IssueEventReported})
require.Contains(t, f.logs, logKey{issueKey{"a", ""}, "build", entity.IssueEventClosed})
require.Contains(t, f.logs, logKey{issueKey{"c", "unit"}, "build", entity.IssueEventOpened})
}
Expand Down Expand Up @@ -535,7 +536,10 @@ func TestTaskWideFlagSurvivesNoOpOpeningBatchAndRetry(t *testing.T) {
require.True(t, f.progress.Finalize)
require.True(t, f.progress.HasTaskWideIssues)
require.Equal(t, "unit/099", f.progress.CursorUnitID)
require.Empty(t, f.logs)
require.Len(t, f.logs, 100)
for _, log := range f.logs {
require.Equal(t, entity.IssueEventReported, log.Event)
}
f.failures["build-issues/list"] = 1
require.ErrorIs(t, f.deliver(msg, false), assert.AnError)
require.True(t, f.progress.HasTaskWideIssues)
Expand Down
3 changes: 3 additions & 0 deletions stovepipe/controller/issuetrack/dlq.go
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,9 @@ func (c *DLQController) finishAbandonment(ctx context.Context, store storage.Sto
if err := persistIssueHistory(ctx, c.materializer, store, current); err != nil {
return entity.IssueTrackingState{}, err
}
if err := persistReportedIssue(ctx, c.materializer, store, progress, current); err != nil {
return entity.IssueTrackingState{}, err
}
}

if err := c.recordAbandonment(ctx, store, queue, progress); err != nil {
Expand Down
41 changes: 41 additions & 0 deletions stovepipe/controller/issuetrack/fixture_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ type logKey struct {

type fixture struct {
requestHistory map[string]entity.RequestLog
requestIssues map[string]map[issueKey]entity.RequestIssueSummary
materializer requestlog.Materializer
failOnPhase entity.IssueTrackingPhase
t *testing.T
Expand Down Expand Up @@ -72,6 +73,7 @@ func newFixture(t *testing.T) *fixture {
t.Helper()
f := &fixture{t: t, mocks: gomock.NewController(t), states: map[issueKey]entity.IssueState{}, logs: map[logKey]entity.IssueLog{}, failures: map[string]int{}, initialLogs: map[logKey]entity.IssueLog{}}
f.factory = storagemock.NewMockFactory(f.mocks)
f.requestIssues = map[string]map[issueKey]entity.RequestIssueSummary{}
f.builds = map[string]entity.Build{}
f.requests = map[string]entity.Request{}
f.buildIssues = map[string]map[issueKey]entity.BuildIssue{}
Expand All @@ -84,6 +86,45 @@ func newFixture(t *testing.T) *fixture {
history := storagemock.NewMockIssueLogStore(f.mocks)
summaries := storagemock.NewMockIssueSummaryStore(f.mocks)
openIssues := storagemock.NewMockOpenIssueStore(f.mocks)
requestIssues := storagemock.NewMockRequestIssueSummaryStore(f.mocks)
store.EXPECT().GetRequestIssueSummaryStore().Return(requestIssues).AnyTimes()
requestIssues.EXPECT().Create(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, issue entity.RequestIssueSummary) error {
if err := f.fail("request-issue/create"); err != nil {
return err
}
if f.requestIssues[issue.RequestID] == nil {
f.requestIssues[issue.RequestID] = map[issueKey]entity.RequestIssueSummary{}
}
key := issueKey{issue.TaskID, issue.UnitID}
if _, ok := f.requestIssues[issue.RequestID][key]; ok {
return storage.ErrAlreadyExists
}
f.requestIssues[issue.RequestID][key] = issue
return nil
}).AnyTimes()
requestIssues.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, request, task, unit string) (entity.RequestIssueSummary, error) {
if err := f.fail("request-issue/get"); err != nil {
return entity.RequestIssueSummary{}, err
}
issue, ok := f.requestIssues[request][issueKey{task, unit}]
if !ok {
return entity.RequestIssueSummary{}, storage.ErrNotFound
}
return issue, nil
}).AnyTimes()
requestIssues.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, issue entity.RequestIssueSummary, oldVersion, newVersion int32) error {
if err := f.fail("request-issue/update"); err != nil {
return err
}
key := issueKey{issue.TaskID, issue.UnitID}
current, ok := f.requestIssues[issue.RequestID][key]
if !ok || current.Version != oldVersion {
return storage.ErrVersionMismatch
}
issue.Version = newVersion
f.requestIssues[issue.RequestID][key] = issue
return nil
}).AnyTimes()
tracking := storagemock.NewMockIssueTrackingStore(f.mocks)
f.factory.EXPECT().For(storage.Config{QueueName: testQueue}).Return(store, nil).AnyTimes()
store.EXPECT().GetIssueStateStore().Return(states).AnyTimes()
Expand Down
19 changes: 19 additions & 0 deletions stovepipe/controller/issuetrack/history.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,3 +82,22 @@ func persistIssueHistory(ctx context.Context, materializer issuelog.Materializer
}
return nil
}

// persistReportedIssue records only reports accepted for this pass, not older rows fenced by recovery.
func persistReportedIssue(ctx context.Context, materializer issuelog.Materializer, store storage.Storage, progress entity.IssueTrackingState, issue entity.IssueState) error {
if issue.LastProcessedBuildID != progress.BuildID {
return nil
}
if issue.Status != entity.IssueStatusOpen {
return nil
}
if issue.FirstSeenBuildID == progress.BuildID {
return nil
}
log := entity.IssueLog{TaskID: issue.TaskID, UnitID: issue.UnitID, Event: entity.IssueEventReported,
BuildID: progress.BuildID, FirstSeenBuildID: issue.FirstSeenBuildID}
if err := materializer.PersistLog(ctx, store, log, issue.FirstSeenBuildID); err != nil {
return fmt.Errorf("record reported issue: %w", err)
}
return nil
}
6 changes: 3 additions & 3 deletions stovepipe/controller/issuetrack/history_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func TestReconciliationHistoryClosureModes(t *testing.T) {
if mode == "reconciled" {
require.NoError(t, f.deliver(msg, false))
}
require.Len(t, f.requestHistory, 2)
require.Len(t, f.requestHistory, 3)
log := f.requestHistory["10:event/issues_reconciled/build"]
require.Equal(t, entity.RequestEventIssuesReconciled, log.Event)
require.Equal(t, mode, log.Metadata[requestlog.MetadataKeyClosureMode])
Expand Down Expand Up @@ -91,7 +91,7 @@ func TestCompletedReconciliationRepairsHistoryBeforeReplacingCheckpoint(t *testi
require.NoError(t, f.deliver(newer, false))
require.Equal(t, "new", f.progress.BuildID)
require.Contains(t, f.requestHistory, "10:event/issues_reconciled/old")
require.Len(t, f.requestHistory, 2) // Only the older validation has completion and reconciliation events.
require.Len(t, f.requestHistory, 4)
}

func TestIssueTrackingDLQRepairsCompletedPassInsteadOfAbandoning(t *testing.T) {
Expand All @@ -105,6 +105,6 @@ func TestIssueTrackingDLQRepairsCompletedPassInsteadOfAbandoning(t *testing.T) {
checkpoint := f.progress
require.NoError(t, f.recoverHistory(msg, failure.Failure{}))
require.Equal(t, checkpoint, f.progress)
require.Len(t, f.requestHistory, 2)
require.Len(t, f.requestHistory, 3)
require.Contains(t, f.requestHistory, "10:event/issues_reconciled/build")
}
18 changes: 14 additions & 4 deletions stovepipe/controller/issuetrack/issuetrack.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,12 @@ func (c *Controller) trackRequest(ctx context.Context, delivery consumer.Deliver
Finalize: finalize,
Version: progress.Version + 1,
}
if progress.RequestID != request.ID {
log := requestlog.NewRequestEventLog(request, entity.RequestEventIssueTrackingStarted, "", nil)
if err := c.requestMaterializer.PersistLog(ctx, store, log); err != nil {
return fmt.Errorf("record start of request issue results: %w", err)
}
}
if progress.Version == 0 {
err = tracking.Create(ctx, next)
} else {
Expand Down Expand Up @@ -411,9 +417,11 @@ func (c *Controller) processIssue(ctx context.Context, store storage.Storage, pr
next.CursorTaskID, next.CursorUnitID = key.task, key.unit
if !missing && current.LastProcessedBuildID == progress.BuildID && current.Status == desired {
if key == (issueKey{progress.CursorTaskID, progress.CursorUnitID}) {
return next, persistIssueHistory(ctx, c.materializer, store, current)
if err := persistIssueHistory(ctx, c.materializer, store, current); err != nil {
return entity.IssueTrackingState{}, err
}
}
return next, nil
return next, persistReportedIssue(ctx, c.materializer, store, progress, current)
}

// Read the issue before this CAS, including when resuming a cursor. A
Expand Down Expand Up @@ -460,9 +468,11 @@ func (c *Controller) processIssue(ctx context.Context, store storage.Storage, pr
}

if missing || current.Status != desired {
return next, persistIssueHistory(ctx, c.materializer, store, target)
if err := persistIssueHistory(ctx, c.materializer, store, target); err != nil {
return entity.IssueTrackingState{}, err
}
}
return next, nil
return next, persistReportedIssue(ctx, c.materializer, store, progress, target)
}

func saveProgress(ctx context.Context, store storage.IssueTrackingStore, next entity.IssueTrackingState) (entity.IssueTrackingState, error) {
Expand Down
4 changes: 3 additions & 1 deletion stovepipe/controller/issuetrack/issuetrack_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,9 @@ func TestUnchangedIssueRecordsTheProcessedBuild(t *testing.T) {
require.Equal(t, "older", issue.FirstSeenBuildID)
require.Equal(t, int64(500), issue.ChangedAtMs)
require.Equal(t, int32(8), issue.Version)
require.Empty(t, f.logs)
require.Len(t, f.logs, 1)
require.Contains(t, f.logs, logKey{issueKey{"task", "unit"}, "build/100", entity.IssueEventReported})
require.Equal(t, "older", f.summaries[issueKey{"task", "unit"}].BuildID)
}

func TestHistoryRetryUsesTheStoredTransition(t *testing.T) {
Expand Down
Loading
Loading