From cd34bba7bb3bdc7d0dba46d024c3f590ac81e250 Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 04:46:04 -0700 Subject: [PATCH 01/20] feat(api, migrations): add delivery config support for jobs and add request/response types for scheduled jobs - Added `delivery_type` (TEXT) and `delivery_config` (JSONB) columns to `jobs` table with appropriate defaults through migrations. - Introduced `CreateScheduleRequest` struct for validation of delivery configuration and scheduling requests. - Updated `job_store` to handle new fields in job creation logic. - Adjusted `.gitignore` and `.dockerignore` to exclude `.tmp` directory. --- .dockerignore | 2 + .gitignore | 2 + internal/api/http/types/schedule.go | 68 +++++++++++++++++++ internal/store/postgres/job_store.go | 18 +++-- .../0018_add_delivery_config_to_jobs.down.sql | 3 + .../0018_add_delivery_config_to_jobs.up.sql | 3 + 6 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 internal/api/http/types/schedule.go create mode 100644 migrations/postgres/0018_add_delivery_config_to_jobs.down.sql create mode 100644 migrations/postgres/0018_add_delivery_config_to_jobs.up.sql diff --git a/.dockerignore b/.dockerignore index e47ef46..900203c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,3 +21,5 @@ AGENTS.md CLAUDE.md GEMINI.md Makefile + +/.tmp/ diff --git a/.gitignore b/.gitignore index 4ccb792..6566944 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ profile.cov vendor/ go.work go.work.sum +/.tmp/ # Built binaries bin/ @@ -52,3 +53,4 @@ tilt.log /CLAUDE.md /GEMINI.md /AGENTS.md + diff --git a/internal/api/http/types/schedule.go b/internal/api/http/types/schedule.go new file mode 100644 index 0000000..ca496e7 --- /dev/null +++ b/internal/api/http/types/schedule.go @@ -0,0 +1,68 @@ +package types + +import ( + "encoding/json" + "errors" + "time" +) + +var ( + ErrScheduledAtMustBeInFuture = errors.New("scheduled_at must be in the future") + ErrJobDefinitionOrAdhocConfigNeeded = errors.New("either job_definition_id or ad-hoc delivery config (delivery_mode and endpoint) must be provided") +) + +type CreateScheduleRequest struct { + JobDefinitionID string `json:"job_definition_id,omitempty"` + ScheduledAt time.Time `json:"scheduled_at" binding:"required"` + Payload json.RawMessage `json:"payload"` + IdempotencyKey string `json:"idempotency_key"` + + // Ad-hoc inline fields + DeliveryMode string `json:"delivery_mode,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + HTTPMethod string `json:"http_method,omitempty"` + HTTPHeaders map[string]string `json:"http_headers,omitempty"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` + RetryPolicy RetryPolicyInput `json:"retry_policy,omitempty"` + ExpectedDurationSeconds int `json:"expected_duration_seconds,omitempty"` + AuthSecretRef string `json:"auth_secret_ref,omitempty"` +} + +func (r CreateScheduleRequest) Validate(now time.Time) error { + if !r.ScheduledAt.After(now) { + return ErrScheduledAtMustBeInFuture + } + + // Validate that we have either a definition ID or the minimum required for an ad-hoc job + if r.JobDefinitionID == "" { + if r.DeliveryMode == "" || r.Endpoint == "" { + return ErrJobDefinitionOrAdhocConfigNeeded + } + } + + return nil +} + +type ScheduleExecutionSummary struct { + ID string `json:"id"` + AttemptNumber int `json:"attempt_number"` + Status string `json:"status"` + ResponseStatus *int `json:"response_status,omitempty"` + DurationMS *int `json:"duration_ms,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + StartedAt string `json:"started_at"` + CompletedAt string `json:"completed_at,omitempty"` +} + +type ScheduleResponse struct { + ID string `json:"id"` + JobType string `json:"job_type"` + JobDefinitionID string `json:"job_definition_id,omitempty"` + BucketID string `json:"bucket_id,omitempty"` + ScheduledAt string `json:"scheduled_at"` + Status string `json:"status"` + IdempotencyKey string `json:"idempotency_key,omitempty"` + Payload json.RawMessage `json:"payload,omitempty"` + CreatedAt string `json:"created_at"` + Executions []ScheduleExecutionSummary `json:"executions,omitempty"` +} diff --git a/internal/store/postgres/job_store.go b/internal/store/postgres/job_store.go index 779532f..324008d 100644 --- a/internal/store/postgres/job_store.go +++ b/internal/store/postgres/job_store.go @@ -14,11 +14,13 @@ type Job struct { CronScheduleID *string `db:"cron_schedule_id"` // non-null for CRON BulkJobID *string `db:"bulk_job_id"` // non-null for BULK_RECORD BucketID *string `db:"bucket_id"` // non-null for ONE_TIME + CRON - RowStart *int `db:"row_start"` // non-null for BULK_RECORD: first row in batch (0-indexed) - RowEnd *int `db:"row_end"` // non-null for BULK_RECORD: last row in batch (inclusive); == RowStart when batch_size=1 - Payload []byte `db:"payload"` // JSONB — resolved request body; array when batch_size > 1 - Context []byte `db:"context"` // JSONB — runtime vars for delivery_config substitution - ScheduledAt *time.Time `db:"scheduled_at"` // non-null for ONE_TIME + CRON + DeliveryType string `db:"delivery_type"` + DeliveryConfig []byte `db:"delivery_config"` + RowStart *int `db:"row_start"` // non-null for BULK_RECORD: first row in batch (0-indexed) + RowEnd *int `db:"row_end"` // non-null for BULK_RECORD: last row in batch (inclusive); == RowStart when batch_size=1 + Payload []byte `db:"payload"` // JSONB — resolved request body; array when batch_size > 1 + Context []byte `db:"context"` // JSONB — runtime vars for delivery_config substitution + ScheduledAt *time.Time `db:"scheduled_at"` // non-null for ONE_TIME + CRON Status string `db:"status"` IdempotencyKey *string `db:"idempotency_key"` CreatedAt time.Time `db:"created_at"` @@ -36,10 +38,12 @@ func (s *JobStore) Create(ctx context.Context, j *Job) error { const q = ` INSERT INTO jobs (job_type, job_def_id, cron_schedule_id, bulk_job_id, bucket_id, - row_start, row_end, payload, context, scheduled_at, status, idempotency_key) + delivery_type, delivery_config, row_start, row_end, payload, context, + scheduled_at, status, idempotency_key) VALUES (:job_type, :job_def_id, :cron_schedule_id, :bulk_job_id, :bucket_id, - :row_start, :row_end, :payload, :context, :scheduled_at, :status, :idempotency_key) + :delivery_type, :delivery_config, :row_start, :row_end, :payload, :context, + :scheduled_at, :status, :idempotency_key) RETURNING id, created_at` rows, err := s.db.NamedQueryContext(ctx, q, j) if err != nil { diff --git a/migrations/postgres/0018_add_delivery_config_to_jobs.down.sql b/migrations/postgres/0018_add_delivery_config_to_jobs.down.sql new file mode 100644 index 0000000..8a1e6f7 --- /dev/null +++ b/migrations/postgres/0018_add_delivery_config_to_jobs.down.sql @@ -0,0 +1,3 @@ +-- 0018_add_delivery_config_to_jobs.down.sql +ALTER TABLE jobs DROP COLUMN IF EXISTS delivery_type; +ALTER TABLE jobs DROP COLUMN IF EXISTS delivery_config; diff --git a/migrations/postgres/0018_add_delivery_config_to_jobs.up.sql b/migrations/postgres/0018_add_delivery_config_to_jobs.up.sql new file mode 100644 index 0000000..f343679 --- /dev/null +++ b/migrations/postgres/0018_add_delivery_config_to_jobs.up.sql @@ -0,0 +1,3 @@ +-- 0018_add_delivery_config_to_jobs.up.sql +ALTER TABLE jobs ADD COLUMN delivery_type TEXT NOT NULL DEFAULT 'HTTP'; +ALTER TABLE jobs ADD COLUMN delivery_config JSONB NOT NULL DEFAULT '{}'; From 1d91b1f4aec48a0caf6d714d2eb715fb7fbab33b Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 04:56:38 -0700 Subject: [PATCH 02/20] feat(api): implement one-time schedules API handlers with delivery config resolution --- .../api/http/handlers/schedule_handler.go | 304 ++++++++++++++++++ internal/api/http/router.go | 9 + internal/api/http/types/schedule.go | 5 +- internal/store/postgres/job_store.go | 75 ++++- 4 files changed, 390 insertions(+), 3 deletions(-) create mode 100644 internal/api/http/handlers/schedule_handler.go diff --git a/internal/api/http/handlers/schedule_handler.go b/internal/api/http/handlers/schedule_handler.go new file mode 100644 index 0000000..9e92ac1 --- /dev/null +++ b/internal/api/http/handlers/schedule_handler.go @@ -0,0 +1,304 @@ +package handlers + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/lib/pq" + + "github.com/chronos-scheduler/chronos/internal/api/http/response" + apitypes "github.com/chronos-scheduler/chronos/internal/api/http/types" + pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" +) + +type ScheduleHandler struct { + jobStore *pgstore.JobStore + bucketStore *pgstore.ScheduleBucketStore + jobDefStore *pgstore.JobDefinitionStore + executionStore *pgstore.JobExecutionStore +} + +func NewScheduleHandler( + jobStore *pgstore.JobStore, + bucketStore *pgstore.ScheduleBucketStore, + jobDefStore *pgstore.JobDefinitionStore, + executionStore *pgstore.JobExecutionStore, +) *ScheduleHandler { + return &ScheduleHandler{ + jobStore: jobStore, + bucketStore: bucketStore, + jobDefStore: jobDefStore, + executionStore: executionStore, + } +} + +func (h *ScheduleHandler) CreateSchedule(c *gin.Context) { + var req apitypes.CreateScheduleRequest + if err := c.ShouldBindJSON(&req); err != nil { + response.RespondBadRequest(c, "invalid request payload") + return + } + + if err := req.Validate(time.Now().UTC()); err != nil { + response.RespondBadRequest(c, err.Error()) + return + } + + deliveryType, deliveryConfig, jobDefID, err := h.resolveDelivery(c.Request.Context(), req) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + response.RespondNotFound(c, "job definition not found") + return + } + response.RespondBadRequest(c, err.Error()) + return + } + + bucketFireAt := req.ScheduledAt.UTC().Truncate(time.Minute) + bucket := &pgstore.ScheduleBucket{FireAt: bucketFireAt} + if err := h.bucketStore.Upsert(c.Request.Context(), bucket); err != nil { + response.RespondInternalError(c, "failed to create schedule bucket") + return + } + + payload := []byte(`{}`) + if len(req.Payload) > 0 { + payload = req.Payload + } + contextPayload := []byte(`{}`) + + job := &pgstore.Job{ + JobType: "ONE_TIME", + JobDefID: jobDefID, + BucketID: &bucket.ID, + DeliveryType: deliveryType, + DeliveryConfig: deliveryConfig, + Payload: payload, + Context: contextPayload, + ScheduledAt: ptrTime(req.ScheduledAt.UTC()), + Status: "PENDING", + } + if strings.TrimSpace(req.IdempotencyKey) != "" { + key := strings.TrimSpace(req.IdempotencyKey) + job.IdempotencyKey = &key + } + + if err := h.jobStore.Create(c.Request.Context(), job); err != nil { + if pqErr, ok := err.(*pq.Error); ok && pqErr.Code == "23505" && job.IdempotencyKey != nil { + existing, getErr := h.jobStore.GetByIdempotencyKey(c.Request.Context(), *job.IdempotencyKey) + if getErr != nil { + response.RespondInternalError(c, "failed to resolve existing idempotent schedule") + return + } + c.JSON(http.StatusOK, toScheduleResponse(existing, nil)) + return + } + response.RespondInternalError(c, "failed to create schedule") + return + } + + c.JSON(http.StatusCreated, toScheduleResponse(job, nil)) +} + +func (h *ScheduleHandler) ListSchedules(c *gin.Context) { + limit := 50 + offset := 0 + if s := strings.TrimSpace(c.Query("limit")); s != "" { + n, err := strconv.Atoi(s) + if err != nil || n <= 0 { + response.RespondBadRequest(c, "invalid limit") + return + } + limit = n + } + if s := strings.TrimSpace(c.Query("offset")); s != "" { + n, err := strconv.Atoi(s) + if err != nil || n < 0 { + response.RespondBadRequest(c, "invalid offset") + return + } + offset = n + } + + filter := pgstore.ListJobsFilter{ + JobType: "ONE_TIME", + Status: strings.TrimSpace(c.Query("status")), + Limit: limit, + Offset: offset, + } + if from := strings.TrimSpace(c.Query("date_from")); from != "" { + t, err := time.Parse(time.RFC3339, from) + if err != nil { + response.RespondBadRequest(c, "invalid date_from; expected RFC3339") + return + } + filter.ScheduledFrom = &t + } + if to := strings.TrimSpace(c.Query("date_to")); to != "" { + t, err := time.Parse(time.RFC3339, to) + if err != nil { + response.RespondBadRequest(c, "invalid date_to; expected RFC3339") + return + } + filter.ScheduledTo = &t + } + + jobs, err := h.jobStore.List(c.Request.Context(), filter) + if err != nil { + response.RespondInternalError(c, "failed to list schedules") + return + } + + items := make([]apitypes.ScheduleResponse, 0, len(jobs)) + for _, j := range jobs { + items = append(items, toScheduleResponse(j, nil)) + } + c.JSON(http.StatusOK, gin.H{"items": items, "limit": limit, "offset": offset}) +} + +func (h *ScheduleHandler) GetSchedule(c *gin.Context) { + job, err := h.jobStore.GetByID(c.Request.Context(), c.Param("id")) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + response.RespondNotFound(c, "schedule not found") + return + } + response.RespondInternalError(c, "failed to get schedule") + return + } + if job.JobType != "ONE_TIME" { + response.RespondNotFound(c, "schedule not found") + return + } + + execs, err := h.executionStore.GetByJobID(c.Request.Context(), job.ID) + if err != nil { + response.RespondInternalError(c, "failed to get schedule executions") + return + } + + c.JSON(http.StatusOK, toScheduleResponse(job, execs)) +} + +func (h *ScheduleHandler) CancelSchedule(c *gin.Context) { + job, err := h.jobStore.GetByID(c.Request.Context(), c.Param("id")) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + response.RespondNotFound(c, "schedule not found") + return + } + response.RespondInternalError(c, "failed to get schedule") + return + } + if job.JobType != "ONE_TIME" { + response.RespondNotFound(c, "schedule not found") + return + } + if job.Status != "PENDING" { + response.RespondError(c, http.StatusConflict, "CONFLICT", "only pending schedules can be cancelled") + return + } + + if err := h.jobStore.Cancel(c.Request.Context(), job.ID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + response.RespondError(c, http.StatusConflict, "CONFLICT", "only pending schedules can be cancelled") + return + } + response.RespondInternalError(c, "failed to cancel schedule") + return + } + c.Status(http.StatusNoContent) +} + +func (h *ScheduleHandler) resolveDelivery(ctx context.Context, req apitypes.CreateScheduleRequest) (string, []byte, *string, error) { + if strings.TrimSpace(req.JobDefinitionID) != "" { + id := strings.TrimSpace(req.JobDefinitionID) + def, err := h.jobDefStore.GetByID(ctx, id) + if err != nil { + return "", nil, nil, err + } + return def.DeliveryType, def.DeliveryConfig, &def.ID, nil + } + + config, _, err := deliveryConfigFromRequest(req.DeliveryMode, req.Endpoint, req.HTTPMethod, req.HTTPHeaders, req.RetryPolicy.BackoffArray) + if err != nil { + return "", nil, nil, err + } + + mode := strings.ToUpper(strings.TrimSpace(req.DeliveryMode)) + return mode, config, nil, nil +} + +func toScheduleResponse(job *pgstore.Job, execs []*pgstore.JobExecution) apitypes.ScheduleResponse { + resp := apitypes.ScheduleResponse{ + ID: job.ID, + JobType: job.JobType, + ScheduledAt: job.ScheduledAt.UTC().Format(time.RFC3339), + Status: job.Status, + Payload: job.Payload, + CreatedAt: job.CreatedAt.UTC().Format(time.RFC3339), + DeliveryMode: job.DeliveryType, + } + if job.JobDefID != nil { + resp.JobDefinitionID = *job.JobDefID + } + if job.BucketID != nil { + resp.BucketID = *job.BucketID + } + if job.IdempotencyKey != nil { + resp.IdempotencyKey = *job.IdempotencyKey + } + + if len(job.DeliveryConfig) > 0 { + var cfg map[string]any + if err := json.Unmarshal(job.DeliveryConfig, &cfg); err == nil { + if url, ok := cfg["url"].(string); ok { + resp.Endpoint = url + } + if topic, ok := cfg["topic"].(string); ok && resp.Endpoint == "" { + resp.Endpoint = topic + } + if method, ok := cfg["method"].(string); ok { + resp.HTTPMethod = method + } + } + } + + if len(execs) > 0 { + resp.Executions = make([]apitypes.ScheduleExecutionSummary, 0, len(execs)) + for _, ex := range execs { + s := apitypes.ScheduleExecutionSummary{ + ID: ex.ID, + AttemptNumber: ex.AttemptNumber, + Status: ex.Status, + StartedAt: ex.StartedAt.UTC().Format(time.RFC3339), + } + if ex.ResponseStatus != nil { + s.ResponseStatus = ex.ResponseStatus + } + if ex.DurationMs != nil { + s.DurationMS = ex.DurationMs + } + if ex.ErrorMessage != nil { + s.ErrorMessage = *ex.ErrorMessage + } + if ex.CompletedAt != nil { + s.CompletedAt = ex.CompletedAt.UTC().Format(time.RFC3339) + } + resp.Executions = append(resp.Executions, s) + } + } + + return resp +} + +func ptrTime(t time.Time) *time.Time { + return &t +} diff --git a/internal/api/http/router.go b/internal/api/http/router.go index 1092feb..c232f9a 100644 --- a/internal/api/http/router.go +++ b/internal/api/http/router.go @@ -86,7 +86,11 @@ func NewRouter(deps RouterDeps) *gin.Engine { apiKeyStore := pgstore.NewAPIKeyStore(deps.PostgresDB) jobDefStore := pgstore.NewJobDefinitionStore(deps.PostgresDB) + jobStore := pgstore.NewJobStore(deps.PostgresDB) + scheduleBucketStore := pgstore.NewScheduleBucketStore(deps.PostgresDB) + jobExecutionStore := pgstore.NewJobExecutionStore(deps.PostgresDB) jobDefHandler := handlers.NewJobDefinitionHandler(jobDefStore) + scheduleHandler := handlers.NewScheduleHandler(jobStore, scheduleBucketStore, jobDefStore, jobExecutionStore) // API v1 apiV1 := r.Group("/api/v1") @@ -107,6 +111,11 @@ func NewRouter(deps RouterDeps) *gin.Engine { authenticated.GET("/job-definitions/:id", jobDefHandler.GetJobDefinition) authenticated.PUT("/job-definitions/:id", jobDefHandler.UpdateJobDefinition) authenticated.DELETE("/job-definitions/:id", jobDefHandler.DeleteJobDefinition) + + authenticated.POST("/schedules", scheduleHandler.CreateSchedule) + authenticated.GET("/schedules", scheduleHandler.ListSchedules) + authenticated.GET("/schedules/:id", scheduleHandler.GetSchedule) + authenticated.DELETE("/schedules/:id", scheduleHandler.CancelSchedule) } } diff --git a/internal/api/http/types/schedule.go b/internal/api/http/types/schedule.go index ca496e7..4c5610b 100644 --- a/internal/api/http/types/schedule.go +++ b/internal/api/http/types/schedule.go @@ -12,7 +12,7 @@ var ( ) type CreateScheduleRequest struct { - JobDefinitionID string `json:"job_definition_id,omitempty"` + JobDefinitionID string `json:"job_definition_id,omitempty" binding:"omitempty,uuid"` ScheduledAt time.Time `json:"scheduled_at" binding:"required"` Payload json.RawMessage `json:"payload"` IdempotencyKey string `json:"idempotency_key"` @@ -59,6 +59,9 @@ type ScheduleResponse struct { JobType string `json:"job_type"` JobDefinitionID string `json:"job_definition_id,omitempty"` BucketID string `json:"bucket_id,omitempty"` + DeliveryMode string `json:"delivery_mode,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + HTTPMethod string `json:"http_method,omitempty"` ScheduledAt string `json:"scheduled_at"` Status string `json:"status"` IdempotencyKey string `json:"idempotency_key,omitempty"` diff --git a/internal/store/postgres/job_store.go b/internal/store/postgres/job_store.go index 324008d..23dbda2 100644 --- a/internal/store/postgres/job_store.go +++ b/internal/store/postgres/job_store.go @@ -2,6 +2,8 @@ package postgres import ( "context" + "database/sql" + "strconv" "time" "github.com/jmoiron/sqlx" @@ -30,6 +32,15 @@ type JobStore struct { db *sqlx.DB } +type ListJobsFilter struct { + JobType string + Status string + ScheduledFrom *time.Time + ScheduledTo *time.Time + Limit int + Offset int +} + func NewJobStore(db *sqlx.DB) *JobStore { return &JobStore{db: db} } @@ -65,6 +76,56 @@ func (s *JobStore) GetByID(ctx context.Context, id string) (*Job, error) { return &j, nil } +func (s *JobStore) GetByIdempotencyKey(ctx context.Context, key string) (*Job, error) { + var j Job + err := s.db.GetContext(ctx, &j, `SELECT * FROM jobs WHERE idempotency_key = $1`, key) + if err != nil { + return nil, err + } + return &j, nil +} + +func (s *JobStore) List(ctx context.Context, f ListJobsFilter) ([]*Job, error) { + if f.Limit <= 0 { + f.Limit = 50 + } + if f.JobType == "" { + f.JobType = "ONE_TIME" + } + + query := `SELECT * FROM jobs WHERE job_type = $1` + args := []any{f.JobType} + argIdx := 2 + + if f.Status != "" { + query += ` AND status = $` + strconv.Itoa(argIdx) + args = append(args, f.Status) + argIdx++ + } + if f.ScheduledFrom != nil { + query += ` AND scheduled_at >= $` + strconv.Itoa(argIdx) + args = append(args, *f.ScheduledFrom) + argIdx++ + } + if f.ScheduledTo != nil { + query += ` AND scheduled_at <= $` + strconv.Itoa(argIdx) + args = append(args, *f.ScheduledTo) + argIdx++ + } + + query += ` ORDER BY scheduled_at ASC LIMIT $` + strconv.Itoa(argIdx) + args = append(args, f.Limit) + argIdx++ + query += ` OFFSET $` + strconv.Itoa(argIdx) + args = append(args, f.Offset) + + var jobs []*Job + if err := s.db.SelectContext(ctx, &jobs, query, args...); err != nil { + return nil, err + } + return jobs, nil +} + // ListByBucket returns all pending jobs for a bucket using FOR UPDATE SKIP LOCKED. // Returns both ONE_TIME and CRON jobs — the fan-out worker is job-type-agnostic. func (s *JobStore) ListByBucket(ctx context.Context, bucketID string) ([]*Job, error) { @@ -115,8 +176,18 @@ func (s *JobStore) UpdateStatus(ctx context.Context, id, status string) error { } func (s *JobStore) Cancel(ctx context.Context, id string) error { - _, err := s.db.ExecContext(ctx, + res, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = 'CANCELLED' WHERE id = $1 AND status = 'PENDING'`, id) - return err + if err != nil { + return err + } + rowsAffected, err := res.RowsAffected() + if err != nil { + return err + } + if rowsAffected == 0 { + return sql.ErrNoRows + } + return nil } From 200ee744a0df682d0dffaf66d97625aab0ee6a5f Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 05:08:08 -0700 Subject: [PATCH 03/20] feat(data-layer): add job tags with indexed filtering and schedule propagation --- .../api/http/handlers/schedule_handler.go | 56 +++++++++++++++++++ internal/api/http/types/schedule.go | 2 + internal/store/postgres/job_store.go | 44 +++++++++------ .../postgres/0019_add_tags_to_jobs.down.sql | 2 + .../postgres/0019_add_tags_to_jobs.up.sql | 2 + 5 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 migrations/postgres/0019_add_tags_to_jobs.down.sql create mode 100644 migrations/postgres/0019_add_tags_to_jobs.up.sql diff --git a/internal/api/http/handlers/schedule_handler.go b/internal/api/http/handlers/schedule_handler.go index 9e92ac1..eeb830d 100644 --- a/internal/api/http/handlers/schedule_handler.go +++ b/internal/api/http/handlers/schedule_handler.go @@ -74,6 +74,16 @@ func (h *ScheduleHandler) CreateSchedule(c *gin.Context) { } contextPayload := []byte(`{}`) + jobTags := pq.StringArray(resolvedTags(req.Tags, nil)) + if jobDefID != nil { + def, err := h.jobDefStore.GetByID(c.Request.Context(), *jobDefID) + if err != nil { + response.RespondInternalError(c, "failed to resolve job definition tags") + return + } + jobTags = pq.StringArray(resolvedTags(req.Tags, []string(def.Tags))) + } + job := &pgstore.Job{ JobType: "ONE_TIME", JobDefID: jobDefID, @@ -84,6 +94,7 @@ func (h *ScheduleHandler) CreateSchedule(c *gin.Context) { Context: contextPayload, ScheduledAt: ptrTime(req.ScheduledAt.UTC()), Status: "PENDING", + Tags: jobTags, } if strings.TrimSpace(req.IdempotencyKey) != "" { key := strings.TrimSpace(req.IdempotencyKey) @@ -133,6 +144,9 @@ func (h *ScheduleHandler) ListSchedules(c *gin.Context) { Limit: limit, Offset: offset, } + if tagsQ := strings.TrimSpace(c.Query("tags")); tagsQ != "" { + filter.Tags = parseTagQuery(tagsQ) + } if from := strings.TrimSpace(c.Query("date_from")); from != "" { t, err := time.Parse(time.RFC3339, from) if err != nil { @@ -255,6 +269,9 @@ func toScheduleResponse(job *pgstore.Job, execs []*pgstore.JobExecution) apitype if job.IdempotencyKey != nil { resp.IdempotencyKey = *job.IdempotencyKey } + if len(job.Tags) > 0 { + resp.Tags = append([]string(nil), job.Tags...) + } if len(job.DeliveryConfig) > 0 { var cfg map[string]any @@ -302,3 +319,42 @@ func toScheduleResponse(job *pgstore.Job, execs []*pgstore.JobExecution) apitype func ptrTime(t time.Time) *time.Time { return &t } + +func parseTagQuery(raw string) []string { + parts := strings.Split(raw, ",") + tags := make([]string, 0, len(parts)) + for _, p := range parts { + t := strings.TrimSpace(p) + if t != "" { + tags = append(tags, t) + } + } + return tags +} + +func resolvedTags(requestTags []string, definitionTags []string) []string { + if len(requestTags) > 0 { + return normalizeTags(requestTags) + } + if len(definitionTags) > 0 { + return normalizeTags(definitionTags) + } + return nil +} + +func normalizeTags(tags []string) []string { + seen := make(map[string]struct{}, len(tags)) + out := make([]string, 0, len(tags)) + for _, t := range tags { + tag := strings.TrimSpace(t) + if tag == "" { + continue + } + if _, ok := seen[tag]; ok { + continue + } + seen[tag] = struct{}{} + out = append(out, tag) + } + return out +} diff --git a/internal/api/http/types/schedule.go b/internal/api/http/types/schedule.go index 4c5610b..82493f4 100644 --- a/internal/api/http/types/schedule.go +++ b/internal/api/http/types/schedule.go @@ -16,6 +16,7 @@ type CreateScheduleRequest struct { ScheduledAt time.Time `json:"scheduled_at" binding:"required"` Payload json.RawMessage `json:"payload"` IdempotencyKey string `json:"idempotency_key"` + Tags []string `json:"tags,omitempty"` // Ad-hoc inline fields DeliveryMode string `json:"delivery_mode,omitempty"` @@ -65,6 +66,7 @@ type ScheduleResponse struct { ScheduledAt string `json:"scheduled_at"` Status string `json:"status"` IdempotencyKey string `json:"idempotency_key,omitempty"` + Tags []string `json:"tags,omitempty"` Payload json.RawMessage `json:"payload,omitempty"` CreatedAt string `json:"created_at"` Executions []ScheduleExecutionSummary `json:"executions,omitempty"` diff --git a/internal/store/postgres/job_store.go b/internal/store/postgres/job_store.go index 23dbda2..106a3af 100644 --- a/internal/store/postgres/job_store.go +++ b/internal/store/postgres/job_store.go @@ -7,25 +7,27 @@ import ( "time" "github.com/jmoiron/sqlx" + "github.com/lib/pq" ) type Job struct { - ID string `db:"id"` - JobType string `db:"job_type"` // ONE_TIME | CRON | BULK_RECORD - JobDefID *string `db:"job_def_id"` // NULL for ad-hoc - CronScheduleID *string `db:"cron_schedule_id"` // non-null for CRON - BulkJobID *string `db:"bulk_job_id"` // non-null for BULK_RECORD - BucketID *string `db:"bucket_id"` // non-null for ONE_TIME + CRON - DeliveryType string `db:"delivery_type"` - DeliveryConfig []byte `db:"delivery_config"` - RowStart *int `db:"row_start"` // non-null for BULK_RECORD: first row in batch (0-indexed) - RowEnd *int `db:"row_end"` // non-null for BULK_RECORD: last row in batch (inclusive); == RowStart when batch_size=1 - Payload []byte `db:"payload"` // JSONB — resolved request body; array when batch_size > 1 - Context []byte `db:"context"` // JSONB — runtime vars for delivery_config substitution - ScheduledAt *time.Time `db:"scheduled_at"` // non-null for ONE_TIME + CRON - Status string `db:"status"` - IdempotencyKey *string `db:"idempotency_key"` - CreatedAt time.Time `db:"created_at"` + ID string `db:"id"` + JobType string `db:"job_type"` // ONE_TIME | CRON | BULK_RECORD + JobDefID *string `db:"job_def_id"` // NULL for ad-hoc + CronScheduleID *string `db:"cron_schedule_id"` // non-null for CRON + BulkJobID *string `db:"bulk_job_id"` // non-null for BULK_RECORD + BucketID *string `db:"bucket_id"` // non-null for ONE_TIME + CRON + DeliveryType string `db:"delivery_type"` + DeliveryConfig []byte `db:"delivery_config"` + RowStart *int `db:"row_start"` // non-null for BULK_RECORD: first row in batch (0-indexed) + RowEnd *int `db:"row_end"` // non-null for BULK_RECORD: last row in batch (inclusive); == RowStart when batch_size=1 + Payload []byte `db:"payload"` // JSONB — resolved request body; array when batch_size > 1 + Context []byte `db:"context"` // JSONB — runtime vars for delivery_config substitution + ScheduledAt *time.Time `db:"scheduled_at"` // non-null for ONE_TIME + CRON + Status string `db:"status"` + IdempotencyKey *string `db:"idempotency_key"` + Tags pq.StringArray `db:"tags"` + CreatedAt time.Time `db:"created_at"` } type JobStore struct { @@ -35,6 +37,7 @@ type JobStore struct { type ListJobsFilter struct { JobType string Status string + Tags []string ScheduledFrom *time.Time ScheduledTo *time.Time Limit int @@ -50,11 +53,11 @@ func (s *JobStore) Create(ctx context.Context, j *Job) error { INSERT INTO jobs (job_type, job_def_id, cron_schedule_id, bulk_job_id, bucket_id, delivery_type, delivery_config, row_start, row_end, payload, context, - scheduled_at, status, idempotency_key) + scheduled_at, status, idempotency_key, tags) VALUES (:job_type, :job_def_id, :cron_schedule_id, :bulk_job_id, :bucket_id, :delivery_type, :delivery_config, :row_start, :row_end, :payload, :context, - :scheduled_at, :status, :idempotency_key) + :scheduled_at, :status, :idempotency_key, :tags) RETURNING id, created_at` rows, err := s.db.NamedQueryContext(ctx, q, j) if err != nil { @@ -102,6 +105,11 @@ func (s *JobStore) List(ctx context.Context, f ListJobsFilter) ([]*Job, error) { args = append(args, f.Status) argIdx++ } + if len(f.Tags) > 0 { + query += ` AND tags @> $` + strconv.Itoa(argIdx) + args = append(args, pq.Array(f.Tags)) + argIdx++ + } if f.ScheduledFrom != nil { query += ` AND scheduled_at >= $` + strconv.Itoa(argIdx) args = append(args, *f.ScheduledFrom) diff --git a/migrations/postgres/0019_add_tags_to_jobs.down.sql b/migrations/postgres/0019_add_tags_to_jobs.down.sql new file mode 100644 index 0000000..b44032a --- /dev/null +++ b/migrations/postgres/0019_add_tags_to_jobs.down.sql @@ -0,0 +1,2 @@ +-- DROP INDEX IF EXISTS idx_jobs_tags_gin; +ALTER TABLE jobs DROP COLUMN IF EXISTS tags; diff --git a/migrations/postgres/0019_add_tags_to_jobs.up.sql b/migrations/postgres/0019_add_tags_to_jobs.up.sql new file mode 100644 index 0000000..9ab9b56 --- /dev/null +++ b/migrations/postgres/0019_add_tags_to_jobs.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE jobs ADD COLUMN tags TEXT[] NOT NULL DEFAULT '{}'; +-- CREATE INDEX IF NOT EXISTS idx_jobs_tags_gin ON jobs USING GIN (tags); From 883e3013428cc99dd9cbe0c9b63f5a6b384fbd0b Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 05:12:08 -0700 Subject: [PATCH 04/20] feat(api): implement executions read API --- .../api/http/handlers/execution_handler.go | 132 ++++++++++++++++++ internal/api/http/router.go | 6 + .../store/postgres/job_execution_store.go | 55 ++++++++ 3 files changed, 193 insertions(+) create mode 100644 internal/api/http/handlers/execution_handler.go diff --git a/internal/api/http/handlers/execution_handler.go b/internal/api/http/handlers/execution_handler.go new file mode 100644 index 0000000..d5a0f39 --- /dev/null +++ b/internal/api/http/handlers/execution_handler.go @@ -0,0 +1,132 @@ +package handlers + +import ( + "database/sql" + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" + + "github.com/chronos-scheduler/chronos/internal/api/http/response" + mongostore "github.com/chronos-scheduler/chronos/internal/store/mongo" + pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" +) + +type ExecutionHandler struct { + executionStore *pgstore.JobExecutionStore + logStore *mongostore.LogStore +} + +func NewExecutionHandler(executionStore *pgstore.JobExecutionStore, logStore *mongostore.LogStore) *ExecutionHandler { + return &ExecutionHandler{executionStore: executionStore, logStore: logStore} +} + +func (h *ExecutionHandler) ListExecutions(c *gin.Context) { + limit := 50 + offset := 0 + if s := strings.TrimSpace(c.Query("limit")); s != "" { + n, err := strconv.Atoi(s) + if err != nil || n <= 0 { + response.RespondBadRequest(c, "invalid limit") + return + } + limit = n + } + if s := strings.TrimSpace(c.Query("offset")); s != "" { + n, err := strconv.Atoi(s) + if err != nil || n < 0 { + response.RespondBadRequest(c, "invalid offset") + return + } + offset = n + } + + filter := pgstore.ListJobExecutionsFilter{ + JobDefID: strings.TrimSpace(c.Query("job_def_id")), + Status: strings.TrimSpace(c.Query("status")), + Limit: limit, + Offset: offset, + } + if from := strings.TrimSpace(c.Query("date_from")); from != "" { + t, err := time.Parse(time.RFC3339, from) + if err != nil { + response.RespondBadRequest(c, "invalid date_from; expected RFC3339") + return + } + filter.StartedFrom = &t + } + if to := strings.TrimSpace(c.Query("date_to")); to != "" { + t, err := time.Parse(time.RFC3339, to) + if err != nil { + response.RespondBadRequest(c, "invalid date_to; expected RFC3339") + return + } + filter.StartedTo = &t + } + + execs, err := h.executionStore.List(c.Request.Context(), filter) + if err != nil { + response.RespondInternalError(c, "failed to list executions") + return + } + + items := make([]gin.H, 0, len(execs)) + for _, e := range execs { + items = append(items, toExecutionSummaryResponse(e)) + } + + c.JSON(http.StatusOK, gin.H{"items": items, "limit": limit, "offset": offset}) +} + +func (h *ExecutionHandler) GetExecution(c *gin.Context) { + execID := c.Param("id") + e, err := h.executionStore.GetByID(c.Request.Context(), execID) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + response.RespondNotFound(c, "execution not found") + return + } + response.RespondInternalError(c, "failed to get execution") + return + } + + logs, err := h.logStore.GetLogsForExecution(c.Request.Context(), execID) + if err != nil { + response.RespondInternalError(c, "failed to get execution logs") + return + } + + c.JSON(http.StatusOK, gin.H{ + "execution": toExecutionSummaryResponse(e), + "logs": logs, + }) +} + +func toExecutionSummaryResponse(e *pgstore.JobExecution) gin.H { + item := gin.H{ + "id": e.ID, + "job_id": e.JobID, + "attempt_number": e.AttemptNumber, + "status": e.Status, + "started_at": e.StartedAt.UTC().Format(time.RFC3339), + } + if e.CompletedAt != nil { + item["completed_at"] = e.CompletedAt.UTC().Format(time.RFC3339) + } + if e.DurationMs != nil { + item["duration_ms"] = *e.DurationMs + } + if e.ResponseStatus != nil { + item["response_status"] = *e.ResponseStatus + } + if e.ErrorMessage != nil { + item["error_message"] = *e.ErrorMessage + } + if e.WorkerID != nil { + item["worker_id"] = *e.WorkerID + } + return item +} diff --git a/internal/api/http/router.go b/internal/api/http/router.go index c232f9a..955d6c0 100644 --- a/internal/api/http/router.go +++ b/internal/api/http/router.go @@ -16,6 +16,7 @@ import ( "github.com/chronos-scheduler/chronos/internal/api/http/handlers" "github.com/chronos-scheduler/chronos/internal/api/http/middleware" "github.com/chronos-scheduler/chronos/internal/api/http/response" + mongostore "github.com/chronos-scheduler/chronos/internal/store/mongo" pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" ) @@ -89,8 +90,10 @@ func NewRouter(deps RouterDeps) *gin.Engine { jobStore := pgstore.NewJobStore(deps.PostgresDB) scheduleBucketStore := pgstore.NewScheduleBucketStore(deps.PostgresDB) jobExecutionStore := pgstore.NewJobExecutionStore(deps.PostgresDB) + logStore := mongostore.NewLogStore(deps.MongoClient, "chronos") jobDefHandler := handlers.NewJobDefinitionHandler(jobDefStore) scheduleHandler := handlers.NewScheduleHandler(jobStore, scheduleBucketStore, jobDefStore, jobExecutionStore) + executionHandler := handlers.NewExecutionHandler(jobExecutionStore, logStore) // API v1 apiV1 := r.Group("/api/v1") @@ -116,6 +119,9 @@ func NewRouter(deps RouterDeps) *gin.Engine { authenticated.GET("/schedules", scheduleHandler.ListSchedules) authenticated.GET("/schedules/:id", scheduleHandler.GetSchedule) authenticated.DELETE("/schedules/:id", scheduleHandler.CancelSchedule) + + authenticated.GET("/executions", executionHandler.ListExecutions) + authenticated.GET("/executions/:id", executionHandler.GetExecution) } } diff --git a/internal/store/postgres/job_execution_store.go b/internal/store/postgres/job_execution_store.go index dcd4c19..8e3439e 100644 --- a/internal/store/postgres/job_execution_store.go +++ b/internal/store/postgres/job_execution_store.go @@ -2,6 +2,7 @@ package postgres import ( "context" + "strconv" "time" "github.com/jmoiron/sqlx" @@ -24,6 +25,15 @@ type JobExecutionStore struct { db *sqlx.DB } +type ListJobExecutionsFilter struct { + JobDefID string + Status string + StartedFrom *time.Time + StartedTo *time.Time + Limit int + Offset int +} + func NewJobExecutionStore(db *sqlx.DB) *JobExecutionStore { return &JobExecutionStore{db: db} } @@ -53,6 +63,51 @@ func (s *JobExecutionStore) GetByID(ctx context.Context, id string) (*JobExecuti return &e, nil } +func (s *JobExecutionStore) List(ctx context.Context, f ListJobExecutionsFilter) ([]*JobExecution, error) { + if f.Limit <= 0 { + f.Limit = 50 + } + + query := `SELECT je.* FROM job_executions je + JOIN jobs j ON j.id = je.job_id + WHERE 1=1` + args := make([]any, 0, 6) + argIdx := 1 + + if f.JobDefID != "" { + query += ` AND j.job_def_id = $` + strconv.Itoa(argIdx) + args = append(args, f.JobDefID) + argIdx++ + } + if f.Status != "" { + query += ` AND je.status = $` + strconv.Itoa(argIdx) + args = append(args, f.Status) + argIdx++ + } + if f.StartedFrom != nil { + query += ` AND je.started_at >= $` + strconv.Itoa(argIdx) + args = append(args, *f.StartedFrom) + argIdx++ + } + if f.StartedTo != nil { + query += ` AND je.started_at <= $` + strconv.Itoa(argIdx) + args = append(args, *f.StartedTo) + argIdx++ + } + + query += ` ORDER BY je.started_at DESC LIMIT $` + strconv.Itoa(argIdx) + args = append(args, f.Limit) + argIdx++ + query += ` OFFSET $` + strconv.Itoa(argIdx) + args = append(args, f.Offset) + + var execs []*JobExecution + if err := s.db.SelectContext(ctx, &execs, query, args...); err != nil { + return nil, err + } + return execs, nil +} + func (s *JobExecutionStore) GetByJobID(ctx context.Context, jobID string) ([]*JobExecution, error) { var execs []*JobExecution err := s.db.SelectContext(ctx, &execs, From 804ca15d53a169dd84781f0268337b815b8a90b7 Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 05:16:00 -0700 Subject: [PATCH 05/20] feat(data-layer): add Kafka producer helper --- go.mod | 6 ++- go.sum | 65 ++++++++++++++++++++++---------- internal/store/kafka/producer.go | 41 ++++++++++++++++++++ 3 files changed, 92 insertions(+), 20 deletions(-) create mode 100644 internal/store/kafka/producer.go diff --git a/go.mod b/go.mod index a12c4bc..65df58f 100644 --- a/go.mod +++ b/go.mod @@ -93,7 +93,7 @@ require ( github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/google/uuid v1.6.0 // indirect + github.com/google/uuid v1.6.0 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mdelapenya/tlscert v0.2.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -120,3 +120,7 @@ require ( go.opentelemetry.io/otel/metric v1.41.0 // indirect go.opentelemetry.io/otel/trace v1.41.0 // indirect ) + +require github.com/segmentio/kafka-go v0.4.50 + +require github.com/pierrec/lz4/v4 v4.1.16 // indirect diff --git a/go.sum b/go.sum index 3275cd2..d528cb6 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -32,14 +34,20 @@ github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpS github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -65,6 +73,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -77,23 +87,30 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8= +github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= -github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -140,12 +157,18 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pierrec/lz4/v4 v4.1.16 h1:kQPfno+wyx6C5572ABwV+Uo3pDFzQ7yhyGchSyRda0c= +github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -153,10 +176,12 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/segmentio/kafka-go v0.4.50 h1:mcyC3tT5WeyWzrFbd6O374t+hmcu1NKt2Pu1L3QaXmc= +github.com/segmentio/kafka-go v0.4.50/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E= github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= @@ -174,6 +199,8 @@ github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjb github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -216,14 +243,16 @@ go.mongodb.org/mongo-driver/v2 v2.5.1 h1:j2U/Qp+wvueSpqitLCSZPT/+ZpVc1xzuwdHWwl7 go.mongodb.org/mongo-driver/v2 v2.5.1/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -241,8 +270,6 @@ golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -253,8 +280,6 @@ golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -267,18 +292,16 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -289,10 +312,14 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/internal/store/kafka/producer.go b/internal/store/kafka/producer.go new file mode 100644 index 0000000..f910260 --- /dev/null +++ b/internal/store/kafka/producer.go @@ -0,0 +1,41 @@ +package kafka + +import ( + "context" + "time" + + kafkago "github.com/segmentio/kafka-go" +) + +type Producer struct { + writer *kafkago.Writer +} + +func NewProducer(brokers []string) *Producer { + return &Producer{ + writer: &kafkago.Writer{ + Addr: kafkago.TCP(brokers...), + RequiredAcks: kafkago.RequireAll, + Async: false, + Balancer: &kafkago.Hash{}, + WriteTimeout: 10 * time.Second, + ReadTimeout: 10 * time.Second, + }, + } +} + +func (p *Producer) Publish(ctx context.Context, topic, key string, value []byte) error { + msg := kafkago.Message{ + Topic: topic, + Key: []byte(key), + Value: value, + } + return p.writer.WriteMessages(ctx, msg) +} + +func (p *Producer) Close() error { + if p == nil || p.writer == nil { + return nil + } + return p.writer.Close() +} From 526438dc707468c16ef6450229a25994227b9564 Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 05:20:15 -0700 Subject: [PATCH 06/20] feat(data-layer): add Kafka consumer helper with at-least-once offset commit --- internal/store/kafka/consumer.go | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 internal/store/kafka/consumer.go diff --git a/internal/store/kafka/consumer.go b/internal/store/kafka/consumer.go new file mode 100644 index 0000000..04d3a97 --- /dev/null +++ b/internal/store/kafka/consumer.go @@ -0,0 +1,52 @@ +package kafka + +import ( + "context" + "errors" + + kafkago "github.com/segmentio/kafka-go" +) + +type Consumer struct { + reader *kafkago.Reader +} + +func NewConsumer(brokers []string, groupID, topic string) *Consumer { + return &Consumer{ + reader: kafkago.NewReader(kafkago.ReaderConfig{ + Brokers: brokers, + GroupID: groupID, + Topic: topic, + }), + } +} + +func (c *Consumer) Consume(ctx context.Context, handler func(msg kafkago.Message) error) error { + for { + msg, err := c.reader.FetchMessage(ctx) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return nil + } + return err + } + + if err := handler(msg); err != nil { + return err + } + + if err := c.reader.CommitMessages(ctx, msg); err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return nil + } + return err + } + } +} + +func (c *Consumer) Close() error { + if c == nil || c.reader == nil { + return nil + } + return c.reader.Close() +} From 60dacc9f9f127d1e6a578192066630e6578e9a94 Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 05:37:16 -0700 Subject: [PATCH 07/20] feat(scheduler): implement Redis SETNX leader election with heartbeat --- internal/scheduler/leader.go | 100 +++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 internal/scheduler/leader.go diff --git a/internal/scheduler/leader.go b/internal/scheduler/leader.go new file mode 100644 index 0000000..0a43354 --- /dev/null +++ b/internal/scheduler/leader.go @@ -0,0 +1,100 @@ +package scheduler + +import ( + "context" + "errors" + "time" + + "github.com/google/uuid" + "github.com/redis/go-redis/v9" + + redisstore "github.com/chronos-scheduler/chronos/internal/store/redis" +) + +const acquireRetryInterval = 1 * time.Second + +// RunWithLeadership acquires the scheduler leader lock and runs fn only while +// this process holds the lock. The lock is refreshed every ttl/2 and fn is +// canceled immediately if leadership is lost. +func RunWithLeadership(ctx context.Context, client *redis.Client, ttl time.Duration, fn func(ctx context.Context)) { + if ttl <= 0 { + ttl = 30 * time.Second + } + + refreshEvery := ttl / 2 + if refreshEvery <= 0 { + refreshEvery = 1 * time.Second + } + + for { + if ctx.Err() != nil { + return + } + + token := uuid.NewString() + acquired, err := redisstore.TryAcquire(ctx, client, redisstore.LockKeyScheduler(), token, ttl) + if err != nil { + if !sleepOrDone(ctx, acquireRetryInterval) { + return + } + continue + } + if !acquired { + if !sleepOrDone(ctx, acquireRetryInterval) { + return + } + continue + } + + leaderCtx, cancelLeader := context.WithCancel(ctx) + fnDone := make(chan struct{}) + go func() { + defer close(fnDone) + fn(leaderCtx) + }() + + ticker := time.NewTicker(refreshEvery) + lostLeadership := false + + for !lostLeadership { + select { + case <-ctx.Done(): + lostLeadership = true + case <-fnDone: + lostLeadership = true + case <-ticker.C: + if err := redisstore.Refresh(ctx, client, redisstore.LockKeyScheduler(), token, ttl); err != nil { + if errors.Is(err, redisstore.ErrLockNotHeld) { + lostLeadership = true + break + } + lostLeadership = true + } + } + } + + ticker.Stop() + cancelLeader() + <-fnDone + _ = releaseIfHeld(ctx, client, token) + } +} + +func releaseIfHeld(ctx context.Context, client *redis.Client, token string) error { + err := redisstore.Release(ctx, client, redisstore.LockKeyScheduler(), token) + if err != nil && !errors.Is(err, redisstore.ErrLockNotHeld) { + return err + } + return nil +} + +func sleepOrDone(ctx context.Context, d time.Duration) bool { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return false + case <-t.C: + return true + } +} From 06a7b81aa56894fcb139e59b65e7677e8205ea28 Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 06:12:50 -0700 Subject: [PATCH 08/20] refactor(scheduler): make bucket dispatcher leader-only and remove scheduler fan-out logic --- internal/scheduler/dispatcher.go | 88 ++++++++++++++++++++++++++++ internal/store/postgres/job_store.go | 10 ++++ internal/store/redis/keys.go | 8 ++- 3 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 internal/scheduler/dispatcher.go diff --git a/internal/scheduler/dispatcher.go b/internal/scheduler/dispatcher.go new file mode 100644 index 0000000..8bc050c --- /dev/null +++ b/internal/scheduler/dispatcher.go @@ -0,0 +1,88 @@ +package scheduler + +import ( + "context" + "encoding/json" + "time" + + kafkastore "github.com/chronos-scheduler/chronos/internal/store/kafka" + pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" +) + +const ( + defaultBucketSeconds = 60 + defaultBucketScanMax = 100 +) + +type BucketTriggerEvent struct { + BucketID string `json:"bucket_id"` +} + +type Dispatcher struct { + bucketStore *pgstore.ScheduleBucketStore + kafkaProducer *kafkastore.Producer + bucketSeconds int +} + +func NewDispatcher( + bucketStore *pgstore.ScheduleBucketStore, + kafkaProducer *kafkastore.Producer, + bucketSeconds int, +) *Dispatcher { + if bucketSeconds <= 0 { + bucketSeconds = defaultBucketSeconds + } + return &Dispatcher{ + bucketStore: bucketStore, + kafkaProducer: kafkaProducer, + bucketSeconds: bucketSeconds, + } +} + +func (d *Dispatcher) Run(ctx context.Context) error { + ticker := time.NewTicker(time.Duration(d.bucketSeconds) * time.Second) + defer ticker.Stop() + + if err := d.dispatchDueBuckets(ctx, time.Now().UTC()); err != nil { + return err + } + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if err := d.dispatchDueBuckets(ctx, time.Now().UTC()); err != nil { + return err + } + } + } +} + +func (d *Dispatcher) dispatchDueBuckets(ctx context.Context, upTo time.Time) error { + buckets, err := d.bucketStore.ListPending(ctx, upTo, defaultBucketScanMax) + if err != nil { + return err + } + + for _, b := range buckets { + if err := d.dispatchSingleBucket(ctx, b); err != nil { + return err + } + } + return nil +} + +func (d *Dispatcher) dispatchSingleBucket(ctx context.Context, bucket *pgstore.ScheduleBucket) error { + event := BucketTriggerEvent{ + BucketID: bucket.ID, + } + payload, err := json.Marshal(event) + if err != nil { + return err + } + if err := d.kafkaProducer.Publish(ctx, "bucket-triggers", bucket.ID, payload); err != nil { + return err + } + return d.bucketStore.MarkFired(ctx, bucket.ID) +} diff --git a/internal/store/postgres/job_store.go b/internal/store/postgres/job_store.go index 106a3af..09e9670 100644 --- a/internal/store/postgres/job_store.go +++ b/internal/store/postgres/job_store.go @@ -183,6 +183,16 @@ func (s *JobStore) UpdateStatus(ctx context.Context, id, status string) error { return err } +func (s *JobStore) UpdateStatusByIDs(ctx context.Context, ids []string, status string) error { + if len(ids) == 0 { + return nil + } + _, err := s.db.ExecContext(ctx, + `UPDATE jobs SET status = $2 WHERE id = ANY($1)`, + pq.Array(ids), status) + return err +} + func (s *JobStore) Cancel(ctx context.Context, id string) error { res, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = 'CANCELLED' WHERE id = $1 AND status = 'PENDING'`, diff --git a/internal/store/redis/keys.go b/internal/store/redis/keys.go index b4c1b18..d152e09 100644 --- a/internal/store/redis/keys.go +++ b/internal/store/redis/keys.go @@ -2,14 +2,18 @@ package redis import "fmt" -func LockKeyScheduler() string { return "chronos:lock:scheduler" } -func LockKeyRetryEngine() string { return "chronos:lock:retry-engine" } +func LockKeyScheduler() string { return "chronos:lock:scheduler" } +func LockKeyRetryEngine() string { return "chronos:lock:retry-engine" } func LockKeyMonitorWatchdog() string { return "chronos:lock:monitor-watchdog" } func IdempotencyKey(executionID string) string { return "chronos:idem:" + executionID } +func SchedulerDispatchIdempotencyKey(jobID string) string { + return "chronos:idem:scheduler:" + jobID +} + func ProgressChannel(jobID string) string { return "chronos:progress:" + jobID } From f6fff55b7f9cab1775bf6da674678aec7eeb91ee Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 06:17:46 -0700 Subject: [PATCH 09/20] test(scheduler): add unit tests for bucket dispatch loop --- internal/scheduler/dispatcher.go | 18 +++- internal/scheduler/dispatcher_test.go | 128 ++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 5 deletions(-) create mode 100644 internal/scheduler/dispatcher_test.go diff --git a/internal/scheduler/dispatcher.go b/internal/scheduler/dispatcher.go index 8bc050c..6b833cb 100644 --- a/internal/scheduler/dispatcher.go +++ b/internal/scheduler/dispatcher.go @@ -5,7 +5,6 @@ import ( "encoding/json" "time" - kafkastore "github.com/chronos-scheduler/chronos/internal/store/kafka" pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" ) @@ -18,15 +17,24 @@ type BucketTriggerEvent struct { BucketID string `json:"bucket_id"` } +type bucketStore interface { + ListPending(ctx context.Context, upTo time.Time, limit int) ([]*pgstore.ScheduleBucket, error) + MarkFired(ctx context.Context, id string) error +} + +type kafkaPublisher interface { + Publish(ctx context.Context, topic, key string, value []byte) error +} + type Dispatcher struct { - bucketStore *pgstore.ScheduleBucketStore - kafkaProducer *kafkastore.Producer + bucketStore bucketStore + kafkaProducer kafkaPublisher bucketSeconds int } func NewDispatcher( - bucketStore *pgstore.ScheduleBucketStore, - kafkaProducer *kafkastore.Producer, + bucketStore bucketStore, + kafkaProducer kafkaPublisher, bucketSeconds int, ) *Dispatcher { if bucketSeconds <= 0 { diff --git a/internal/scheduler/dispatcher_test.go b/internal/scheduler/dispatcher_test.go new file mode 100644 index 0000000..4fa7df8 --- /dev/null +++ b/internal/scheduler/dispatcher_test.go @@ -0,0 +1,128 @@ +package scheduler + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" +) + +type fakeBucketStore struct { + buckets []*pgstore.ScheduleBucket + firedIDs []string + listCalled int +} + +func (s *fakeBucketStore) ListPending(_ context.Context, upTo time.Time, _ int) ([]*pgstore.ScheduleBucket, error) { + s.listCalled++ + out := make([]*pgstore.ScheduleBucket, 0) + for _, b := range s.buckets { + if b.Status == "PENDING" && !b.FireAt.After(upTo) { + out = append(out, b) + } + } + return out, nil +} + +func (s *fakeBucketStore) MarkFired(_ context.Context, id string) error { + s.firedIDs = append(s.firedIDs, id) + for _, b := range s.buckets { + if b.ID == id { + b.Status = "FIRED" + now := time.Now().UTC() + b.FiredAt = &now + break + } + } + return nil +} + +type fakePublisher struct { + err error + messages []publishedMsg +} + +type publishedMsg struct { + topic string + key string + value []byte +} + +func (p *fakePublisher) Publish(_ context.Context, topic, key string, value []byte) error { + if p.err != nil { + return p.err + } + p.messages = append(p.messages, publishedMsg{topic: topic, key: key, value: value}) + return nil +} + +func TestDispatcher_PublishesOnlyDueBuckets(t *testing.T) { + now := time.Date(2026, 4, 21, 10, 5, 0, 0, time.UTC) + store := &fakeBucketStore{buckets: []*pgstore.ScheduleBucket{ + {ID: "due-1", FireAt: now.Add(-time.Minute), Status: "PENDING"}, + {ID: "future-1", FireAt: now.Add(time.Minute), Status: "PENDING"}, + {ID: "already-fired", FireAt: now.Add(-2 * time.Minute), Status: "FIRED"}, + }} + pub := &fakePublisher{} + d := NewDispatcher(store, pub, 60) + + if err := d.dispatchDueBuckets(context.Background(), now); err != nil { + t.Fatalf("dispatchDueBuckets: %v", err) + } + if len(pub.messages) != 1 { + t.Fatalf("published messages: got %d want 1", len(pub.messages)) + } + if pub.messages[0].topic != "bucket-triggers" || pub.messages[0].key != "due-1" { + t.Fatalf("published message mismatch: %+v", pub.messages[0]) + } + + var event BucketTriggerEvent + if err := json.Unmarshal(pub.messages[0].value, &event); err != nil { + t.Fatalf("unmarshal event: %v", err) + } + if event.BucketID != "due-1" { + t.Fatalf("event bucket_id: got %q want %q", event.BucketID, "due-1") + } +} + +func TestDispatcher_MarksBucketFiredAfterPublish(t *testing.T) { + now := time.Date(2026, 4, 21, 10, 5, 0, 0, time.UTC) + store := &fakeBucketStore{buckets: []*pgstore.ScheduleBucket{ + {ID: "due-2", FireAt: now.Add(-time.Second), Status: "PENDING"}, + }} + pub := &fakePublisher{} + d := NewDispatcher(store, pub, 60) + + if err := d.dispatchDueBuckets(context.Background(), now); err != nil { + t.Fatalf("dispatchDueBuckets: %v", err) + } + if len(store.firedIDs) != 1 || store.firedIDs[0] != "due-2" { + t.Fatalf("fired ids: got %#v", store.firedIDs) + } + if store.buckets[0].Status != "FIRED" { + t.Fatalf("bucket status: got %q want FIRED", store.buckets[0].Status) + } +} + +func TestDispatcher_PublishFailureKeepsBucketPending(t *testing.T) { + now := time.Date(2026, 4, 21, 10, 5, 0, 0, time.UTC) + store := &fakeBucketStore{buckets: []*pgstore.ScheduleBucket{ + {ID: "due-3", FireAt: now.Add(-time.Second), Status: "PENDING"}, + }} + pub := &fakePublisher{err: errors.New("kafka unavailable")} + d := NewDispatcher(store, pub, 60) + + err := d.dispatchDueBuckets(context.Background(), now) + if err == nil { + t.Fatal("expected publish error, got nil") + } + if len(store.firedIDs) != 0 { + t.Fatalf("expected no fired buckets, got %#v", store.firedIDs) + } + if store.buckets[0].Status != "PENDING" { + t.Fatalf("bucket status: got %q want PENDING", store.buckets[0].Status) + } +} From f78a6d554387c0e50b62e5329b8abe0a3b3ebb8f Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 06:23:20 -0700 Subject: [PATCH 10/20] feat(job-executor): add fan-out consumer from bucket-triggers to job-dispatch --- cmd/job-executor/main.go | 51 ++++++++++ internal/executor/fanout_consumer.go | 140 +++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 internal/executor/fanout_consumer.go diff --git a/cmd/job-executor/main.go b/cmd/job-executor/main.go index 2b4b219..73e33cc 100644 --- a/cmd/job-executor/main.go +++ b/cmd/job-executor/main.go @@ -12,6 +12,10 @@ import ( "go.uber.org/zap" "github.com/chronos-scheduler/chronos/internal/config" + "github.com/chronos-scheduler/chronos/internal/executor" + kafkastore "github.com/chronos-scheduler/chronos/internal/store/kafka" + pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" + redisstore "github.com/chronos-scheduler/chronos/internal/store/redis" "github.com/chronos-scheduler/chronos/internal/telemetry" ) @@ -34,6 +38,39 @@ func main() { zap.Strings("kafka_brokers", cfg.KafkaBrokers), ) + db, err := pgstore.Connect(cfg.PostgresDSN) + if err != nil { + logger.Fatal("postgres connection failed", zap.Error(err)) + } + logger.Info("postgres connected") + + redisClient, err := redisstore.Connect(cfg.RedisAddr) + if err != nil { + logger.Fatal("redis connection failed", zap.Error(err)) + } + logger.Info("redis connected") + + kafkaProducer := kafkastore.NewProducer(cfg.KafkaBrokers) + defer func() { + if err := kafkaProducer.Close(); err != nil { + logger.Error("kafka producer close error", zap.Error(err)) + } + }() + kafkaConsumer := kafkastore.NewConsumer(cfg.KafkaBrokers, "chronos-executor-fanout", "bucket-triggers") + defer func() { + if err := kafkaConsumer.Close(); err != nil { + logger.Error("kafka consumer close error", zap.Error(err)) + } + }() + + fanout := executor.NewFanoutConsumer( + logger, + kafkaConsumer, + kafkaProducer, + pgstore.NewJobStore(db), + redisClient, + ) + mux := http.NewServeMux() mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -53,6 +90,19 @@ func main() { } }() + workerCtx, workerCancel := context.WithCancel(context.Background()) + defer workerCancel() + + go func() { + logger.Info("fanout consumer started", + zap.String("consumer_group", "chronos-executor-fanout"), + zap.String("topic", "bucket-triggers"), + ) + if err := fanout.Run(workerCtx); err != nil { + logger.Fatal("fanout consumer exited with error", zap.Error(err)) + } + }() + quit := make(chan os.Signal, 1) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) sig := <-quit @@ -60,6 +110,7 @@ func main() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() + workerCancel() if err := srv.Shutdown(ctx); err != nil { logger.Error("http server shutdown error", zap.Error(err)) diff --git a/internal/executor/fanout_consumer.go b/internal/executor/fanout_consumer.go new file mode 100644 index 0000000..626399a --- /dev/null +++ b/internal/executor/fanout_consumer.go @@ -0,0 +1,140 @@ +package executor + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/redis/go-redis/v9" + kafkago "github.com/segmentio/kafka-go" + "go.uber.org/zap" + + kafkastore "github.com/chronos-scheduler/chronos/internal/store/kafka" + pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" + redisstore "github.com/chronos-scheduler/chronos/internal/store/redis" +) + +const schedulerDispatchIdempotencyTTL = 25 * time.Hour + +type BucketTriggerEvent struct { + BucketID string `json:"bucket_id"` +} + +type JobDispatchEvent struct { + JobID string `json:"job_id"` + JobDefID string `json:"job_def_id,omitempty"` + AttemptNumber int `json:"attempt_number"` + Payload json.RawMessage `json:"payload"` + DeliveryType string `json:"delivery_type"` + DeliveryConfig json.RawMessage `json:"delivery_config"` + SchemaVersion string `json:"schema_version"` +} + +type FanoutConsumer struct { + logger *zap.Logger + consumer *kafkastore.Consumer + producer *kafkastore.Producer + jobStore *pgstore.JobStore + redis *redis.Client +} + +func NewFanoutConsumer( + logger *zap.Logger, + consumer *kafkastore.Consumer, + producer *kafkastore.Producer, + jobStore *pgstore.JobStore, + redisClient *redis.Client, +) *FanoutConsumer { + return &FanoutConsumer{ + logger: logger, + consumer: consumer, + producer: producer, + jobStore: jobStore, + redis: redisClient, + } +} + +func (f *FanoutConsumer) Run(ctx context.Context) error { + return f.consumer.Consume(ctx, func(msg kafkago.Message) error { + var trigger BucketTriggerEvent + if err := json.Unmarshal(msg.Value, &trigger); err != nil { + return fmt.Errorf("decode bucket trigger: %w", err) + } + if trigger.BucketID == "" { + return fmt.Errorf("bucket trigger missing bucket_id") + } + return f.processBucket(ctx, trigger.BucketID) + }) +} + +func (f *FanoutConsumer) processBucket(ctx context.Context, bucketID string) error { + jobs, err := f.jobStore.ListByBucket(ctx, bucketID) + if err != nil { + return fmt.Errorf("list jobs by bucket %s: %w", bucketID, err) + } + if len(jobs) == 0 { + return nil + } + + dispatchedIDs := make([]string, 0, len(jobs)) + for _, job := range jobs { + ok, err := redisstore.SetIfNotExists(ctx, f.redis, redisstore.SchedulerDispatchIdempotencyKey(job.ID), schedulerDispatchIdempotencyTTL) + if err != nil { + return fmt.Errorf("set scheduler idempotency for job %s: %w", job.ID, err) + } + if !ok { + continue + } + + event := JobDispatchEvent{ + JobID: job.ID, + AttemptNumber: 1, + Payload: job.Payload, + DeliveryType: job.DeliveryType, + DeliveryConfig: job.DeliveryConfig, + SchemaVersion: "1", + } + if job.JobDefID != nil { + event.JobDefID = *job.JobDefID + } + + payload, err := json.Marshal(event) + if err != nil { + _ = f.flushDispatchedStatus(ctx, dispatchedIDs) + return fmt.Errorf("marshal job-dispatch event for job %s: %w", job.ID, err) + } + + key := job.ID + if job.JobDefID != nil && *job.JobDefID != "" { + key = *job.JobDefID + } + if err := f.producer.Publish(ctx, "job-dispatch", key, payload); err != nil { + _ = f.flushDispatchedStatus(ctx, dispatchedIDs) + return fmt.Errorf("publish job-dispatch for job %s: %w", job.ID, err) + } + + dispatchedIDs = append(dispatchedIDs, job.ID) + } + + if err := f.flushDispatchedStatus(ctx, dispatchedIDs); err != nil { + return err + } + + f.logger.Info("bucket fan-out complete", + zap.String("bucket_id", bucketID), + zap.Int("claimed_jobs", len(jobs)), + zap.Int("dispatched_jobs", len(dispatchedIDs)), + ) + return nil +} + +func (f *FanoutConsumer) flushDispatchedStatus(ctx context.Context, jobIDs []string) error { + if len(jobIDs) == 0 { + return nil + } + if err := f.jobStore.UpdateStatusByIDs(ctx, jobIDs, "DISPATCHED"); err != nil { + return fmt.Errorf("mark jobs dispatched: %w", err) + } + return nil +} From 3e51c824595de4ec43c59872c3ae4c4407b76757 Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 06:55:11 -0700 Subject: [PATCH 11/20] feat(job-executor): implement dispatch consumer and worker pool for job-dispatch --- cmd/job-executor/main.go | 59 ++++++++++++++++- internal/executor/dispatch_consumer.go | 55 ++++++++++++++++ internal/executor/pool.go | 88 ++++++++++++++++++++++++++ 3 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 internal/executor/dispatch_consumer.go create mode 100644 internal/executor/pool.go diff --git a/cmd/job-executor/main.go b/cmd/job-executor/main.go index 73e33cc..9346c58 100644 --- a/cmd/job-executor/main.go +++ b/cmd/job-executor/main.go @@ -6,6 +6,7 @@ import ( "net/http" "os" "os/signal" + "strconv" "syscall" "time" @@ -62,6 +63,12 @@ func main() { logger.Error("kafka consumer close error", zap.Error(err)) } }() + dispatchConsumerClient := kafkastore.NewConsumer(cfg.KafkaBrokers, "chronos-executor-dispatch", "job-dispatch") + defer func() { + if err := dispatchConsumerClient.Close(); err != nil { + logger.Error("dispatch kafka consumer close error", zap.Error(err)) + } + }() fanout := executor.NewFanoutConsumer( logger, @@ -70,6 +77,17 @@ func main() { pgstore.NewJobStore(db), redisClient, ) + workers := executorWorkersFromEnv() + queueSize := executorQueueSizeFromEnv(workers) + dispatchPool := executor.NewWorkerPool(workers, queueSize, func(ctx context.Context, task *executor.DispatchTask) error { + logger.Info("dispatch task received", + zap.String("job_id", task.Event.JobID), + zap.String("delivery_type", task.Event.DeliveryType), + zap.Int("attempt", task.Event.AttemptNumber), + ) + return nil + }) + dispatchConsumer := executor.NewDispatchConsumer(logger, dispatchConsumerClient, dispatchPool) mux := http.NewServeMux() mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { @@ -83,15 +101,26 @@ func main() { Handler: mux, } + workerCtx, workerCancel := context.WithCancel(context.Background()) + defer workerCancel() + go func() { logger.Info("http server listening", zap.String("addr", srv.Addr)) if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { logger.Fatal("http server error", zap.Error(err)) } }() - - workerCtx, workerCancel := context.WithCancel(context.Background()) - defer workerCancel() + go func() { + logger.Info("dispatch consumer started", + zap.String("consumer_group", "chronos-executor-dispatch"), + zap.String("topic", "job-dispatch"), + zap.Int("workers", workers), + zap.Int("queue_size", queueSize), + ) + if err := dispatchConsumer.Run(workerCtx); err != nil { + logger.Fatal("dispatch consumer exited with error", zap.Error(err)) + } + }() go func() { logger.Info("fanout consumer started", @@ -118,3 +147,27 @@ func main() { logger.Info("job-executor stopped") } + +func executorWorkersFromEnv() int { + raw := os.Getenv("EXECUTOR_WORKERS") + if raw == "" { + return 1 + } + n, err := strconv.Atoi(raw) + if err != nil || n <= 0 { + return 1 + } + return n +} + +func executorQueueSizeFromEnv(workers int) int { + raw := os.Getenv("EXECUTOR_QUEUE_SIZE") + if raw == "" { + return workers + } + n, err := strconv.Atoi(raw) + if err != nil || n <= 0 { + return workers + } + return n +} diff --git a/internal/executor/dispatch_consumer.go b/internal/executor/dispatch_consumer.go new file mode 100644 index 0000000..46023de --- /dev/null +++ b/internal/executor/dispatch_consumer.go @@ -0,0 +1,55 @@ +package executor + +import ( + "context" + "encoding/json" + "fmt" + + kafkastore "github.com/chronos-scheduler/chronos/internal/store/kafka" + kafkago "github.com/segmentio/kafka-go" + "go.uber.org/zap" +) + +type DispatchConsumer struct { + logger *zap.Logger + consumer *kafkastore.Consumer + pool *WorkerPool +} + +func NewDispatchConsumer(logger *zap.Logger, consumer *kafkastore.Consumer, pool *WorkerPool) *DispatchConsumer { + return &DispatchConsumer{ + logger: logger, + consumer: consumer, + pool: pool, + } +} + +func (d *DispatchConsumer) Run(ctx context.Context) error { + if err := d.pool.Validate(); err != nil { + return err + } + d.pool.Start(ctx) + defer d.pool.Close() + + return d.consumer.Consume(ctx, func(msg kafkago.Message) error { + var event JobDispatchEvent + if err := json.Unmarshal(msg.Value, &event); err != nil { + return fmt.Errorf("decode job-dispatch event: %w", err) + } + if event.JobID == "" { + return fmt.Errorf("job-dispatch event missing job_id") + } + + task := &DispatchTask{Message: msg, Event: event} + if err := d.pool.Enqueue(ctx, task); err != nil { + return fmt.Errorf("enqueue dispatch task: %w", err) + } + + d.logger.Debug("dispatch task enqueued", + zap.String("job_id", event.JobID), + zap.Int("queue_len", d.pool.QueueLen()), + zap.Int("queue_cap", d.pool.QueueCap()), + ) + return nil + }) +} diff --git a/internal/executor/pool.go b/internal/executor/pool.go new file mode 100644 index 0000000..5706850 --- /dev/null +++ b/internal/executor/pool.go @@ -0,0 +1,88 @@ +package executor + +import ( + "context" + "fmt" + "sync" + + kafkago "github.com/segmentio/kafka-go" +) + +type DispatchTask struct { + Message kafkago.Message + Event JobDispatchEvent +} + +type TaskProcessor func(ctx context.Context, task *DispatchTask) error + +type WorkerPool struct { + workers int + tasks chan *DispatchTask + processor TaskProcessor + wg sync.WaitGroup +} + +func NewWorkerPool(workers, queueSize int, processor TaskProcessor) *WorkerPool { + if workers <= 0 { + workers = 10 + } + if queueSize <= 0 { + queueSize = workers * 4 + } + return &WorkerPool{ + workers: workers, + tasks: make(chan *DispatchTask, queueSize), + processor: processor, + } +} + +func (p *WorkerPool) Start(ctx context.Context) { + for i := 0; i < p.workers; i++ { + p.wg.Add(1) + go func() { + defer p.wg.Done() + for { + select { + case <-ctx.Done(): + return + case task, ok := <-p.tasks: + if !ok { + return + } + if p.processor != nil { + _ = p.processor(ctx, task) + } + } + } + }() + } +} + +func (p *WorkerPool) Enqueue(ctx context.Context, task *DispatchTask) error { + select { + case <-ctx.Done(): + return ctx.Err() + case p.tasks <- task: + return nil + } +} + +func (p *WorkerPool) Close() { + close(p.tasks) + p.wg.Wait() +} + +func (p *WorkerPool) QueueLen() int { + return len(p.tasks) +} + +func (p *WorkerPool) QueueCap() int { + return cap(p.tasks) +} + +func (p *WorkerPool) Validate() error { + if p.processor == nil { + return fmt.Errorf("worker pool processor is required") + } + return nil +} From a0405e448c94251e6cf4bd5bd866dcc461ca6c5c Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 07:09:00 -0700 Subject: [PATCH 12/20] feat(scheduler): add adaptive shard-based bucket fan-out for scalable job dispatch --- internal/executor/fanout_consumer.go | 117 +++++++++++++++----------- internal/scheduler/dispatcher.go | 56 ++++++++++-- internal/scheduler/dispatcher_test.go | 49 +++++++++-- internal/store/postgres/job_store.go | 42 +++++++++ 4 files changed, 203 insertions(+), 61 deletions(-) diff --git a/internal/executor/fanout_consumer.go b/internal/executor/fanout_consumer.go index 626399a..3107246 100644 --- a/internal/executor/fanout_consumer.go +++ b/internal/executor/fanout_consumer.go @@ -16,9 +16,12 @@ import ( ) const schedulerDispatchIdempotencyTTL = 25 * time.Hour +const fanoutClaimBatchSize = 1000 type BucketTriggerEvent struct { - BucketID string `json:"bucket_id"` + BucketID string `json:"bucket_id"` + ShardID int `json:"shard_id"` + ShardCount int `json:"shard_count"` } type JobDispatchEvent struct { @@ -64,67 +67,85 @@ func (f *FanoutConsumer) Run(ctx context.Context) error { if trigger.BucketID == "" { return fmt.Errorf("bucket trigger missing bucket_id") } - return f.processBucket(ctx, trigger.BucketID) + if trigger.ShardCount <= 0 { + trigger.ShardCount = 1 + } + if trigger.ShardID < 0 || trigger.ShardID >= trigger.ShardCount { + trigger.ShardID = 0 + } + return f.processBucketShard(ctx, trigger.BucketID, trigger.ShardID, trigger.ShardCount) }) } -func (f *FanoutConsumer) processBucket(ctx context.Context, bucketID string) error { - jobs, err := f.jobStore.ListByBucket(ctx, bucketID) - if err != nil { - return fmt.Errorf("list jobs by bucket %s: %w", bucketID, err) - } - if len(jobs) == 0 { - return nil - } +func (f *FanoutConsumer) processBucketShard(ctx context.Context, bucketID string, shardID, shardCount int) error { + totalClaimed := 0 + totalDispatched := 0 - dispatchedIDs := make([]string, 0, len(jobs)) - for _, job := range jobs { - ok, err := redisstore.SetIfNotExists(ctx, f.redis, redisstore.SchedulerDispatchIdempotencyKey(job.ID), schedulerDispatchIdempotencyTTL) + for { + jobs, err := f.jobStore.ListByBucketShard(ctx, bucketID, shardID, shardCount, fanoutClaimBatchSize) if err != nil { - return fmt.Errorf("set scheduler idempotency for job %s: %w", job.ID, err) - } - if !ok { - continue + return fmt.Errorf("list jobs by bucket %s shard=%d/%d: %w", bucketID, shardID, shardCount, err) } - - event := JobDispatchEvent{ - JobID: job.ID, - AttemptNumber: 1, - Payload: job.Payload, - DeliveryType: job.DeliveryType, - DeliveryConfig: job.DeliveryConfig, - SchemaVersion: "1", + if len(jobs) == 0 { + break } - if job.JobDefID != nil { - event.JobDefID = *job.JobDefID + totalClaimed += len(jobs) + + dispatchedIDs := make([]string, 0, len(jobs)) + for _, job := range jobs { + ok, err := redisstore.SetIfNotExists(ctx, f.redis, redisstore.SchedulerDispatchIdempotencyKey(job.ID), schedulerDispatchIdempotencyTTL) + if err != nil { + return fmt.Errorf("set scheduler idempotency for job %s: %w", job.ID, err) + } + if !ok { + // Already faned out in a previous attempt; ensure row can progress. + dispatchedIDs = append(dispatchedIDs, job.ID) + continue + } + + event := JobDispatchEvent{ + JobID: job.ID, + AttemptNumber: 1, + Payload: job.Payload, + DeliveryType: job.DeliveryType, + DeliveryConfig: job.DeliveryConfig, + SchemaVersion: "1", + } + if job.JobDefID != nil { + event.JobDefID = *job.JobDefID + } + + payload, err := json.Marshal(event) + if err != nil { + _ = f.flushDispatchedStatus(ctx, dispatchedIDs) + return fmt.Errorf("marshal job-dispatch event for job %s: %w", job.ID, err) + } + + key := job.ID + if job.JobDefID != nil && *job.JobDefID != "" { + key = *job.JobDefID + } + if err := f.producer.Publish(ctx, "job-dispatch", key, payload); err != nil { + _ = f.flushDispatchedStatus(ctx, dispatchedIDs) + return fmt.Errorf("publish job-dispatch for job %s: %w", job.ID, err) + } + + dispatchedIDs = append(dispatchedIDs, job.ID) } - payload, err := json.Marshal(event) - if err != nil { - _ = f.flushDispatchedStatus(ctx, dispatchedIDs) - return fmt.Errorf("marshal job-dispatch event for job %s: %w", job.ID, err) - } - - key := job.ID - if job.JobDefID != nil && *job.JobDefID != "" { - key = *job.JobDefID + if err := f.flushDispatchedStatus(ctx, dispatchedIDs); err != nil { + return err } - if err := f.producer.Publish(ctx, "job-dispatch", key, payload); err != nil { - _ = f.flushDispatchedStatus(ctx, dispatchedIDs) - return fmt.Errorf("publish job-dispatch for job %s: %w", job.ID, err) - } - - dispatchedIDs = append(dispatchedIDs, job.ID) - } - if err := f.flushDispatchedStatus(ctx, dispatchedIDs); err != nil { - return err + totalDispatched += len(dispatchedIDs) } - f.logger.Info("bucket fan-out complete", + f.logger.Info("bucket shard fan-out complete", zap.String("bucket_id", bucketID), - zap.Int("claimed_jobs", len(jobs)), - zap.Int("dispatched_jobs", len(dispatchedIDs)), + zap.Int("shard_id", shardID), + zap.Int("shard_count", shardCount), + zap.Int("claimed_jobs", totalClaimed), + zap.Int("dispatched_jobs", totalDispatched), ) return nil } diff --git a/internal/scheduler/dispatcher.go b/internal/scheduler/dispatcher.go index 6b833cb..3fa701f 100644 --- a/internal/scheduler/dispatcher.go +++ b/internal/scheduler/dispatcher.go @@ -3,6 +3,7 @@ package scheduler import ( "context" "encoding/json" + "fmt" "time" pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" @@ -11,10 +12,14 @@ import ( const ( defaultBucketSeconds = 60 defaultBucketScanMax = 100 + defaultShardCount = 128 + targetRowsPerShard = 1000 ) type BucketTriggerEvent struct { - BucketID string `json:"bucket_id"` + BucketID string `json:"bucket_id"` + ShardID int `json:"shard_id"` + ShardCount int `json:"shard_count"` } type bucketStore interface { @@ -22,28 +27,41 @@ type bucketStore interface { MarkFired(ctx context.Context, id string) error } +type jobCounter interface { + CountPendingByBucket(ctx context.Context, bucketID string) (int, error) +} + type kafkaPublisher interface { Publish(ctx context.Context, topic, key string, value []byte) error } type Dispatcher struct { bucketStore bucketStore + jobCounter jobCounter kafkaProducer kafkaPublisher bucketSeconds int + shardCount int } func NewDispatcher( bucketStore bucketStore, + jobCounter jobCounter, kafkaProducer kafkaPublisher, bucketSeconds int, + shardCount int, ) *Dispatcher { if bucketSeconds <= 0 { bucketSeconds = defaultBucketSeconds } + if shardCount <= 0 { + shardCount = defaultShardCount + } return &Dispatcher{ bucketStore: bucketStore, + jobCounter: jobCounter, kafkaProducer: kafkaProducer, bucketSeconds: bucketSeconds, + shardCount: shardCount, } } @@ -82,15 +100,39 @@ func (d *Dispatcher) dispatchDueBuckets(ctx context.Context, upTo time.Time) err } func (d *Dispatcher) dispatchSingleBucket(ctx context.Context, bucket *pgstore.ScheduleBucket) error { - event := BucketTriggerEvent{ - BucketID: bucket.ID, - } - payload, err := json.Marshal(event) + pendingJobs, err := d.jobCounter.CountPendingByBucket(ctx, bucket.ID) if err != nil { return err } - if err := d.kafkaProducer.Publish(ctx, "bucket-triggers", bucket.ID, payload); err != nil { - return err + effectiveShards := d.computeShardCount(pendingJobs) + for shardID := 0; shardID < effectiveShards; shardID++ { + event := BucketTriggerEvent{ + BucketID: bucket.ID, + ShardID: shardID, + ShardCount: effectiveShards, + } + payload, err := json.Marshal(event) + if err != nil { + return err + } + key := fmt.Sprintf("%s:%d", bucket.ID, shardID) + if err := d.kafkaProducer.Publish(ctx, "bucket-triggers", key, payload); err != nil { + return err + } } return d.bucketStore.MarkFired(ctx, bucket.ID) } + +func (d *Dispatcher) computeShardCount(pendingJobs int) int { + if pendingJobs <= 0 { + return 0 + } + shards := (pendingJobs + targetRowsPerShard - 1) / targetRowsPerShard + if shards < 1 { + shards = 1 + } + if shards > d.shardCount { + shards = d.shardCount + } + return shards +} diff --git a/internal/scheduler/dispatcher_test.go b/internal/scheduler/dispatcher_test.go index 4fa7df8..6c6e33c 100644 --- a/internal/scheduler/dispatcher_test.go +++ b/internal/scheduler/dispatcher_test.go @@ -16,6 +16,14 @@ type fakeBucketStore struct { listCalled int } +type fakeJobCounter struct { + pendingByBucket map[string]int +} + +func (c *fakeJobCounter) CountPendingByBucket(_ context.Context, bucketID string) (int, error) { + return c.pendingByBucket[bucketID], nil +} + func (s *fakeBucketStore) ListPending(_ context.Context, upTo time.Time, _ int) ([]*pgstore.ScheduleBucket, error) { s.listCalled++ out := make([]*pgstore.ScheduleBucket, 0) @@ -67,15 +75,16 @@ func TestDispatcher_PublishesOnlyDueBuckets(t *testing.T) { {ID: "already-fired", FireAt: now.Add(-2 * time.Minute), Status: "FIRED"}, }} pub := &fakePublisher{} - d := NewDispatcher(store, pub, 60) + counter := &fakeJobCounter{pendingByBucket: map[string]int{"due-1": 2500}} + d := NewDispatcher(store, counter, pub, 60, 4) if err := d.dispatchDueBuckets(context.Background(), now); err != nil { t.Fatalf("dispatchDueBuckets: %v", err) } - if len(pub.messages) != 1 { - t.Fatalf("published messages: got %d want 1", len(pub.messages)) + if len(pub.messages) != 3 { + t.Fatalf("published messages: got %d want 3", len(pub.messages)) } - if pub.messages[0].topic != "bucket-triggers" || pub.messages[0].key != "due-1" { + if pub.messages[0].topic != "bucket-triggers" || pub.messages[0].key != "due-1:0" { t.Fatalf("published message mismatch: %+v", pub.messages[0]) } @@ -86,6 +95,12 @@ func TestDispatcher_PublishesOnlyDueBuckets(t *testing.T) { if event.BucketID != "due-1" { t.Fatalf("event bucket_id: got %q want %q", event.BucketID, "due-1") } + if event.ShardCount != 3 { + t.Fatalf("event shard_count: got %d want 3", event.ShardCount) + } + if event.ShardID != 0 { + t.Fatalf("event shard_id: got %d want 0", event.ShardID) + } } func TestDispatcher_MarksBucketFiredAfterPublish(t *testing.T) { @@ -94,7 +109,8 @@ func TestDispatcher_MarksBucketFiredAfterPublish(t *testing.T) { {ID: "due-2", FireAt: now.Add(-time.Second), Status: "PENDING"}, }} pub := &fakePublisher{} - d := NewDispatcher(store, pub, 60) + counter := &fakeJobCounter{pendingByBucket: map[string]int{"due-2": 10}} + d := NewDispatcher(store, counter, pub, 60, 4) if err := d.dispatchDueBuckets(context.Background(), now); err != nil { t.Fatalf("dispatchDueBuckets: %v", err) @@ -113,7 +129,8 @@ func TestDispatcher_PublishFailureKeepsBucketPending(t *testing.T) { {ID: "due-3", FireAt: now.Add(-time.Second), Status: "PENDING"}, }} pub := &fakePublisher{err: errors.New("kafka unavailable")} - d := NewDispatcher(store, pub, 60) + counter := &fakeJobCounter{pendingByBucket: map[string]int{"due-3": 10}} + d := NewDispatcher(store, counter, pub, 60, 4) err := d.dispatchDueBuckets(context.Background(), now) if err == nil { @@ -126,3 +143,23 @@ func TestDispatcher_PublishFailureKeepsBucketPending(t *testing.T) { t.Fatalf("bucket status: got %q want PENDING", store.buckets[0].Status) } } + +func TestDispatcher_EmptyBucketPublishesNoShards(t *testing.T) { + now := time.Date(2026, 4, 21, 10, 5, 0, 0, time.UTC) + store := &fakeBucketStore{buckets: []*pgstore.ScheduleBucket{ + {ID: "due-empty", FireAt: now.Add(-time.Second), Status: "PENDING"}, + }} + pub := &fakePublisher{} + counter := &fakeJobCounter{pendingByBucket: map[string]int{"due-empty": 0}} + d := NewDispatcher(store, counter, pub, 60, 4) + + if err := d.dispatchDueBuckets(context.Background(), now); err != nil { + t.Fatalf("dispatchDueBuckets: %v", err) + } + if len(pub.messages) != 0 { + t.Fatalf("expected no published messages, got %d", len(pub.messages)) + } + if len(store.firedIDs) != 1 || store.firedIDs[0] != "due-empty" { + t.Fatalf("fired ids: got %#v", store.firedIDs) + } +} diff --git a/internal/store/postgres/job_store.go b/internal/store/postgres/job_store.go index 09e9670..70e8af8 100644 --- a/internal/store/postgres/job_store.go +++ b/internal/store/postgres/job_store.go @@ -88,6 +88,16 @@ func (s *JobStore) GetByIdempotencyKey(ctx context.Context, key string) (*Job, e return &j, nil } +func (s *JobStore) CountPendingByBucket(ctx context.Context, bucketID string) (int, error) { + var count int + if err := s.db.GetContext(ctx, &count, + `SELECT COUNT(*) FROM jobs WHERE bucket_id = $1 AND status = 'PENDING'`, + bucketID); err != nil { + return 0, err + } + return count, nil +} + func (s *JobStore) List(ctx context.Context, f ListJobsFilter) ([]*Job, error) { if f.Limit <= 0 { f.Limit = 50 @@ -147,6 +157,38 @@ func (s *JobStore) ListByBucket(ctx context.Context, bucketID string) ([]*Job, e return jobs, err } +// ListByBucketShard returns pending jobs for one shard of a bucket. +// Sharding is deterministic on job ID hash to allow parallel fan-out workers to +// process the same bucket safely with disjoint keyspaces. +// +// TODO(scale): For very large buckets, replace runtime `(abs(hashtext(id::text)) % shard_count)` +// filtering with a precomputed `fanout_shard` column (set at insert time) and an +// index like `(bucket_id, status, fanout_shard, created_at)` to avoid per-row hash +// computation in hot fan-out queries. +func (s *JobStore) ListByBucketShard(ctx context.Context, bucketID string, shardID, shardCount, limit int) ([]*Job, error) { + if shardCount <= 0 { + shardCount = 1 + } + if shardID < 0 { + shardID = 0 + } + if limit <= 0 { + limit = 1000 + } + + var jobs []*Job + err := s.db.SelectContext(ctx, &jobs, + `SELECT * FROM jobs + WHERE bucket_id = $1 + AND status = 'PENDING' + AND (abs(hashtext(id::text)) % $3) = $2 + ORDER BY created_at ASC + LIMIT $4 + FOR UPDATE SKIP LOCKED`, + bucketID, shardID, shardCount, limit) + return jobs, err +} + // ListByBulkJob returns BULK_RECORD jobs for a given upload, optionally filtered by status. // Ordered by row_start so batches are processed in CSV order. // Uses FOR UPDATE SKIP LOCKED for concurrent bulk-executor worker claiming. From 882bc7b3557b88dc42c557fb1cab6d10fbea43bd Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 07:17:33 -0700 Subject: [PATCH 13/20] feat(job-executor): implement HTTP dispatcher with timeout and response classification --- internal/executor/dispatch/http.go | 137 +++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 internal/executor/dispatch/http.go diff --git a/internal/executor/dispatch/http.go b/internal/executor/dispatch/http.go new file mode 100644 index 0000000..a056f82 --- /dev/null +++ b/internal/executor/dispatch/http.go @@ -0,0 +1,137 @@ +package dispatch + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" +) + +const defaultTimeoutSeconds = 30 +const responseBodyPreviewLimit = 1024 + +type JobDefinition struct { + JobID string + DeliveryConfig []byte + TimeoutSeconds int +} + +type DispatchResult struct { + StatusCode int + DurationMS int + ResponseBodyPreview string + Retriable bool + PermanentFailure bool + TimedOut bool +} + +type httpDeliveryConfig struct { + URL string `json:"url"` + Method string `json:"method"` + HeadersTemplate map[string]string `json:"headers_template"` +} + +func Dispatch(ctx context.Context, job *JobDefinition, payload []byte, execID string, attempt int) (*DispatchResult, error) { + cfg, err := parseHTTPConfig(job) + if err != nil { + return nil, err + } + + timeout := job.TimeoutSeconds + if timeout <= 0 { + timeout = defaultTimeoutSeconds + } + timeoutCtx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(timeoutCtx, cfg.Method, cfg.URL, bytes.NewReader(payload)) + if err != nil { + return nil, fmt.Errorf("build http request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + for k, v := range cfg.HeadersTemplate { + req.Header.Set(k, v) + } + req.Header.Set("X-Job-Id", job.JobID) + req.Header.Set("X-Execution-Id", execID) + req.Header.Set("X-Attempt-Number", fmt.Sprintf("%d", attempt)) + + client := &http.Client{} + start := time.Now() + resp, err := client.Do(req) + durationMS := int(time.Since(start).Milliseconds()) + if err != nil { + res := &DispatchResult{DurationMS: durationMS, Retriable: true} + if errors.Is(err, context.DeadlineExceeded) || isNetTimeout(err) { + res.TimedOut = true + } + return res, nil + } + defer resp.Body.Close() + + preview, _ := readResponsePreview(resp.Body, responseBodyPreviewLimit) + result := &DispatchResult{ + StatusCode: resp.StatusCode, + DurationMS: durationMS, + ResponseBodyPreview: preview, + } + + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return result, nil + case resp.StatusCode >= 400 && resp.StatusCode < 500: + result.PermanentFailure = true + return result, nil + default: + result.Retriable = true + return result, nil + } +} + +func parseHTTPConfig(job *JobDefinition) (*httpDeliveryConfig, error) { + if job == nil { + return nil, fmt.Errorf("job definition is required") + } + if job.JobID == "" { + return nil, fmt.Errorf("job id is required") + } + if len(job.DeliveryConfig) == 0 { + return nil, fmt.Errorf("delivery config is required") + } + + cfg := &httpDeliveryConfig{} + if err := json.Unmarshal(job.DeliveryConfig, cfg); err != nil { + return nil, fmt.Errorf("parse delivery config: %w", err) + } + cfg.URL = strings.TrimSpace(cfg.URL) + if cfg.URL == "" { + return nil, fmt.Errorf("delivery config missing url") + } + cfg.Method = strings.ToUpper(strings.TrimSpace(cfg.Method)) + if cfg.Method == "" { + cfg.Method = http.MethodPost + } + if cfg.HeadersTemplate == nil { + cfg.HeadersTemplate = map[string]string{} + } + return cfg, nil +} + +func readResponsePreview(r io.Reader, limit int64) (string, error) { + body, err := io.ReadAll(io.LimitReader(r, limit)) + if err != nil { + return "", err + } + return string(body), nil +} + +func isNetTimeout(err error) bool { + var ne net.Error + return errors.As(err, &ne) && ne.Timeout() +} From c5fbe1c22e66b3a49eef60b4b653972f770790f3 Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 07:24:57 -0700 Subject: [PATCH 14/20] feat(job-executor): implement Kafka dispatcher for Kafka-delivery jobs --- internal/executor/dispatch/kafka.go | 69 +++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 internal/executor/dispatch/kafka.go diff --git a/internal/executor/dispatch/kafka.go b/internal/executor/dispatch/kafka.go new file mode 100644 index 0000000..1c3fea6 --- /dev/null +++ b/internal/executor/dispatch/kafka.go @@ -0,0 +1,69 @@ +package dispatch + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + kafkastore "github.com/chronos-scheduler/chronos/internal/store/kafka" +) + +type KafkaDispatcher struct { + producer *kafkastore.Producer +} + +type kafkaDeliveryConfig struct { + Topic string `json:"topic"` +} + +func NewKafkaDispatcher(producer *kafkastore.Producer) *KafkaDispatcher { + return &KafkaDispatcher{producer: producer} +} + +func (d *KafkaDispatcher) Dispatch(ctx context.Context, job *JobDefinition, payload []byte) (*DispatchResult, error) { + if d == nil || d.producer == nil { + return nil, fmt.Errorf("kafka producer is required") + } + cfg, err := parseKafkaConfig(job) + if err != nil { + return nil, err + } + + key := job.JobID + if key == "" { + key = "adhoc" + } + + start := time.Now() + if err := d.producer.Publish(ctx, cfg.Topic, key, payload); err != nil { + return &DispatchResult{ + DurationMS: int(time.Since(start).Milliseconds()), + Retriable: true, + }, nil + } + + return &DispatchResult{ + DurationMS: int(time.Since(start).Milliseconds()), + }, nil +} + +func parseKafkaConfig(job *JobDefinition) (*kafkaDeliveryConfig, error) { + if job == nil { + return nil, fmt.Errorf("job definition is required") + } + if len(job.DeliveryConfig) == 0 { + return nil, fmt.Errorf("delivery config is required") + } + + cfg := &kafkaDeliveryConfig{} + if err := json.Unmarshal(job.DeliveryConfig, cfg); err != nil { + return nil, fmt.Errorf("parse kafka delivery config: %w", err) + } + cfg.Topic = strings.TrimSpace(cfg.Topic) + if cfg.Topic == "" { + return nil, fmt.Errorf("delivery config missing topic") + } + return cfg, nil +} From 755cfbca1e2af2406c4279389e23a94de58e2197 Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 07:38:31 -0700 Subject: [PATCH 15/20] feat(job-executor): implement result reporting to Postgres, MongoDB, and Kafka 1. Denormalized retry/timeout onto jobs (so executor is self-sufficient) 2. Added persistent next_retry_at on job_executions --- .dockerignore | 1 + .gitignore | 1 + cmd/job-executor/main.go | 31 ++- .../api/http/handlers/schedule_handler.go | 86 +++++-- internal/executor/processor.go | 96 +++++++ internal/executor/result.go | 241 ++++++++++++++++++ .../store/postgres/job_execution_store.go | 16 ++ internal/store/postgres/job_store.go | 54 ++-- ...etry_config_and_add_next_retry_at.down.sql | 8 + ..._retry_config_and_add_next_retry_at.up.sql | 8 + 10 files changed, 492 insertions(+), 50 deletions(-) create mode 100644 internal/executor/processor.go create mode 100644 internal/executor/result.go create mode 100644 migrations/postgres/0020_denormalize_retry_config_and_add_next_retry_at.down.sql create mode 100644 migrations/postgres/0020_denormalize_retry_config_and_add_next_retry_at.up.sql diff --git a/.dockerignore b/.dockerignore index 900203c..d525e05 100644 --- a/.dockerignore +++ b/.dockerignore @@ -23,3 +23,4 @@ GEMINI.md Makefile /.tmp/ +/.cache/ diff --git a/.gitignore b/.gitignore index 6566944..9c730ab 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ vendor/ go.work go.work.sum /.tmp/ +/.cache/ # Built binaries bin/ diff --git a/cmd/job-executor/main.go b/cmd/job-executor/main.go index 9346c58..d60737f 100644 --- a/cmd/job-executor/main.go +++ b/cmd/job-executor/main.go @@ -14,7 +14,9 @@ import ( "github.com/chronos-scheduler/chronos/internal/config" "github.com/chronos-scheduler/chronos/internal/executor" + "github.com/chronos-scheduler/chronos/internal/executor/dispatch" kafkastore "github.com/chronos-scheduler/chronos/internal/store/kafka" + mongostore "github.com/chronos-scheduler/chronos/internal/store/mongo" pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" redisstore "github.com/chronos-scheduler/chronos/internal/store/redis" "github.com/chronos-scheduler/chronos/internal/telemetry" @@ -45,6 +47,15 @@ func main() { } logger.Info("postgres connected") + mongoClient, err := mongostore.Connect(cfg.MongoURI) + if err != nil { + logger.Fatal("mongodb connection failed", zap.Error(err)) + } + defer func() { + _ = mongoClient.Disconnect(context.Background()) + }() + logger.Info("mongodb connected") + redisClient, err := redisstore.Connect(cfg.RedisAddr) if err != nil { logger.Fatal("redis connection failed", zap.Error(err)) @@ -77,14 +88,24 @@ func main() { pgstore.NewJobStore(db), redisClient, ) + + jobStore := pgstore.NewJobStore(db) + executionStore := pgstore.NewJobExecutionStore(db) + logStore := mongostore.NewLogStore(mongoClient, "chronos") + kafkaDispatcher := dispatch.NewKafkaDispatcher(kafkaProducer) + workerID, _ := os.Hostname() + reporter := executor.NewResultReporter(logger, jobStore, executionStore, logStore, kafkaProducer, workerID) + taskProcessor := executor.NewTaskProcessor(logger, jobStore, executionStore, kafkaDispatcher, reporter) + workers := executorWorkersFromEnv() queueSize := executorQueueSizeFromEnv(workers) dispatchPool := executor.NewWorkerPool(workers, queueSize, func(ctx context.Context, task *executor.DispatchTask) error { - logger.Info("dispatch task received", - zap.String("job_id", task.Event.JobID), - zap.String("delivery_type", task.Event.DeliveryType), - zap.Int("attempt", task.Event.AttemptNumber), - ) + if err := taskProcessor.Process(ctx, task); err != nil { + logger.Error("dispatch task failed", + zap.String("job_id", task.Event.JobID), + zap.Error(err), + ) + } return nil }) dispatchConsumer := executor.NewDispatchConsumer(logger, dispatchConsumerClient, dispatchPool) diff --git a/internal/api/http/handlers/schedule_handler.go b/internal/api/http/handlers/schedule_handler.go index eeb830d..7bce661 100644 --- a/internal/api/http/handlers/schedule_handler.go +++ b/internal/api/http/handlers/schedule_handler.go @@ -51,7 +51,7 @@ func (h *ScheduleHandler) CreateSchedule(c *gin.Context) { return } - deliveryType, deliveryConfig, jobDefID, err := h.resolveDelivery(c.Request.Context(), req) + plan, err := h.resolveDispatchPlan(c.Request.Context(), req) if err != nil { if errors.Is(err, sql.ErrNoRows) { response.RespondNotFound(c, "job definition not found") @@ -75,26 +75,25 @@ func (h *ScheduleHandler) CreateSchedule(c *gin.Context) { contextPayload := []byte(`{}`) jobTags := pq.StringArray(resolvedTags(req.Tags, nil)) - if jobDefID != nil { - def, err := h.jobDefStore.GetByID(c.Request.Context(), *jobDefID) - if err != nil { - response.RespondInternalError(c, "failed to resolve job definition tags") - return - } - jobTags = pq.StringArray(resolvedTags(req.Tags, []string(def.Tags))) + if plan.JobDefinition != nil { + jobTags = pq.StringArray(resolvedTags(req.Tags, []string(plan.JobDefinition.Tags))) } job := &pgstore.Job{ - JobType: "ONE_TIME", - JobDefID: jobDefID, - BucketID: &bucket.ID, - DeliveryType: deliveryType, - DeliveryConfig: deliveryConfig, - Payload: payload, - Context: contextPayload, - ScheduledAt: ptrTime(req.ScheduledAt.UTC()), - Status: "PENDING", - Tags: jobTags, + JobType: "ONE_TIME", + JobDefID: plan.JobDefinitionID, + BucketID: &bucket.ID, + DeliveryType: plan.DeliveryType, + DeliveryConfig: plan.DeliveryConfig, + TimeoutSeconds: plan.TimeoutSeconds, + MaxRetries: plan.MaxRetries, + RetryStrategy: plan.RetryStrategy, + RetryBaseDelayS: plan.RetryBaseDelayS, + Payload: payload, + Context: contextPayload, + ScheduledAt: ptrTime(req.ScheduledAt.UTC()), + Status: "PENDING", + Tags: jobTags, } if strings.TrimSpace(req.IdempotencyKey) != "" { key := strings.TrimSpace(req.IdempotencyKey) @@ -231,23 +230,58 @@ func (h *ScheduleHandler) CancelSchedule(c *gin.Context) { c.Status(http.StatusNoContent) } -func (h *ScheduleHandler) resolveDelivery(ctx context.Context, req apitypes.CreateScheduleRequest) (string, []byte, *string, error) { +type scheduleDispatchPlan struct { + DeliveryType string + DeliveryConfig []byte + JobDefinitionID *string + JobDefinition *pgstore.JobDefinition + TimeoutSeconds int + MaxRetries int + RetryStrategy string + RetryBaseDelayS int +} + +func (h *ScheduleHandler) resolveDispatchPlan(ctx context.Context, req apitypes.CreateScheduleRequest) (*scheduleDispatchPlan, error) { if strings.TrimSpace(req.JobDefinitionID) != "" { id := strings.TrimSpace(req.JobDefinitionID) def, err := h.jobDefStore.GetByID(ctx, id) if err != nil { - return "", nil, nil, err + return nil, err } - return def.DeliveryType, def.DeliveryConfig, &def.ID, nil - } - - config, _, err := deliveryConfigFromRequest(req.DeliveryMode, req.Endpoint, req.HTTPMethod, req.HTTPHeaders, req.RetryPolicy.BackoffArray) + return &scheduleDispatchPlan{ + DeliveryType: def.DeliveryType, + DeliveryConfig: def.DeliveryConfig, + JobDefinitionID: &def.ID, + JobDefinition: def, + TimeoutSeconds: def.TimeoutSeconds, + MaxRetries: def.MaxRetries, + RetryStrategy: def.RetryStrategy, + RetryBaseDelayS: def.RetryBaseDelayS, + }, nil + } + + config, retryBaseDelay, err := deliveryConfigFromRequest(req.DeliveryMode, req.Endpoint, req.HTTPMethod, req.HTTPHeaders, req.RetryPolicy.BackoffArray) if err != nil { - return "", nil, nil, err + return nil, err } mode := strings.ToUpper(strings.TrimSpace(req.DeliveryMode)) - return mode, config, nil, nil + timeout := req.TimeoutSeconds + if timeout <= 0 { + timeout = 30 + } + maxRetries := req.RetryPolicy.MaxAttempts + if maxRetries < 0 { + maxRetries = 0 + } + return &scheduleDispatchPlan{ + DeliveryType: mode, + DeliveryConfig: config, + TimeoutSeconds: timeout, + MaxRetries: maxRetries, + RetryStrategy: "exponential", + RetryBaseDelayS: retryBaseDelay, + }, nil } func toScheduleResponse(job *pgstore.Job, execs []*pgstore.JobExecution) apitypes.ScheduleResponse { diff --git a/internal/executor/processor.go b/internal/executor/processor.go new file mode 100644 index 0000000..ccc4ff2 --- /dev/null +++ b/internal/executor/processor.go @@ -0,0 +1,96 @@ +package executor + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/chronos-scheduler/chronos/internal/executor/dispatch" + pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" + "go.uber.org/zap" +) + +type TaskProcessorImpl struct { + logger *zap.Logger + jobStore *pgstore.JobStore + executionStore *pgstore.JobExecutionStore + httpDispatcher func(ctx context.Context, job *dispatch.JobDefinition, payload []byte, execID string, attempt int) (*dispatch.DispatchResult, error) + kafkaDispatcher *dispatch.KafkaDispatcher + reporter *ResultReporter + workerID string +} + +func NewTaskProcessor( + logger *zap.Logger, + jobStore *pgstore.JobStore, + executionStore *pgstore.JobExecutionStore, + kafkaDispatcher *dispatch.KafkaDispatcher, + reporter *ResultReporter, +) *TaskProcessorImpl { + workerID, _ := os.Hostname() + return &TaskProcessorImpl{ + logger: logger, + jobStore: jobStore, + executionStore: executionStore, + httpDispatcher: dispatch.Dispatch, + kafkaDispatcher: kafkaDispatcher, + reporter: reporter, + workerID: workerID, + } +} + +func (p *TaskProcessorImpl) Process(ctx context.Context, task *DispatchTask) error { + job, err := p.jobStore.GetByID(ctx, task.Event.JobID) + if err != nil { + return fmt.Errorf("load job %s: %w", task.Event.JobID, err) + } + + attempt := task.Event.AttemptNumber + if attempt <= 0 { + attempt = 1 + } + exec := &pgstore.JobExecution{ + JobID: job.ID, + AttemptNumber: attempt, + Status: "EXECUTING", + WorkerID: &p.workerID, + } + if err := p.executionStore.Create(ctx, exec); err != nil { + return fmt.Errorf("create execution: %w", err) + } + + dispatchDef := &dispatch.JobDefinition{ + JobID: job.ID, + DeliveryConfig: job.DeliveryConfig, + TimeoutSeconds: job.TimeoutSeconds, + } + + var result *dispatch.DispatchResult + switch strings.ToUpper(strings.TrimSpace(job.DeliveryType)) { + case "HTTP": + result, err = p.httpDispatcher(ctx, dispatchDef, job.Payload, exec.ID, attempt) + case "KAFKA": + result, err = p.kafkaDispatcher.Dispatch(ctx, dispatchDef, job.Payload) + default: + return fmt.Errorf("unsupported delivery type: %s", job.DeliveryType) + } + if result == nil { + result = &dispatch.DispatchResult{Retriable: true} + } + + if reportErr := p.reporter.Report(ctx, job, exec, result, err); reportErr != nil { + return reportErr + } + + p.logger.Info("dispatch processed", + zap.String("job_id", job.ID), + zap.String("execution_id", exec.ID), + zap.String("delivery_type", job.DeliveryType), + zap.Int("attempt", attempt), + zap.Int("status_code", result.StatusCode), + zap.Bool("retriable", result.Retriable), + zap.Bool("permanent_failure", result.PermanentFailure), + ) + return nil +} diff --git a/internal/executor/result.go b/internal/executor/result.go new file mode 100644 index 0000000..308a031 --- /dev/null +++ b/internal/executor/result.go @@ -0,0 +1,241 @@ +package executor + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/chronos-scheduler/chronos/internal/executor/dispatch" + kafkastore "github.com/chronos-scheduler/chronos/internal/store/kafka" + mongostore "github.com/chronos-scheduler/chronos/internal/store/mongo" + pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" + "go.uber.org/zap" +) + +const jobEventsTopic = "job-events" + +type ResultReporter struct { + logger *zap.Logger + jobStore *pgstore.JobStore + executionStore *pgstore.JobExecutionStore + logStore *mongostore.LogStore + producer *kafkastore.Producer + workerID string +} + +func NewResultReporter( + logger *zap.Logger, + jobStore *pgstore.JobStore, + executionStore *pgstore.JobExecutionStore, + logStore *mongostore.LogStore, + producer *kafkastore.Producer, + workerID string, +) *ResultReporter { + return &ResultReporter{ + logger: logger, + jobStore: jobStore, + executionStore: executionStore, + logStore: logStore, + producer: producer, + workerID: workerID, + } +} + +func (r *ResultReporter) Report( + ctx context.Context, + job *pgstore.Job, + exec *pgstore.JobExecution, + result *dispatch.DispatchResult, + dispatchErr error, +) error { + if job == nil || exec == nil || result == nil { + return fmt.Errorf("report requires job, execution, and result") + } + + now := time.Now().UTC() + var ( + jobStatus string + execStatus string + eventType string + eventStatus string + errorMessage *string + nextRetryAt *time.Time + responseStatus *int + ) + + if result.StatusCode > 0 { + responseStatus = &result.StatusCode + } + + switch { + case dispatchErr == nil && !result.Retriable && !result.PermanentFailure && !result.TimedOut: + jobStatus = "SUCCEEDED" + execStatus = "SUCCEEDED" + eventType = "JOB_COMPLETED" + eventStatus = "SUCCEEDED" + case result.PermanentFailure: + jobStatus = "DEAD" + execStatus = "DEAD" + eventType = "JOB_DLQ" + eventStatus = "DEAD" + msg := fmt.Sprintf("permanent failure: status=%d", result.StatusCode) + errorMessage = &msg + case result.TimedOut: + jobStatus = "FAILED" + execStatus = "TIMED_OUT" + eventType = "JOB_FAILED" + eventStatus = "TIMED_OUT" + msg := "dispatch timeout" + errorMessage = &msg + nextRetryAt = computeNextRetryAt(now, exec.AttemptNumber, job) + default: + jobStatus = "FAILED" + execStatus = "FAILED" + eventType = "JOB_FAILED" + eventStatus = "FAILED" + msg := "dispatch failed" + if dispatchErr != nil { + msg = dispatchErr.Error() + } else if result.StatusCode > 0 { + msg = fmt.Sprintf("dispatch failed: status=%d", result.StatusCode) + } + errorMessage = &msg + nextRetryAt = computeNextRetryAt(now, exec.AttemptNumber, job) + } + + if err := r.executionStore.UpdateTerminal(ctx, exec.ID, execStatus, responseStatus, errorMessage, result.DurationMS, nextRetryAt); err != nil { + return fmt.Errorf("update execution terminal status: %w", err) + } + if err := r.jobStore.UpdateStatus(ctx, job.ID, jobStatus); err != nil { + return fmt.Errorf("update job status: %w", err) + } + + if err := r.logExecution(ctx, job, exec, result, execStatus, errorMessage); err != nil { + return err + } + + event := map[string]any{ + "type": eventType, + "job_id": job.ID, + "execution_id": exec.ID, + "attempt_number": exec.AttemptNumber, + "status": eventStatus, + "duration_ms": result.DurationMS, + "schema_version": "1", + } + if job.JobDefID != nil && *job.JobDefID != "" { + event["job_def_id"] = *job.JobDefID + } + if responseStatus != nil { + event["response_status"] = *responseStatus + } + if errorMessage != nil { + event["error_message"] = *errorMessage + } + if nextRetryAt != nil { + event["next_retry_at"] = nextRetryAt.Format(time.RFC3339) + } + if eventType == "JOB_COMPLETED" { + event["completed_at"] = now.Format(time.RFC3339Nano) + } + + payload, err := json.Marshal(event) + if err != nil { + return fmt.Errorf("marshal job event: %w", err) + } + if err := r.producer.Publish(ctx, jobEventsTopic, job.ID, payload); err != nil { + return fmt.Errorf("publish job event: %w", err) + } + return nil +} + +func (r *ResultReporter) logExecution( + ctx context.Context, + job *pgstore.Job, + exec *pgstore.JobExecution, + result *dispatch.DispatchResult, + status string, + errorMessage *string, +) error { + log := &mongostore.ExecutionLog{ + ExecutionID: exec.ID, + ScheduledJobID: job.ID, + AttemptNumber: exec.AttemptNumber, + Status: status, + DeliveryMode: strings.ToUpper(strings.TrimSpace(job.DeliveryType)), + Error: errorMessage, + WorkerID: r.workerID, + CreatedAt: time.Now().UTC(), + } + if job.JobDefID != nil && *job.JobDefID != "" { + log.JobDefID = *job.JobDefID + } + + switch log.DeliveryMode { + case "HTTP": + method, url := extractHTTPLogFields(job.DeliveryConfig) + log.Request = &mongostore.HTTPRequest{ + URL: url, + Method: method, + Headers: map[string]string{}, + BodySizeBytes: len(job.Payload), + } + if result.StatusCode > 0 || result.ResponseBodyPreview != "" { + log.Response = &mongostore.HTTPResponse{ + StatusCode: result.StatusCode, + Headers: map[string]string{}, + BodyPreview: result.ResponseBodyPreview, + LatencyMs: result.DurationMS, + } + } + case "KAFKA": + topic := extractKafkaTopic(job.DeliveryConfig) + log.KafkaDelivery = &mongostore.KafkaDelivery{ + Topic: topic, + LatencyMs: result.DurationMS, + } + } + if err := r.logStore.InsertExecutionLog(ctx, log); err != nil { + return fmt.Errorf("insert execution log: %w", err) + } + return nil +} + +func computeNextRetryAt(now time.Time, attempt int, job *pgstore.Job) *time.Time { + if job == nil || job.MaxRetries <= 0 || attempt >= job.MaxRetries { + return nil + } + base := job.RetryBaseDelayS + if base <= 0 { + base = 30 + } + delay := time.Duration(base) * time.Second + if strings.EqualFold(job.RetryStrategy, "EXPONENTIAL") && attempt > 0 { + delay = delay * time.Duration(1<<(attempt-1)) + } + t := now.Add(delay) + return &t +} + +func extractHTTPLogFields(deliveryConfig []byte) (string, string) { + var cfg struct { + URL string `json:"url"` + Method string `json:"method"` + } + _ = json.Unmarshal(deliveryConfig, &cfg) + method := strings.ToUpper(strings.TrimSpace(cfg.Method)) + if method == "" { + method = "POST" + } + return method, strings.TrimSpace(cfg.URL) +} + +func extractKafkaTopic(deliveryConfig []byte) string { + var cfg struct { + Topic string `json:"topic"` + } + _ = json.Unmarshal(deliveryConfig, &cfg) + return strings.TrimSpace(cfg.Topic) +} diff --git a/internal/store/postgres/job_execution_store.go b/internal/store/postgres/job_execution_store.go index 8e3439e..205044f 100644 --- a/internal/store/postgres/job_execution_store.go +++ b/internal/store/postgres/job_execution_store.go @@ -2,6 +2,7 @@ package postgres import ( "context" + "fmt" "strconv" "time" @@ -18,6 +19,7 @@ type JobExecution struct { DurationMs *int `db:"duration_ms"` ResponseStatus *int `db:"response_status"` ErrorMessage *string `db:"error_message"` + NextRetryAt *time.Time `db:"next_retry_at"` WorkerID *string `db:"worker_id"` } @@ -143,6 +145,20 @@ func (s *JobExecutionStore) MarkTimedOut(ctx context.Context, id string, duratio return err } +func (s *JobExecutionStore) UpdateTerminal(ctx context.Context, id string, status string, responseStatus *int, errMsg *string, durationMs int, nextRetryAt *time.Time) error { + switch status { + case "SUCCEEDED", "FAILED", "TIMED_OUT", "DEAD": + default: + return fmt.Errorf("unsupported execution terminal status: %s", status) + } + _, err := s.db.ExecContext(ctx, + `UPDATE job_executions + SET status = $2, completed_at = NOW(), duration_ms = $3, response_status = $4, error_message = $5, next_retry_at = $6 + WHERE id = $1`, + id, status, durationMs, responseStatus, errMsg, nextRetryAt) + return err +} + // ListRetryable returns failed executions whose job has remaining retry budget. // Joins jobs → job_definitions to compare attempt_number < max_retries. // Works across all job types (ONE_TIME, CRON, BULK_RECORD). diff --git a/internal/store/postgres/job_store.go b/internal/store/postgres/job_store.go index 70e8af8..3606128 100644 --- a/internal/store/postgres/job_store.go +++ b/internal/store/postgres/job_store.go @@ -11,23 +11,27 @@ import ( ) type Job struct { - ID string `db:"id"` - JobType string `db:"job_type"` // ONE_TIME | CRON | BULK_RECORD - JobDefID *string `db:"job_def_id"` // NULL for ad-hoc - CronScheduleID *string `db:"cron_schedule_id"` // non-null for CRON - BulkJobID *string `db:"bulk_job_id"` // non-null for BULK_RECORD - BucketID *string `db:"bucket_id"` // non-null for ONE_TIME + CRON - DeliveryType string `db:"delivery_type"` - DeliveryConfig []byte `db:"delivery_config"` - RowStart *int `db:"row_start"` // non-null for BULK_RECORD: first row in batch (0-indexed) - RowEnd *int `db:"row_end"` // non-null for BULK_RECORD: last row in batch (inclusive); == RowStart when batch_size=1 - Payload []byte `db:"payload"` // JSONB — resolved request body; array when batch_size > 1 - Context []byte `db:"context"` // JSONB — runtime vars for delivery_config substitution - ScheduledAt *time.Time `db:"scheduled_at"` // non-null for ONE_TIME + CRON - Status string `db:"status"` - IdempotencyKey *string `db:"idempotency_key"` - Tags pq.StringArray `db:"tags"` - CreatedAt time.Time `db:"created_at"` + ID string `db:"id"` + JobType string `db:"job_type"` // ONE_TIME | CRON | BULK_RECORD + JobDefID *string `db:"job_def_id"` // NULL for ad-hoc + CronScheduleID *string `db:"cron_schedule_id"` // non-null for CRON + BulkJobID *string `db:"bulk_job_id"` // non-null for BULK_RECORD + BucketID *string `db:"bucket_id"` // non-null for ONE_TIME + CRON + DeliveryType string `db:"delivery_type"` + DeliveryConfig []byte `db:"delivery_config"` + TimeoutSeconds int `db:"timeout_seconds"` + MaxRetries int `db:"max_retries"` + RetryStrategy string `db:"retry_strategy"` + RetryBaseDelayS int `db:"retry_base_delay_s"` + RowStart *int `db:"row_start"` // non-null for BULK_RECORD: first row in batch (0-indexed) + RowEnd *int `db:"row_end"` // non-null for BULK_RECORD: last row in batch (inclusive); == RowStart when batch_size=1 + Payload []byte `db:"payload"` // JSONB — resolved request body; array when batch_size > 1 + Context []byte `db:"context"` // JSONB — runtime vars for delivery_config substitution + ScheduledAt *time.Time `db:"scheduled_at"` // non-null for ONE_TIME + CRON + Status string `db:"status"` + IdempotencyKey *string `db:"idempotency_key"` + Tags pq.StringArray `db:"tags"` + CreatedAt time.Time `db:"created_at"` } type JobStore struct { @@ -49,14 +53,26 @@ func NewJobStore(db *sqlx.DB) *JobStore { } func (s *JobStore) Create(ctx context.Context, j *Job) error { + if j.TimeoutSeconds <= 0 { + j.TimeoutSeconds = 30 + } + if j.MaxRetries < 0 { + j.MaxRetries = 0 + } + if j.RetryStrategy == "" { + j.RetryStrategy = "exponential" + } + if j.RetryBaseDelayS <= 0 { + j.RetryBaseDelayS = 60 + } const q = ` INSERT INTO jobs (job_type, job_def_id, cron_schedule_id, bulk_job_id, bucket_id, - delivery_type, delivery_config, row_start, row_end, payload, context, + delivery_type, delivery_config, timeout_seconds, max_retries, retry_strategy, retry_base_delay_s, row_start, row_end, payload, context, scheduled_at, status, idempotency_key, tags) VALUES (:job_type, :job_def_id, :cron_schedule_id, :bulk_job_id, :bucket_id, - :delivery_type, :delivery_config, :row_start, :row_end, :payload, :context, + :delivery_type, :delivery_config, :timeout_seconds, :max_retries, :retry_strategy, :retry_base_delay_s, :row_start, :row_end, :payload, :context, :scheduled_at, :status, :idempotency_key, :tags) RETURNING id, created_at` rows, err := s.db.NamedQueryContext(ctx, q, j) diff --git a/migrations/postgres/0020_denormalize_retry_config_and_add_next_retry_at.down.sql b/migrations/postgres/0020_denormalize_retry_config_and_add_next_retry_at.down.sql new file mode 100644 index 0000000..422cfe8 --- /dev/null +++ b/migrations/postgres/0020_denormalize_retry_config_and_add_next_retry_at.down.sql @@ -0,0 +1,8 @@ +ALTER TABLE job_executions + DROP COLUMN IF EXISTS next_retry_at; + +ALTER TABLE jobs + DROP COLUMN IF EXISTS retry_base_delay_s, + DROP COLUMN IF EXISTS retry_strategy, + DROP COLUMN IF EXISTS max_retries, + DROP COLUMN IF EXISTS timeout_seconds; diff --git a/migrations/postgres/0020_denormalize_retry_config_and_add_next_retry_at.up.sql b/migrations/postgres/0020_denormalize_retry_config_and_add_next_retry_at.up.sql new file mode 100644 index 0000000..48050a3 --- /dev/null +++ b/migrations/postgres/0020_denormalize_retry_config_and_add_next_retry_at.up.sql @@ -0,0 +1,8 @@ +ALTER TABLE jobs + ADD COLUMN timeout_seconds INT NOT NULL DEFAULT 30, + ADD COLUMN max_retries INT NOT NULL DEFAULT 0, + ADD COLUMN retry_strategy TEXT NOT NULL DEFAULT 'exponential', + ADD COLUMN retry_base_delay_s INT NOT NULL DEFAULT 60; + +ALTER TABLE job_executions + ADD COLUMN next_retry_at TIMESTAMPTZ; From d7c2280fd8dc5a38d85dda1b1179ecd9cd238c60 Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 07:43:45 -0700 Subject: [PATCH 16/20] infra(tilt): add job-executor deployment to Kind cluster --- Tiltfile | 2 +- cmd/job-executor/main.go | 8 ++++---- deployments/docker-compose/docker-compose.yml | 1 + deployments/kind/app/job-executor.yaml | 2 ++ deployments/kind/infra-manifests/kafka.yaml | 1 + 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Tiltfile b/Tiltfile index 677de9f..fa76bd8 100644 --- a/Tiltfile +++ b/Tiltfile @@ -80,7 +80,7 @@ docker_build('job-executor', '.', k8s_yaml('deployments/kind/app/job-executor.yaml') k8s_resource('job-executor', - resource_deps=['chronos-postgres', 'chronos-redis', 'chronos-kafka']) + resource_deps=['chronos-postgres', 'chronos-mongodb', 'chronos-redis', 'chronos-kafka']) docker_build('bulk-ingestor', '.', dockerfile='build/package/bulk-ingestor/Dockerfile', diff --git a/cmd/job-executor/main.go b/cmd/job-executor/main.go index d60737f..ae59c7d 100644 --- a/cmd/job-executor/main.go +++ b/cmd/job-executor/main.go @@ -172,11 +172,11 @@ func main() { func executorWorkersFromEnv() int { raw := os.Getenv("EXECUTOR_WORKERS") if raw == "" { - return 1 + return 10 } n, err := strconv.Atoi(raw) if err != nil || n <= 0 { - return 1 + return 10 } return n } @@ -184,11 +184,11 @@ func executorWorkersFromEnv() int { func executorQueueSizeFromEnv(workers int) int { raw := os.Getenv("EXECUTOR_QUEUE_SIZE") if raw == "" { - return workers + return 10 } n, err := strconv.Atoi(raw) if err != nil || n <= 0 { - return workers + return 10 } return n } diff --git a/deployments/docker-compose/docker-compose.yml b/deployments/docker-compose/docker-compose.yml index b88ebb9..a7df68a 100644 --- a/deployments/docker-compose/docker-compose.yml +++ b/deployments/docker-compose/docker-compose.yml @@ -94,6 +94,7 @@ services: entrypoint: > bash -c " kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic bucket-triggers --partitions 12 --replication-factor 1 --config retention.ms=3600000 && + kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic job-dispatch --partitions 24 --replication-factor 1 --config retention.ms=86400000 && kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic bulk-records --partitions 24 --replication-factor 1 --config retention.ms=86400000 && kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic bulk-results --partitions 24 --replication-factor 1 --config retention.ms=86400000 && kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic job-events --partitions 12 --replication-factor 1 --config retention.ms=604800000 && diff --git a/deployments/kind/app/job-executor.yaml b/deployments/kind/app/job-executor.yaml index 0aed307..7f58d29 100644 --- a/deployments/kind/app/job-executor.yaml +++ b/deployments/kind/app/job-executor.yaml @@ -9,6 +9,8 @@ data: CHRONOS_LOGFORMAT: "json" CHRONOS_REDISADDR: "chronos-redis-master.chronos-infra.svc.cluster.local:6379" CHRONOS_KAFKABROKERS: "chronos-kafka.chronos-infra.svc.cluster.local:9092" + EXECUTOR_WORKERS: "10" + EXECUTOR_QUEUE_SIZE: "10" CHRONOS_LEADERLOCKTTLSECONDS: "30" --- apiVersion: v1 diff --git a/deployments/kind/infra-manifests/kafka.yaml b/deployments/kind/infra-manifests/kafka.yaml index 9b95140..23f55ad 100644 --- a/deployments/kind/infra-manifests/kafka.yaml +++ b/deployments/kind/infra-manifests/kafka.yaml @@ -108,6 +108,7 @@ spec: echo "Waiting for Kafka..." until nc -z chronos-kafka.chronos-infra.svc.cluster.local 9092; do sleep 3; done kafka-topics --bootstrap-server chronos-kafka.chronos-infra.svc.cluster.local:9092 --create --if-not-exists --topic bucket-triggers --partitions 12 --replication-factor 1 --config retention.ms=3600000 + kafka-topics --bootstrap-server chronos-kafka.chronos-infra.svc.cluster.local:9092 --create --if-not-exists --topic job-dispatch --partitions 24 --replication-factor 1 --config retention.ms=86400000 kafka-topics --bootstrap-server chronos-kafka.chronos-infra.svc.cluster.local:9092 --create --if-not-exists --topic bulk-records --partitions 24 --replication-factor 1 --config retention.ms=86400000 kafka-topics --bootstrap-server chronos-kafka.chronos-infra.svc.cluster.local:9092 --create --if-not-exists --topic bulk-results --partitions 24 --replication-factor 1 --config retention.ms=86400000 kafka-topics --bootstrap-server chronos-kafka.chronos-infra.svc.cluster.local:9092 --create --if-not-exists --topic job-events --partitions 12 --replication-factor 1 --config retention.ms=604800000 From b48c3a17df68250587162061734c6583a87b36b5 Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 08:00:35 -0700 Subject: [PATCH 17/20] test(job-executor): add end-to-end integration test for HTTP dispatch flow --- go.mod | 7 +- internal/executor/result.go | 9 +- test/integration/job_execution_test.go | 267 +++++++++++++++++++++++++ 3 files changed, 278 insertions(+), 5 deletions(-) create mode 100644 test/integration/job_execution_test.go diff --git a/go.mod b/go.mod index 65df58f..5b80bb9 100644 --- a/go.mod +++ b/go.mod @@ -86,7 +86,7 @@ require ( github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-connections v0.6.0 github.com/docker/go-units v0.5.0 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -121,6 +121,9 @@ require ( go.opentelemetry.io/otel/trace v1.41.0 // indirect ) -require github.com/segmentio/kafka-go v0.4.50 +require ( + github.com/docker/docker v28.3.3+incompatible + github.com/segmentio/kafka-go v0.4.50 +) require github.com/pierrec/lz4/v4 v4.1.16 // indirect diff --git a/internal/executor/result.go b/internal/executor/result.go index 308a031..d338499 100644 --- a/internal/executor/result.go +++ b/internal/executor/result.go @@ -8,7 +8,6 @@ import ( "time" "github.com/chronos-scheduler/chronos/internal/executor/dispatch" - kafkastore "github.com/chronos-scheduler/chronos/internal/store/kafka" mongostore "github.com/chronos-scheduler/chronos/internal/store/mongo" pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" "go.uber.org/zap" @@ -21,16 +20,20 @@ type ResultReporter struct { jobStore *pgstore.JobStore executionStore *pgstore.JobExecutionStore logStore *mongostore.LogStore - producer *kafkastore.Producer + producer jobEventPublisher workerID string } +type jobEventPublisher interface { + Publish(ctx context.Context, topic, key string, value []byte) error +} + func NewResultReporter( logger *zap.Logger, jobStore *pgstore.JobStore, executionStore *pgstore.JobExecutionStore, logStore *mongostore.LogStore, - producer *kafkastore.Producer, + producer jobEventPublisher, workerID string, ) *ResultReporter { return &ResultReporter{ diff --git a/test/integration/job_execution_test.go b/test/integration/job_execution_test.go new file mode 100644 index 0000000..4e12f12 --- /dev/null +++ b/test/integration/job_execution_test.go @@ -0,0 +1,267 @@ +package integration_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/netip" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/network" + kafkago "github.com/segmentio/kafka-go" + "github.com/testcontainers/testcontainers-go" + tcmongo "github.com/testcontainers/testcontainers-go/modules/mongodb" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.uber.org/zap" + + "github.com/chronos-scheduler/chronos/internal/executor" + kafkastore "github.com/chronos-scheduler/chronos/internal/store/kafka" + mongostore "github.com/chronos-scheduler/chronos/internal/store/mongo" + pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" +) + +func TestJobExecution_HTTPFlow(t *testing.T) { + ctx := context.Background() + pgdb := startPostgres(t) + mdb := startMongo(t) + kafkaBroker := startKafka(t) + if kafkaBroker == "" { + t.Skip("kafka unavailable for integration test") + } + if err := ensureTopic(ctx, kafkaBroker, "job-events"); err != nil { + t.Fatalf("create job-events topic: %v", err) + } + + var calls int32 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&calls, 1) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(target.Close) + + jobStore := pgstore.NewJobStore(pgdb) + execStore := pgstore.NewJobExecutionStore(pgdb) + logStore := mongostore.NewLogStore(mdb, "chronos_test") + logger := zap.NewNop() + + producer := kafkastore.NewProducer([]string{kafkaBroker}) + t.Cleanup(func() { _ = producer.Close() }) + reporter := executor.NewResultReporter(logger, jobStore, execStore, logStore, producer, "itest-worker") + processor := executor.NewTaskProcessor(logger, jobStore, execStore, nil, reporter) + + deliveryConfig, _ := json.Marshal(map[string]any{ + "url": target.URL, + "method": "POST", + }) + now := time.Now().UTC().Add(10 * time.Second) + job := &pgstore.Job{ + JobType: "ONE_TIME", + DeliveryType: "HTTP", + DeliveryConfig: deliveryConfig, + TimeoutSeconds: 10, + MaxRetries: 2, + RetryStrategy: "exponential", + RetryBaseDelayS: 5, + Payload: []byte(`{"message":"hello"}`), + Context: []byte(`{}`), + ScheduledAt: &now, + Status: "DISPATCHED", + Tags: []string{}, + } + if err := jobStore.Create(ctx, job); err != nil { + t.Fatalf("create job: %v", err) + } + + task := &executor.DispatchTask{ + Event: executor.JobDispatchEvent{ + JobID: job.ID, + AttemptNumber: 1, + DeliveryType: "HTTP", + }, + } + if err := processor.Process(ctx, task); err != nil { + t.Fatalf("process task: %v", err) + } + + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("expected target called once, got %d", got) + } + + gotJob, err := jobStore.GetByID(ctx, job.ID) + if err != nil { + t.Fatalf("get job: %v", err) + } + if gotJob.Status != "SUCCEEDED" { + t.Fatalf("job status: got %s want SUCCEEDED", gotJob.Status) + } + + execs, err := execStore.GetByJobID(ctx, job.ID) + if err != nil { + t.Fatalf("list executions: %v", err) + } + if len(execs) != 1 { + t.Fatalf("expected 1 execution, got %d", len(execs)) + } + if execs[0].Status != "SUCCEEDED" { + t.Fatalf("execution status: got %s want SUCCEEDED", execs[0].Status) + } + + logs, err := logStore.GetLogsForExecution(ctx, execs[0].ID) + if err != nil { + t.Fatalf("get logs: %v", err) + } + if len(logs) != 1 { + t.Fatalf("expected 1 log, got %d", len(logs)) + } + if logs[0].Status != "SUCCEEDED" { + t.Fatalf("log status: got %s want SUCCEEDED", logs[0].Status) + } + + reader := kafkago.NewReader(kafkago.ReaderConfig{ + Brokers: []string{kafkaBroker}, + GroupID: fmt.Sprintf("itest-job-events-%d", time.Now().UnixNano()), + Topic: "job-events", + }) + t.Cleanup(func() { _ = reader.Close() }) + + readCtx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + msg, err := reader.FetchMessage(readCtx) + if err != nil { + t.Fatalf("read job-events message: %v", err) + } + var evt map[string]any + if err := json.Unmarshal(msg.Value, &evt); err != nil { + t.Fatalf("decode job-events payload: %v", err) + } + if evt["type"] != "JOB_COMPLETED" { + t.Fatalf("event type: got %v want JOB_COMPLETED", evt["type"]) + } +} + +func startPostgres(t *testing.T) *pgstore.DB { + t.Helper() + ctx := context.Background() + ctr, err := tcpostgres.Run(ctx, + "postgres:16-alpine", + tcpostgres.WithDatabase("chronos"), + tcpostgres.WithUsername("chronos"), + tcpostgres.WithPassword("chronos"), + tcpostgres.BasicWaitStrategies(), + tcpostgres.WithSQLDriver("postgres"), + ) + if err != nil { + t.Fatalf("start postgres: %v", err) + } + t.Cleanup(func() { _ = ctr.Terminate(ctx) }) + + dsn, err := ctr.ConnectionString(ctx, "sslmode=disable") + if err != nil { + t.Fatalf("postgres dsn: %v", err) + } + if err := pgstore.Migrate(dsn, migrationsDir()); err != nil { + t.Fatalf("migrate: %v", err) + } + db, err := pgstore.Connect(dsn) + if err != nil { + t.Fatalf("connect postgres: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +func startMongo(t *testing.T) *mongo.Client { + t.Helper() + ctx := context.Background() + ctr, err := tcmongo.Run(ctx, "mongo:7") + if err != nil { + t.Fatalf("start mongo: %v", err) + } + t.Cleanup(func() { _ = ctr.Terminate(ctx) }) + uri, err := ctr.ConnectionString(ctx) + if err != nil { + t.Fatalf("mongo uri: %v", err) + } + client, err := mongostore.Connect(uri) + if err != nil { + t.Fatalf("connect mongo: %v", err) + } + t.Cleanup(func() { _ = client.Disconnect(ctx) }) + return client +} + +func startKafka(t *testing.T) string { + t.Helper() + ctx := context.Background() + ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "docker.redpanda.com/redpandadata/redpanda:v24.2.8", + ExposedPorts: []string{"9092/tcp"}, + Cmd: []string{ + "redpanda", "start", + "--overprovisioned", + "--smp", "1", + "--memory", "512M", + "--reserve-memory", "0M", + "--check=false", + "--node-id", "0", + "--kafka-addr", "PLAINTEXT://0.0.0.0:9092", + "--advertise-kafka-addr", "PLAINTEXT://127.0.0.1:9092", + }, + WaitingFor: wait.ForListeningPort("9092/tcp").WithStartupTimeout(90 * time.Second), + HostConfigModifier: func(hc *container.HostConfig) { + hc.AutoRemove = true + hc.PortBindings = network.PortMap{ + network.MustParsePort("9092/tcp"): []network.PortBinding{{HostIP: netip.MustParseAddr("127.0.0.1"), HostPort: "9092"}}, + } + }, + }, + Started: true, + }) + if err != nil { + t.Skipf("start kafka/redpanda: %v", err) + return "" + } + t.Cleanup(func() { _ = ctr.Terminate(ctx) }) + return "127.0.0.1:9092" +} + +func ensureTopic(ctx context.Context, broker, topic string) error { + conn, err := kafkago.Dial("tcp", broker) + if err != nil { + return err + } + defer conn.Close() + + controller, err := conn.Controller() + if err != nil { + return err + } + controllerAddr := fmt.Sprintf("%s:%d", controller.Host, controller.Port) + ctrlConn, err := kafkago.Dial("tcp", controllerAddr) + if err != nil { + return err + } + defer ctrlConn.Close() + + return ctrlConn.CreateTopics(kafkago.TopicConfig{ + Topic: topic, + NumPartitions: 1, + ReplicationFactor: 1, + }) +} + +func migrationsDir() string { + _, file, _, _ := runtime.Caller(0) + return filepath.Join(filepath.Dir(file), "../../migrations/postgres") +} From 4eff5f9787bca8940990c04890f1c9632b206799 Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 08:25:37 -0700 Subject: [PATCH 18/20] test(scheduler): add phase 6 smoke test script --- internal/store/postgres/job_store.go | 3 + ...nforce_jobs_tags_not_null_default.down.sql | 3 + ..._enforce_jobs_tags_not_null_default.up.sql | 7 ++ scripts/smoke-test-phase6.sh | 118 ++++++++++++++++++ 4 files changed, 131 insertions(+) create mode 100644 migrations/postgres/0021_enforce_jobs_tags_not_null_default.down.sql create mode 100644 migrations/postgres/0021_enforce_jobs_tags_not_null_default.up.sql create mode 100755 scripts/smoke-test-phase6.sh diff --git a/internal/store/postgres/job_store.go b/internal/store/postgres/job_store.go index 3606128..61aa00c 100644 --- a/internal/store/postgres/job_store.go +++ b/internal/store/postgres/job_store.go @@ -53,6 +53,9 @@ func NewJobStore(db *sqlx.DB) *JobStore { } func (s *JobStore) Create(ctx context.Context, j *Job) error { + if j.Tags == nil { + j.Tags = pq.StringArray{} + } if j.TimeoutSeconds <= 0 { j.TimeoutSeconds = 30 } diff --git a/migrations/postgres/0021_enforce_jobs_tags_not_null_default.down.sql b/migrations/postgres/0021_enforce_jobs_tags_not_null_default.down.sql new file mode 100644 index 0000000..3c44fc6 --- /dev/null +++ b/migrations/postgres/0021_enforce_jobs_tags_not_null_default.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE jobs + ALTER COLUMN tags DROP NOT NULL, + ALTER COLUMN tags DROP DEFAULT; diff --git a/migrations/postgres/0021_enforce_jobs_tags_not_null_default.up.sql b/migrations/postgres/0021_enforce_jobs_tags_not_null_default.up.sql new file mode 100644 index 0000000..d6f6d76 --- /dev/null +++ b/migrations/postgres/0021_enforce_jobs_tags_not_null_default.up.sql @@ -0,0 +1,7 @@ +UPDATE jobs +SET tags = '{}' +WHERE tags IS NULL; + +ALTER TABLE jobs + ALTER COLUMN tags SET DEFAULT '{}'::TEXT[], + ALTER COLUMN tags SET NOT NULL; diff --git a/scripts/smoke-test-phase6.sh b/scripts/smoke-test-phase6.sh new file mode 100755 index 0000000..f1cbf1e --- /dev/null +++ b/scripts/smoke-test-phase6.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Phase 6 smoke test: one-time scheduled job end-to-end. +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:8080}" +API_KEY="${API_KEY:-}" +WAIT_TIMEOUT_SECONDS="${WAIT_TIMEOUT_SECONDS:-240}" +POLL_INTERVAL_SECONDS="${POLL_INTERVAL_SECONDS:-5}" + +if [[ -z "${API_KEY}" ]]; then + echo "API_KEY is required. Seed one with: scripts/seed-api-key.sh" + exit 1 +fi + +json_get() { + local jq_filter="$1" + local py_expr="$2" + if command -v jq >/dev/null 2>&1; then + jq -r "${jq_filter}" + else + python3 -c 'import json,sys; data=json.load(sys.stdin); print(eval(sys.argv[1], {}, {"data": data}))' "${py_expr}" + fi +} + +echo "Running Phase 6 smoke test against ${BASE_URL}..." + +NOW_EPOCH="$(date -u +%s)" +SCHEDULED_AT="$(python3 - <<'PY' +import datetime +print((datetime.datetime.now(datetime.timezone.utc)+datetime.timedelta(seconds=90)).replace(microsecond=0).isoformat().replace("+00:00","Z")) +PY +)" +RUN_ID="${NOW_EPOCH}" +JOB_NAME="phase6-smoke-${RUN_ID}" + +echo "Creating job definition: ${JOB_NAME}" +CREATE_DEF_RESP="$( + curl -sS -X POST "${BASE_URL}/api/v1/job-definitions" \ + -H "X-Chronos-API-Key: ${API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{ + \"name\":\"${JOB_NAME}\", + \"delivery_mode\":\"HTTP\", + \"endpoint\":\"https://httpbin.org/post\", + \"http_method\":\"POST\", + \"timeout_seconds\":15, + \"retry_policy\":{\"max_attempts\":2,\"backoff_array\":[30,120]} + }" +)" +JOB_DEF_ID="$(echo "${CREATE_DEF_RESP}" | json_get '.id // ""' 'data.get("id","")')" +if [[ -z "${JOB_DEF_ID}" || "${JOB_DEF_ID}" == "None" ]]; then + echo "Failed to create job definition." + echo "${CREATE_DEF_RESP}" + exit 1 +fi +echo "Job definition created: ${JOB_DEF_ID}" + +echo "Scheduling one-time job at ${SCHEDULED_AT}" +CREATE_SCHED_RESP="$( + curl -sS -X POST "${BASE_URL}/api/v1/schedules" \ + -H "X-Chronos-API-Key: ${API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{ + \"job_definition_id\":\"${JOB_DEF_ID}\", + \"scheduled_at\":\"${SCHEDULED_AT}\", + \"payload\":{\"phase\":\"6\",\"run_id\":\"${RUN_ID}\"}, + \"idempotency_key\":\"phase6-smoke-${RUN_ID}\" + }" +)" +SCHEDULE_ID="$(echo "${CREATE_SCHED_RESP}" | json_get '.id // ""' 'data.get("id","")')" +if [[ -z "${SCHEDULE_ID}" || "${SCHEDULE_ID}" == "None" ]]; then + echo "Failed to create schedule." + echo "${CREATE_SCHED_RESP}" + exit 1 +fi +echo "Schedule created: ${SCHEDULE_ID}" + +echo "Polling executions for job_definition_id=${JOB_DEF_ID} (timeout ${WAIT_TIMEOUT_SECONDS}s)..." +DEADLINE=$(( NOW_EPOCH + WAIT_TIMEOUT_SECONDS )) +EXEC_ID="" +EXEC_STATUS="" +while [[ "$(date -u +%s)" -lt "${DEADLINE}" ]]; do + LIST_EXEC_RESP="$( + curl -sS "${BASE_URL}/api/v1/executions?job_def_id=${JOB_DEF_ID}&limit=1" \ + -H "X-Chronos-API-Key: ${API_KEY}" + )" + EXEC_ID="$(echo "${LIST_EXEC_RESP}" | json_get '.items[0].id // ""' 'data.get("items",[{}])[0].get("id","")')" + EXEC_STATUS="$(echo "${LIST_EXEC_RESP}" | json_get '.items[0].status // ""' 'data.get("items",[{}])[0].get("status","")')" + + if [[ -n "${EXEC_ID}" && "${EXEC_ID}" != "None" && -n "${EXEC_STATUS}" ]]; then + echo "Execution observed: id=${EXEC_ID} status=${EXEC_STATUS}" + if [[ "${EXEC_STATUS}" == "SUCCEEDED" || "${EXEC_STATUS}" == "FAILED" || "${EXEC_STATUS}" == "TIMED_OUT" || "${EXEC_STATUS}" == "DEAD" ]]; then + break + fi + fi + sleep "${POLL_INTERVAL_SECONDS}" +done + +if [[ -z "${EXEC_ID}" || "${EXEC_ID}" == "None" ]]; then + echo "Timed out waiting for execution to appear." + exit 1 +fi + +GET_EXEC_RESP="$( + curl -sS "${BASE_URL}/api/v1/executions/${EXEC_ID}" \ + -H "X-Chronos-API-Key: ${API_KEY}" +)" +LOG_COUNT="$(echo "${GET_EXEC_RESP}" | json_get '.logs | length' 'len(data.get("logs",[]))')" +FINAL_STATUS="$(echo "${GET_EXEC_RESP}" | json_get '.execution.status // ""' 'data.get("execution",{}).get("status","")')" +echo "Execution final status: ${FINAL_STATUS}" +echo "Execution logs count: ${LOG_COUNT}" + +if [[ "${LOG_COUNT}" == "0" || "${LOG_COUNT}" == "None" ]]; then + echo "Expected execution logs, found none." + exit 1 +fi + +echo "Phase 6 smoke test completed." From 57668ec1abaf720049d21388a934269683a94b0c Mon Sep 17 00:00:00 2001 From: chandan-m Date: Tue, 21 Apr 2026 08:30:39 -0700 Subject: [PATCH 19/20] feat(scheduler): integrate dispatcher and leadership loop with Kafka producer for scalable job scheduling --- cmd/chronos-server/main.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/cmd/chronos-server/main.go b/cmd/chronos-server/main.go index 8553a3f..058bf7a 100644 --- a/cmd/chronos-server/main.go +++ b/cmd/chronos-server/main.go @@ -13,6 +13,8 @@ import ( apihttp "github.com/chronos-scheduler/chronos/internal/api/http" "github.com/chronos-scheduler/chronos/internal/config" + "github.com/chronos-scheduler/chronos/internal/scheduler" + kafkastore "github.com/chronos-scheduler/chronos/internal/store/kafka" mongostore "github.com/chronos-scheduler/chronos/internal/store/mongo" pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" redisstore "github.com/chronos-scheduler/chronos/internal/store/redis" @@ -70,6 +72,13 @@ func main() { } logger.Info("redis connected") + kafkaProducer := kafkastore.NewProducer(cfg.KafkaBrokers) + defer func() { + if err := kafkaProducer.Close(); err != nil { + logger.Error("kafka producer close error", zap.Error(err)) + } + }() + router := apihttp.NewRouter(apihttp.RouterDeps{ Logger: logger, PostgresDB: db, @@ -83,6 +92,16 @@ func main() { Handler: router, } + dispatcher := scheduler.NewDispatcher( + pgstore.NewScheduleBucketStore(db), + pgstore.NewJobStore(db), + kafkaProducer, + cfg.SchedulerBucketSeconds, + 0, + ) + backgroundCtx, cancelBackground := context.WithCancel(context.Background()) + defer cancelBackground() + // Start HTTP server in background go func() { logger.Info("http server listening", zap.String("addr", srv.Addr)) @@ -90,12 +109,27 @@ func main() { logger.Fatal("http server error", zap.Error(err)) } }() + go func() { + lockTTL := time.Duration(cfg.LeaderLockTTLSeconds) * time.Second + logger.Info("scheduler leader loop started", + zap.Int("bucket_seconds", cfg.SchedulerBucketSeconds), + zap.Duration("lock_ttl", lockTTL), + ) + scheduler.RunWithLeadership(backgroundCtx, redisClient, lockTTL, func(leaderCtx context.Context) { + logger.Info("scheduler leadership acquired") + if err := dispatcher.Run(leaderCtx); err != nil { + logger.Error("scheduler dispatcher exited with error", zap.Error(err)) + } + logger.Info("scheduler leadership released") + }) + }() // Block until SIGINT or SIGTERM quit := make(chan os.Signal, 1) signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) sig := <-quit logger.Info("shutdown signal received", zap.String("signal", sig.String())) + cancelBackground() // Graceful shutdown with 10s timeout ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) From 41b59bf7d780b2c05d782f8839dbec699794128b Mon Sep 17 00:00:00 2001 From: chandan-m Date: Wed, 22 Apr 2026 00:05:41 -0700 Subject: [PATCH 20/20] feat(scheduler): integrate dispatcher and leadership loop with Kafka producer for scalable job scheduling --- internal/store/postgres/job_store.go | 3 + test/integration/job_execution_fast_test.go | 166 ++++++++++++++++++++ test/integration/job_execution_test.go | 65 +++++--- 3 files changed, 212 insertions(+), 22 deletions(-) create mode 100644 test/integration/job_execution_fast_test.go diff --git a/internal/store/postgres/job_store.go b/internal/store/postgres/job_store.go index 61aa00c..dfc4449 100644 --- a/internal/store/postgres/job_store.go +++ b/internal/store/postgres/job_store.go @@ -56,6 +56,9 @@ func (s *JobStore) Create(ctx context.Context, j *Job) error { if j.Tags == nil { j.Tags = pq.StringArray{} } + if len(j.DeliveryConfig) == 0 { + j.DeliveryConfig = []byte(`{}`) + } if j.TimeoutSeconds <= 0 { j.TimeoutSeconds = 30 } diff --git a/test/integration/job_execution_fast_test.go b/test/integration/job_execution_fast_test.go new file mode 100644 index 0000000..71e41d4 --- /dev/null +++ b/test/integration/job_execution_fast_test.go @@ -0,0 +1,166 @@ +package integration_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + "time" + + tcmongo "github.com/testcontainers/testcontainers-go/modules/mongodb" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.uber.org/zap" + + "github.com/chronos-scheduler/chronos/internal/executor" + mongostore "github.com/chronos-scheduler/chronos/internal/store/mongo" + pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" +) + +type fakePublisher struct { + lastTopic string +} + +func (f *fakePublisher) Publish(_ context.Context, topic, _ string, _ []byte) error { + f.lastTopic = topic + return nil +} + +func TestJobExecution_HTTPFlow(t *testing.T) { + ctx := context.Background() + pgdb := startPostgresFast(t) + mdb := startMongoFast(t) + + var calls int32 + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&calls, 1) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + t.Cleanup(target.Close) + + jobStore := pgstore.NewJobStore(pgdb) + execStore := pgstore.NewJobExecutionStore(pgdb) + logStore := mongostore.NewLogStore(mdb, "chronos_test") + pub := &fakePublisher{} + logger := zap.NewNop() + + reporter := executor.NewResultReporter(logger, jobStore, execStore, logStore, pub, "itest-worker") + processor := executor.NewTaskProcessor(logger, jobStore, execStore, nil, reporter) + + deliveryConfig, _ := json.Marshal(map[string]any{"url": target.URL, "method": "POST"}) + now := time.Now().UTC().Add(10 * time.Second) + job := &pgstore.Job{ + JobType: "ONE_TIME", + DeliveryType: "HTTP", + DeliveryConfig: deliveryConfig, + TimeoutSeconds: 10, + MaxRetries: 2, + RetryStrategy: "exponential", + RetryBaseDelayS: 5, + Payload: []byte(`{"message":"hello"}`), + Context: []byte(`{}`), + ScheduledAt: &now, + Status: "DISPATCHED", + Tags: []string{}, + } + if err := jobStore.Create(ctx, job); err != nil { + t.Fatalf("create job: %v", err) + } + + task := &executor.DispatchTask{ + Event: executor.JobDispatchEvent{JobID: job.ID, AttemptNumber: 1, DeliveryType: "HTTP"}, + } + if err := processor.Process(ctx, task); err != nil { + t.Fatalf("process task: %v", err) + } + + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("expected target called once, got %d", got) + } + if pub.lastTopic != "job-events" { + t.Fatalf("event topic: got %s want job-events", pub.lastTopic) + } + + gotJob, err := jobStore.GetByID(ctx, job.ID) + if err != nil { + t.Fatalf("get job: %v", err) + } + if gotJob.Status != "SUCCEEDED" { + t.Fatalf("job status: got %s want SUCCEEDED", gotJob.Status) + } + + execs, err := execStore.GetByJobID(ctx, job.ID) + if err != nil { + t.Fatalf("list executions: %v", err) + } + if len(execs) != 1 || execs[0].Status != "SUCCEEDED" { + t.Fatalf("execution mismatch") + } + logs, err := logStore.GetLogsForExecution(ctx, execs[0].ID) + if err != nil { + t.Fatalf("get logs: %v", err) + } + if len(logs) != 1 || logs[0].Status != "SUCCEEDED" { + t.Fatalf("log mismatch") + } +} + +func startPostgresFast(t *testing.T) *pgstore.DB { + t.Helper() + ctx := context.Background() + ctr, err := tcpostgres.Run(ctx, + "postgres:16-alpine", + tcpostgres.WithDatabase("chronos"), + tcpostgres.WithUsername("chronos"), + tcpostgres.WithPassword("chronos"), + tcpostgres.BasicWaitStrategies(), + tcpostgres.WithSQLDriver("postgres"), + ) + if err != nil { + t.Fatalf("start postgres: %v", err) + } + t.Cleanup(func() { _ = ctr.Terminate(ctx) }) + dsn, err := ctr.ConnectionString(ctx, "sslmode=disable") + if err != nil { + t.Fatalf("postgres dsn: %v", err) + } + if err := pgstore.Migrate(dsn, migrationsDirFast()); err != nil { + t.Fatalf("migrate: %v", err) + } + db, err := pgstore.Connect(dsn) + if err != nil { + t.Fatalf("connect postgres: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +func startMongoFast(t *testing.T) *mongo.Client { + t.Helper() + ctx := context.Background() + ctr, err := tcmongo.Run(ctx, "mongo:7") + if err != nil { + t.Fatalf("start mongo: %v", err) + } + t.Cleanup(func() { _ = ctr.Terminate(ctx) }) + uri, err := ctr.ConnectionString(ctx) + if err != nil { + t.Fatalf("mongo uri: %v", err) + } + client, err := mongostore.Connect(uri) + if err != nil { + t.Fatalf("connect mongo: %v", err) + } + t.Cleanup(func() { _ = client.Disconnect(ctx) }) + return client +} + +func migrationsDirFast() string { + _, file, _, _ := runtime.Caller(0) + return filepath.Join(filepath.Dir(file), "../../migrations/postgres") +} diff --git a/test/integration/job_execution_test.go b/test/integration/job_execution_test.go index 4e12f12..866ca5b 100644 --- a/test/integration/job_execution_test.go +++ b/test/integration/job_execution_test.go @@ -1,3 +1,6 @@ +//go:build local_kafka +// +build local_kafka + package integration_test import ( @@ -29,7 +32,7 @@ import ( pgstore "github.com/chronos-scheduler/chronos/internal/store/postgres" ) -func TestJobExecution_HTTPFlow(t *testing.T) { +func TestJobExecution_HTTPFlow_KafkaLocal(t *testing.T) { ctx := context.Background() pgdb := startPostgres(t) mdb := startMongo(t) @@ -218,7 +221,10 @@ func startKafka(t *testing.T) string { "--kafka-addr", "PLAINTEXT://0.0.0.0:9092", "--advertise-kafka-addr", "PLAINTEXT://127.0.0.1:9092", }, - WaitingFor: wait.ForListeningPort("9092/tcp").WithStartupTimeout(90 * time.Second), + WaitingFor: wait.ForAll( + wait.ForListeningPort("9092/tcp"), + wait.ForLog("Successfully started Redpanda"), + ).WithDeadline(120 * time.Second), HostConfigModifier: func(hc *container.HostConfig) { hc.AutoRemove = true hc.PortBindings = network.PortMap{ @@ -237,28 +243,43 @@ func startKafka(t *testing.T) string { } func ensureTopic(ctx context.Context, broker, topic string) error { - conn, err := kafkago.Dial("tcp", broker) - if err != nil { - return err - } - defer conn.Close() + var lastErr error + for i := 0; i < 30; i++ { + conn, err := kafkago.Dial("tcp", broker) + if err != nil { + lastErr = err + time.Sleep(1 * time.Second) + continue + } - controller, err := conn.Controller() - if err != nil { - return err - } - controllerAddr := fmt.Sprintf("%s:%d", controller.Host, controller.Port) - ctrlConn, err := kafkago.Dial("tcp", controllerAddr) - if err != nil { - return err - } - defer ctrlConn.Close() + controller, err := conn.Controller() + _ = conn.Close() + if err != nil { + lastErr = err + time.Sleep(1 * time.Second) + continue + } - return ctrlConn.CreateTopics(kafkago.TopicConfig{ - Topic: topic, - NumPartitions: 1, - ReplicationFactor: 1, - }) + controllerAddr := fmt.Sprintf("%s:%d", controller.Host, controller.Port) + ctrlConn, err := kafkago.Dial("tcp", controllerAddr) + if err != nil { + lastErr = err + time.Sleep(1 * time.Second) + continue + } + err = ctrlConn.CreateTopics(kafkago.TopicConfig{ + Topic: topic, + NumPartitions: 1, + ReplicationFactor: 1, + }) + _ = ctrlConn.Close() + if err == nil { + return nil + } + lastErr = err + time.Sleep(1 * time.Second) + } + return lastErr } func migrationsDir() string {