From d4edbc25486f20452d1f6a4f0771c57950e3f2bf Mon Sep 17 00:00:00 2001 From: Chung Wu Date: Wed, 10 Jan 2024 21:14:24 +0800 Subject: [PATCH 1/8] bump version to v2; add context to interfaces --- cmd/captin/main.go | 7 +- core/captin.go | 29 +- destinations/filters/base.go | 10 +- destinations/filters/desired_hook.go | 8 +- destinations/filters/environment.go | 8 +- destinations/filters/source.go | 8 +- destinations/filters/validate.go | 8 +- dispatcher/delayers/goroutine.go | 9 +- errors/dispatcher_error.go | 5 +- errors/execution_error.go | 2 +- errors/unretryable_errors.go | 4 +- examples/api/incoming/http.go | 8 +- examples/api/main.go | 8 +- examples/api/test/incoming/http_test.go | 6 +- examples/redis/incoming/http.go | 8 +- examples/redis/main.go | 4 +- examples/redis/stores/redis_store.go | 4 +- examples/redis/test/incoming/http_test.go | 6 +- examples/with-filters/filters/target_id.go | 4 +- examples/with-filters/main.go | 6 +- examples/with-filters/middlewares/logger.go | 2 +- go.mod | 7 +- go.sum | 32 +- interfaces/dispatcher.go | 4 +- interfaces/document_store.go | 6 +- interfaces/interfaces.go | 9 +- interfaces/store.go | 15 +- internal/document_stores/null_store.go | 11 +- internal/outgoing/custom.go | 14 +- internal/outgoing/dispatcher.go | 139 ++++---- internal/stores/memstore.go | 19 +- internal/throttles/throttler.go | 10 +- models/config.go | 3 +- models/config_mapper.go | 2 +- models/destination.go | 18 +- models/incoming_event.go | 2 +- senders/beanstlakd_sender.go | 9 +- senders/console_sender.go | 14 +- senders/http_proxy_sender.go | 12 +- senders/http_sender.go | 8 +- senders/sqs_sender.go | 8 +- test/core/captin_test.go | 14 +- test/desination/filters/desired_hook_test.go | 28 +- test/desination/filters/environment_test.go | 32 +- test/desination/filters/source_test.go | 18 +- test/desination/filters/validate_test.go | 28 +- test/dispatcher/delayers/goroutine_test.go | 10 +- test/dispatcher/tracking_test.go | 2 +- .../document_stores/null_store_test.go | 16 +- test/internal/helpers/select_field_test.go | 2 +- test/internal/outgoing/custom_test.go | 43 +-- test/internal/outgoing/dispatcher_test.go | 311 +++++++++--------- test/internal/stores/memstore_test.go | 5 +- test/internal/throttles/throttler_test.go | 34 +- test/mocks/document_store_mock.go | 11 +- test/mocks/sender_mock.go | 10 +- test/mocks/store_mock.go | 27 +- test/mocks/throttle_mock.go | 6 +- test/models/config_mapper_test.go | 4 +- test/models/config_test.go | 2 +- test/models/destination_test.go | 2 +- test/models/incoming_event_test.go | 2 +- test/senders/beanstlakd_sender_test.go | 7 +- test/senders/sqs_sender_test.go | 10 +- 64 files changed, 605 insertions(+), 515 deletions(-) diff --git a/cmd/captin/main.go b/cmd/captin/main.go index 2189d73..559b59d 100644 --- a/cmd/captin/main.go +++ b/cmd/captin/main.go @@ -1,14 +1,15 @@ package main import ( + "context" "os" "os/signal" "path/filepath" "syscall" "time" - core "github.com/shoplineapp/captin/core" - models "github.com/shoplineapp/captin/models" + core "github.com/shoplineapp/captin/v2/core" + models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" ) @@ -31,7 +32,7 @@ func main() { go func() { for enabled && captin.IsRunning() != true { - captin.Execute(models.IncomingEvent{ + captin.Execute(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, diff --git a/core/captin.go b/core/captin.go index 218ef2c..554c1e9 100644 --- a/core/captin.go +++ b/core/captin.go @@ -1,19 +1,20 @@ package core import ( + "context" "fmt" - destination_filters "github.com/shoplineapp/captin/destinations/filters" - d "github.com/shoplineapp/captin/dispatcher" - interfaces "github.com/shoplineapp/captin/interfaces" - outgoing "github.com/shoplineapp/captin/internal/outgoing" - models "github.com/shoplineapp/captin/models" - senders "github.com/shoplineapp/captin/senders" - - captin_errors "github.com/shoplineapp/captin/errors" - documentStores "github.com/shoplineapp/captin/internal/document_stores" - stores "github.com/shoplineapp/captin/internal/stores" - throttles "github.com/shoplineapp/captin/internal/throttles" + destination_filters "github.com/shoplineapp/captin/v2/destinations/filters" + d "github.com/shoplineapp/captin/v2/dispatcher" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + outgoing "github.com/shoplineapp/captin/v2/internal/outgoing" + models "github.com/shoplineapp/captin/v2/models" + senders "github.com/shoplineapp/captin/v2/senders" + + captin_errors "github.com/shoplineapp/captin/v2/errors" + documentStores "github.com/shoplineapp/captin/v2/internal/document_stores" + stores "github.com/shoplineapp/captin/v2/internal/stores" + throttles "github.com/shoplineapp/captin/v2/internal/throttles" log "github.com/sirupsen/logrus" ) @@ -116,7 +117,7 @@ func (c Captin) IsRunning() bool { } // Execute - Execute for events -func (c *Captin) Execute(ie interfaces.IncomingEventInterface) (bool, []interfaces.ErrorInterface) { +func (c *Captin) Execute(ctx context.Context, ie interfaces.IncomingEventInterface) (bool, []interfaces.ErrorInterface) { c.Status = STATUS_RUNNING e := ie.(models.IncomingEvent) @@ -131,7 +132,7 @@ func (c *Captin) Execute(ie interfaces.IncomingEventInterface) (bool, []interfac destinations = append(destinations, models.Destination{Config: config}) } - destinations = outgoing.Custom{}.Sift(&e, destinations, c.filters, c.middlewares) + destinations = outgoing.Custom{}.Sift(ctx, &e, destinations, c.filters, c.middlewares) cLogger.WithFields(log.Fields{ "event": e, "destinations": destinations, @@ -143,7 +144,7 @@ func (c *Captin) Execute(ie interfaces.IncomingEventInterface) (bool, []interfac dispatcher.SetMiddlewares(c.dispatchMiddlewares) dispatcher.SetErrorHandler(c.dispatchErrorHandler) dispatcher.SetDelayer(c.dispatchDelayer) - dispatcher.Dispatch(e, c.store, c.throttler, c.DocumentStoreMapping) + dispatcher.Dispatch(ctx, e, c.store, c.throttler, c.DocumentStoreMapping) errors := dispatcher.GetErrors() diff --git a/destinations/filters/base.go b/destinations/filters/base.go index 347c0db..6d43122 100644 --- a/destinations/filters/base.go +++ b/destinations/filters/base.go @@ -1,16 +1,18 @@ package destination_filters import ( - models "github.com/shoplineapp/captin/models" + "context" + + models "github.com/shoplineapp/captin/v2/models" ) // DestinationMiddleware - Interface for third-party application to add extra handling on destinations type DestinationMiddlewareInterface interface { - Apply(e *models.IncomingEvent, d []models.Destination) []models.Destination + Apply(ctx context.Context, e *models.IncomingEvent, d []models.Destination) []models.Destination } // DestinationFilter - Interface for third-party application to filter destination by event type DestinationFilterInterface interface { - Run(e models.IncomingEvent, c models.Destination) (bool, error) - Applicable(e models.IncomingEvent, c models.Destination) bool + Run(ctx context.Context, e models.IncomingEvent, c models.Destination) (bool, error) + Applicable(ctx context.Context, e models.IncomingEvent, c models.Destination) bool } diff --git a/destinations/filters/desired_hook.go b/destinations/filters/desired_hook.go index 0d99ce2..3a84dd0 100644 --- a/destinations/filters/desired_hook.go +++ b/destinations/filters/desired_hook.go @@ -1,7 +1,9 @@ package destination_filters import ( - models "github.com/shoplineapp/captin/models" + "context" + + models "github.com/shoplineapp/captin/v2/models" ) func isPresent(str string, list []string) bool { @@ -27,7 +29,7 @@ type DesiredHookFilter struct { } // Run - Get desired hooks in control and filter out exclusion -func (f DesiredHookFilter) Run(e models.IncomingEvent, d models.Destination) (bool, error) { +func (f DesiredHookFilter) Run(ctx context.Context, e models.IncomingEvent, d models.Destination) (bool, error) { hook := d.Config.GetName() list := e.Control["desired_hooks"] switch list.(type) { @@ -42,6 +44,6 @@ func (f DesiredHookFilter) Run(e models.IncomingEvent, d models.Destination) (bo } // Applicable - Check if desired hooks is present -func (f DesiredHookFilter) Applicable(e models.IncomingEvent, d models.Destination) bool { +func (f DesiredHookFilter) Applicable(ctx context.Context, e models.IncomingEvent, d models.Destination) bool { return e.Control["desired_hooks"] != nil } diff --git a/destinations/filters/environment.go b/destinations/filters/environment.go index 7fed0a1..c231011 100644 --- a/destinations/filters/environment.go +++ b/destinations/filters/environment.go @@ -1,7 +1,9 @@ package destination_filters import ( - models "github.com/shoplineapp/captin/models" + "context" + + models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" ) @@ -12,7 +14,7 @@ type EnvironmentFilter struct { } // Destination needs to be enabled by ENV Variable {Config Name}_ENABLED, e.g, WAPOS_SYNC_ENABLED -func (f EnvironmentFilter) Run(e models.IncomingEvent, d models.Destination) (bool, error) { +func (f EnvironmentFilter) Run(ctx context.Context, e models.IncomingEvent, d models.Destination) (bool, error) { variableName, value := d.Config.GetByEnv("enabled") isEnabled := value != "false" @@ -23,6 +25,6 @@ func (f EnvironmentFilter) Run(e models.IncomingEvent, d models.Destination) (bo return isEnabled, nil } -func (f EnvironmentFilter) Applicable(e models.IncomingEvent, d models.Destination) bool { +func (f EnvironmentFilter) Applicable(ctx context.Context, e models.IncomingEvent, d models.Destination) bool { return true } diff --git a/destinations/filters/source.go b/destinations/filters/source.go index 6c57f71..ff713ce 100644 --- a/destinations/filters/source.go +++ b/destinations/filters/source.go @@ -1,17 +1,19 @@ package destination_filters import ( - models "github.com/shoplineapp/captin/models" + "context" + + models "github.com/shoplineapp/captin/v2/models" ) type SourceFilter struct { DestinationFilterInterface } -func (f SourceFilter) Run(e models.IncomingEvent, d models.Destination) (bool, error) { +func (f SourceFilter) Run(ctx context.Context, e models.IncomingEvent, d models.Destination) (bool, error) { return e.Source != d.Config.GetSource(), nil } -func (f SourceFilter) Applicable(e models.IncomingEvent, d models.Destination) bool { +func (f SourceFilter) Applicable(ctx context.Context, e models.IncomingEvent, d models.Destination) bool { return d.Config.GetAllowLoopback() == false } diff --git a/destinations/filters/validate.go b/destinations/filters/validate.go index 756b300..2326bf5 100644 --- a/destinations/filters/validate.go +++ b/destinations/filters/validate.go @@ -1,10 +1,12 @@ package destination_filters import ( + "context" "encoding/json" "fmt" + "github.com/robertkrimen/otto" - models "github.com/shoplineapp/captin/models" + models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" ) @@ -14,7 +16,7 @@ type ValidateFilter struct { DestinationFilterInterface } -func (f ValidateFilter) Run(e models.IncomingEvent, d models.Destination) (bool, error) { +func (f ValidateFilter) Run(ctx context.Context, e models.IncomingEvent, d models.Destination) (bool, error) { payloadJson, _ := json.Marshal(e.Payload) configJson, _ := json.Marshal(d.Config) template := fmt.Sprintf( @@ -39,6 +41,6 @@ func (f ValidateFilter) Run(e models.IncomingEvent, d models.Destination) (bool, return valid, err } -func (f ValidateFilter) Applicable(e models.IncomingEvent, d models.Destination) bool { +func (f ValidateFilter) Applicable(ctx context.Context, e models.IncomingEvent, d models.Destination) bool { return (d.Config.GetValidate()) != "" } diff --git a/dispatcher/delayers/goroutine.go b/dispatcher/delayers/goroutine.go index df64c7e..64ae28a 100644 --- a/dispatcher/delayers/goroutine.go +++ b/dispatcher/delayers/goroutine.go @@ -1,13 +1,14 @@ package dispatcher_delayers import ( + "context" "fmt" "time" - "github.com/shoplineapp/captin/dispatcher" - "github.com/shoplineapp/captin/interfaces" - "github.com/shoplineapp/captin/models" + "github.com/shoplineapp/captin/v2/dispatcher" + "github.com/shoplineapp/captin/v2/interfaces" + "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" ) @@ -17,7 +18,7 @@ type GoroutineDelayer struct { var dLogger = log.WithFields(log.Fields{"class": "Goroutine"}) -func (d GoroutineDelayer) Execute(evt interfaces.IncomingEventInterface, dest interfaces.DestinationInterface, exec func()) { +func (d GoroutineDelayer) Execute(ctx context.Context, evt interfaces.IncomingEventInterface, dest interfaces.DestinationInterface, exec func()) { event := d.TapDelayedEvent(evt.(models.IncomingEvent), dest.(models.Destination)) config := dest.GetConfig() diff --git a/errors/dispatcher_error.go b/errors/dispatcher_error.go index 66d063f..fe0b725 100644 --- a/errors/dispatcher_error.go +++ b/errors/dispatcher_error.go @@ -2,8 +2,9 @@ package errors import ( "fmt" - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" ) // DispatcherError - Error when send events diff --git a/errors/execution_error.go b/errors/execution_error.go index bd160c1..6b1449a 100644 --- a/errors/execution_error.go +++ b/errors/execution_error.go @@ -2,7 +2,7 @@ package errors import ( "fmt" - interfaces "github.com/shoplineapp/captin/interfaces" + interfaces "github.com/shoplineapp/captin/v2/interfaces" ) // ExecutionError - Error on executing events diff --git a/errors/unretryable_errors.go b/errors/unretryable_errors.go index 5cfc5fc..72c54ee 100644 --- a/errors/unretryable_errors.go +++ b/errors/unretryable_errors.go @@ -3,8 +3,8 @@ package errors import ( "fmt" - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" ) type UnretryableError struct { diff --git a/examples/api/incoming/http.go b/examples/api/incoming/http.go index e884bf4..3a2a7dc 100644 --- a/examples/api/incoming/http.go +++ b/examples/api/incoming/http.go @@ -2,8 +2,8 @@ package incoming import ( "github.com/gin-gonic/gin" - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" "net/http" ) @@ -18,7 +18,7 @@ func (h *HttpEventHandler) Setup(c interfaces.CaptinInterface) { func (h HttpEventHandler) SetRoutes(router *gin.Engine) { router.GET("/", func(c *gin.Context) { - c.String(200, "github.com/shoplineapp/captin aboard") + c.String(200, "github.com/shoplineapp/captin/v2 aboard") }) router.POST("/api/events", func(c *gin.Context) { h.HandleEventCreation(c) @@ -35,7 +35,7 @@ func (h HttpEventHandler) HandleEventCreation(c *gin.Context) { return } - _, errors := h.captin.Execute(event) + _, errors := h.captin.Execute(c, event) if len(errors) > 0 { c.JSON(http.StatusBadRequest, gin.H{"error": "Error occurred when handling event"}) return diff --git a/examples/api/main.go b/examples/api/main.go index 964dcda..7af800b 100644 --- a/examples/api/main.go +++ b/examples/api/main.go @@ -7,10 +7,10 @@ import ( "github.com/gin-gonic/gin" - core "github.com/shoplineapp/captin/core" - incoming "github.com/shoplineapp/captin/incoming" - stores "github.com/shoplineapp/captin/internal/stores" - models "github.com/shoplineapp/captin/models" + core "github.com/shoplineapp/captin/v2/core" + incoming "github.com/shoplineapp/captin/v2/incoming" + stores "github.com/shoplineapp/captin/v2/internal/stores" + models "github.com/shoplineapp/captin/v2/models" ) func main() { diff --git a/examples/api/test/incoming/http_test.go b/examples/api/test/incoming/http_test.go index 84a2862..15805dc 100644 --- a/examples/api/test/incoming/http_test.go +++ b/examples/api/test/incoming/http_test.go @@ -7,9 +7,9 @@ import ( "net/http/httptest" "testing" - . "github.com/shoplineapp/captin/incoming" - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + . "github.com/shoplineapp/captin/v2/incoming" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) diff --git a/examples/redis/incoming/http.go b/examples/redis/incoming/http.go index f120822..ec66781 100644 --- a/examples/redis/incoming/http.go +++ b/examples/redis/incoming/http.go @@ -4,8 +4,8 @@ import ( "net/http" "github.com/gin-gonic/gin" - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" ) type HttpEventHandler struct { @@ -19,7 +19,7 @@ func (h *HttpEventHandler) Setup(c interfaces.CaptinInterface) { func (h HttpEventHandler) SetRoutes(router *gin.Engine) { router.GET("/", func(c *gin.Context) { - c.String(200, "github.com/shoplineapp/captin aboard") + c.String(200, "github.com/shoplineapp/captin/v2 aboard") }) router.POST("/api/events", func(c *gin.Context) { h.HandleEventCreation(c) @@ -36,7 +36,7 @@ func (h HttpEventHandler) HandleEventCreation(c *gin.Context) { return } - _, errs := h.captin.Execute(event) + _, errs := h.captin.Execute(c, event) if len(errs) > 0 { c.JSON(http.StatusBadRequest, gin.H{"error": "Error occurred when handling event"}) return diff --git a/examples/redis/main.go b/examples/redis/main.go index a243b7a..2b78516 100644 --- a/examples/redis/main.go +++ b/examples/redis/main.go @@ -10,8 +10,8 @@ import ( incoming "redis_example/incoming" stores "redis_example/stores" - core "github.com/shoplineapp/captin/core" - "github.com/shoplineapp/captin/models" + core "github.com/shoplineapp/captin/v2/core" + "github.com/shoplineapp/captin/v2/models" ) func main() { diff --git a/examples/redis/stores/redis_store.go b/examples/redis/stores/redis_store.go index eef0d1f..0e3d23f 100644 --- a/examples/redis/stores/redis_store.go +++ b/examples/redis/stores/redis_store.go @@ -4,8 +4,8 @@ import ( "fmt" "time" - interfaces "github.com/shoplineapp/captin/interfaces" - "github.com/shoplineapp/captin/models" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + "github.com/shoplineapp/captin/v2/models" lock "github.com/bsm/redis-lock" "github.com/go-redis/redis" diff --git a/examples/redis/test/incoming/http_test.go b/examples/redis/test/incoming/http_test.go index 6e4f373..c91ae96 100644 --- a/examples/redis/test/incoming/http_test.go +++ b/examples/redis/test/incoming/http_test.go @@ -7,9 +7,9 @@ import ( "net/http/httptest" "testing" - . "github.com/shoplineapp/captin/incoming" - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + . "github.com/shoplineapp/captin/v2/incoming" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) diff --git a/examples/with-filters/filters/target_id.go b/examples/with-filters/filters/target_id.go index e21c351..7dc2447 100644 --- a/examples/with-filters/filters/target_id.go +++ b/examples/with-filters/filters/target_id.go @@ -1,8 +1,8 @@ package filters import ( - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" ) type TargetIdFilter struct { diff --git a/examples/with-filters/main.go b/examples/with-filters/main.go index acef579..8bceddf 100644 --- a/examples/with-filters/main.go +++ b/examples/with-filters/main.go @@ -8,9 +8,9 @@ import ( filters "example-with-filters/filters" middlewares "example-with-filters/middlewares" - core "github.com/shoplineapp/captin/core" - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + core "github.com/shoplineapp/captin/v2/core" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" ) func main() { diff --git a/examples/with-filters/middlewares/logger.go b/examples/with-filters/middlewares/logger.go index 202fa36..47f7dba 100644 --- a/examples/with-filters/middlewares/logger.go +++ b/examples/with-filters/middlewares/logger.go @@ -2,7 +2,7 @@ package middlewares import ( "fmt" - models "github.com/shoplineapp/captin/models" + models "github.com/shoplineapp/captin/v2/models" ) type LoggerMiddleware struct{} diff --git a/go.mod b/go.mod index 57ecabb..6576539 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/shoplineapp/captin +module github.com/shoplineapp/captin/v2 go 1.15 @@ -12,8 +12,11 @@ require ( github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d github.com/sirupsen/logrus v1.4.2 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.4 github.com/thoas/go-funk v0.7.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 + go.opentelemetry.io/otel/trace v1.21.0 golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect diff --git a/go.sum b/go.sum index aed3706..82519ec 100644 --- a/go.sum +++ b/go.sum @@ -5,7 +5,16 @@ github.com/beanstalkd/go-beanstalk v0.0.0-20190515041346-390b03b3064a/go.mod h1: github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -14,8 +23,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joeycumines/statsd v1.0.1-0.20201117043332-bb35aa955658 h1:qg1swZu2+awU2o2Vq0HiIfbvyUBV0MnCeG/BKoXN+Dg= github.com/joeycumines/statsd v1.0.1-0.20201117043332-bb35aa955658/go.mod h1:SLKAkQ5CgPBRFFIv3JAjQjBWEOmJJxHn33bwAnFFVMU= -github.com/joeycumines/statsd v1.3.0 h1:Rc1ST4OF518jOVmNO8jdh0pBqX42eC1hBr2Px1V/jYw= -github.com/joeycumines/statsd v1.3.0/go.mod h1:SLKAkQ5CgPBRFFIv3JAjQjBWEOmJJxHn33bwAnFFVMU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -34,14 +41,26 @@ github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d/go.mod h1:xvqspo github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/thoas/go-funk v0.7.0 h1:GmirKrs6j6zJbhJIficOsz2aAI7700KsU/5YrdHRM1Y= github.com/thoas/go-funk v0.7.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -59,5 +78,6 @@ gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/interfaces/dispatcher.go b/interfaces/dispatcher.go index aef7829..f23e9a8 100644 --- a/interfaces/dispatcher.go +++ b/interfaces/dispatcher.go @@ -1,6 +1,8 @@ package interfaces +import "context" + // DispatchDelayerInterface - class for deciding how event message is delayed type DispatchDelayerInterface interface { - Execute(e IncomingEventInterface, d DestinationInterface, exec func()) () + Execute(ctx context.Context, e IncomingEventInterface, d DestinationInterface, exec func()) } diff --git a/interfaces/document_store.go b/interfaces/document_store.go index 0d92d08..6cfe1eb 100644 --- a/interfaces/document_store.go +++ b/interfaces/document_store.go @@ -1,7 +1,9 @@ package interfaces +import "context" + // StoreInterface - Store for throttle events type DocumentStoreInterface interface { - // GetDocument - Get value from store, return the document map - GetDocument(e IncomingEventInterface) (map[string]interface{}) + // GetDocument - Get value from store, return the document map + GetDocument(ctx context.Context, e IncomingEventInterface) map[string]interface{} } diff --git a/interfaces/interfaces.go b/interfaces/interfaces.go index 179217e..0e8805a 100644 --- a/interfaces/interfaces.go +++ b/interfaces/interfaces.go @@ -1,18 +1,19 @@ package interfaces import ( + "context" "time" ) // CaptinInterface - Captin Interface type CaptinInterface interface { - Execute(e IncomingEventInterface) (bool, []ErrorInterface) + Execute(ctx context.Context, e IncomingEventInterface) (bool, []ErrorInterface) IsRunning() bool } // EventSenderInterface - Event Sender Interface type EventSenderInterface interface { - SendEvent(e IncomingEventInterface, d DestinationInterface) error + SendEvent(ctx context.Context, e IncomingEventInterface, d DestinationInterface) error } // ThrottleInterface - interface for a throttle object @@ -23,9 +24,9 @@ type EventSenderInterface interface { // Throttle: t t2 type ThrottleInterface interface { // CanTrigger - Check if can trigger - CanTrigger(id string, period time.Duration) (bool, time.Duration, error) + CanTrigger(ctx context.Context, id string, period time.Duration) (canTrigger bool, ttl time.Duration, err error) } type ErrorHandlerInterface interface { - Exec(e ErrorInterface) + Exec(ctx context.Context, e ErrorInterface) } diff --git a/interfaces/store.go b/interfaces/store.go index 03cf918..e8c9153 100644 --- a/interfaces/store.go +++ b/interfaces/store.go @@ -1,26 +1,27 @@ package interfaces import ( + "context" "time" ) // StoreInterface - Store for throttle events type StoreInterface interface { // Get - Get value from store, return with remaining time - Get(key string) (string, bool, time.Duration, error) + Get(ctx context.Context, key string) (payload string, exists bool, ttl time.Duration, err error) // Set - Set value into store with ttl - Set(key string, value string, ttl time.Duration) (bool, error) + Set(ctx context.Context, key string, value string, ttl time.Duration) (bool, error) // Update - Update value for key - Update(key string, value string) (bool, error) + Update(ctx context.Context, key string, value string) (bool, error) // Remove - Remove value for key - Remove(key string) (bool, error) + Remove(ctx context.Context, key string) (bool, error) - DataKey(e IncomingEventInterface, dest DestinationInterface, prefix string, suffix string) string + DataKey(ctx context.Context, e IncomingEventInterface, dest DestinationInterface, prefix string, suffix string) string - Enqueue(key string, value string, ttl time.Duration) (bool, error) + Enqueue(ctx context.Context, key string, value string, ttl time.Duration) (bool, error) - GetQueue(key string) ([]string, bool, time.Duration, error) + GetQueue(ctx context.Context, key string) (values []string, exists bool, ttl time.Duration, err error) } diff --git a/internal/document_stores/null_store.go b/internal/document_stores/null_store.go index 1bcf3ff..3c55ca2 100644 --- a/internal/document_stores/null_store.go +++ b/internal/document_stores/null_store.go @@ -1,7 +1,9 @@ package document_stores import ( - interfaces "github.com/shoplineapp/captin/interfaces" + "context" + + interfaces "github.com/shoplineapp/captin/v2/interfaces" ) // NullDocumentStore - Null data store @@ -11,11 +13,10 @@ type NullDocumentStore struct { // NewNullDocumentStore - Create new NullDocumentStore func NewNullDocumentStore() *NullDocumentStore { - return &NullDocumentStore{} + return &NullDocumentStore{} } // Get - Get value from store, return with remaining time -func (ms NullDocumentStore) GetDocument(e interfaces.IncomingEventInterface) (map[string]interface{}) { - return map[string]interface{}{} +func (ms NullDocumentStore) GetDocument(ctx context.Context, e interfaces.IncomingEventInterface) map[string]interface{} { + return map[string]interface{}{} } - diff --git a/internal/outgoing/custom.go b/internal/outgoing/custom.go index fb961b6..e9ab362 100644 --- a/internal/outgoing/custom.go +++ b/internal/outgoing/custom.go @@ -1,8 +1,9 @@ package outgoing import ( - destination_filters "github.com/shoplineapp/captin/destinations/filters" - models "github.com/shoplineapp/captin/models" + "context" + + destination_filters "github.com/shoplineapp/captin/v2/destinations/filters" log "github.com/sirupsen/logrus" ) @@ -11,7 +12,7 @@ var cLogger = log.WithFields(log.Fields{"class": "Custom"}) type Custom struct{} // Sift - Custom check will filter ineligible destination -func (c Custom) Sift(e *models.IncomingEvent, destinations []models.Destination, filters []destination_filters.DestinationFilterInterface, middlewares []destination_filters.DestinationMiddlewareInterface) []models.Destination { +func (c Custom) Sift(ctx context.Context, e *models.IncomingEvent, destinations []models.Destination, filters []destination_filters.DestinationFilterInterface, middlewares []destination_filters.DestinationMiddlewareInterface) []models.Destination { cLogger.WithFields(log.Fields{ "event": e, "destinations": destinations, @@ -22,12 +23,13 @@ func (c Custom) Sift(e *models.IncomingEvent, destinations []models.Destination, for _, destination := range destinations { eligible := true for _, filter := range filters { - if eligible == false || filter.Applicable(*e, destination) == false { + if !filter.Applicable(ctx, *e, destination) { continue } - valid, _ := filter.Run(*e, destination) - if valid != true { + valid, _ := filter.Run(ctx, *e, destination) + if !valid { eligible = false + break } } if eligible { diff --git a/internal/outgoing/dispatcher.go b/internal/outgoing/dispatcher.go index 6a229c7..9774d13 100644 --- a/internal/outgoing/dispatcher.go +++ b/internal/outgoing/dispatcher.go @@ -1,6 +1,7 @@ package outgoing import ( + "context" "encoding/json" "fmt" "strconv" @@ -8,13 +9,13 @@ import ( "time" "github.com/mohae/deepcopy" - destination_filters "github.com/shoplineapp/captin/destinations/filters" - "github.com/shoplineapp/captin/dispatcher" - captin_errors "github.com/shoplineapp/captin/errors" - interfaces "github.com/shoplineapp/captin/interfaces" - documentStores "github.com/shoplineapp/captin/internal/document_stores" - helpers "github.com/shoplineapp/captin/internal/helpers" - models "github.com/shoplineapp/captin/models" + destination_filters "github.com/shoplineapp/captin/v2/destinations/filters" + "github.com/shoplineapp/captin/v2/dispatcher" + captin_errors "github.com/shoplineapp/captin/v2/errors" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + documentStores "github.com/shoplineapp/captin/v2/internal/document_stores" + "github.com/shoplineapp/captin/v2/internal/helpers" + models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" ) @@ -74,7 +75,7 @@ func (d *Dispatcher) GetErrors() []interfaces.ErrorInterface { return d.Errors } -func (d *Dispatcher) OnError(evt interfaces.IncomingEventInterface, err interfaces.ErrorInterface) { +func (d *Dispatcher) OnError(ctx context.Context, evt interfaces.IncomingEventInterface, err interfaces.ErrorInterface) { d.muErrors.Lock() defer d.muErrors.Unlock() d.Errors = append(d.Errors, err) @@ -86,7 +87,7 @@ func (d *Dispatcher) OnError(evt interfaces.IncomingEventInterface, err interfac "destination": dispatcherErr.Destination, "reason": dispatcherErr.Error(), }).Error("Failed to dispatch event") - d.TriggerErrorHandler(dispatcherErr) + d.TriggerErrorHandler(ctx, dispatcherErr) default: dLogger.WithFields(log.Fields{"event": evt, "error": err}).Error("Unhandled error on dispatcher") } @@ -94,6 +95,7 @@ func (d *Dispatcher) OnError(evt interfaces.IncomingEventInterface, err interfac // Dispatch - Dispatch an event to outgoing webhook func (d *Dispatcher) Dispatch( + ctx context.Context, event interfaces.IncomingEventInterface, store interfaces.StoreInterface, throttler interfaces.ThrottleInterface, @@ -103,7 +105,7 @@ func (d *Dispatcher) Dispatch( e := event.(models.IncomingEvent) for _, destination := range d.destinations { config := destination.Config - canTrigger, timeRemain, err := throttler.CanTrigger(getEventKey(store, e, destination), config.GetThrottleValue()) + canTrigger, timeRemain, err := throttler.CanTrigger(ctx, getEventKey(ctx, store, e, destination), config.GetThrottleValue()) documentStore := d.getDocumentStore(destination, documentStoreMappings) if err != nil { @@ -111,7 +113,7 @@ func (d *Dispatcher) Dispatch( // Send without throttling go func(e models.IncomingEvent, destination models.Destination, documentStore interfaces.DocumentStoreInterface) { - d.sendEvent(e, destination, store, documentStore) + d.sendEvent(ctx, e, destination, store, documentStore) responses <- 1 }(e, destination, documentStore) continue @@ -119,14 +121,14 @@ func (d *Dispatcher) Dispatch( if canTrigger { go func(e models.IncomingEvent, destination models.Destination, documentStore interfaces.DocumentStoreInterface) { - d.sendEvent(e, destination, store, documentStore) + d.sendEvent(ctx, e, destination, store, documentStore) responses <- 1 }(e, destination, documentStore) } else if !config.GetThrottleTrailingDisabled() { - go func(e models.IncomingEvent, destination models.Destination, documentStore interfaces.DocumentStoreInterface) { - d.processDelayedEvent(e, timeRemain, destination, store, documentStore) + go func(ctx context.Context, e models.IncomingEvent, destination models.Destination, documentStore interfaces.DocumentStoreInterface) { + d.processDelayedEvent(ctx, e, timeRemain, destination, store, documentStore) responses <- 1 - }(e, destination, documentStore) + }(ctx, e, destination, documentStore) } else { dLogger.WithFields(log.Fields{"event": e, "destination": destination}).Info("Cannot trigger send event") responses <- 0 @@ -139,10 +141,10 @@ func (d *Dispatcher) Dispatch( return nil } -func (d *Dispatcher) TriggerErrorHandler(err *captin_errors.DispatcherError) { +func (d *Dispatcher) TriggerErrorHandler(ctx context.Context, err *captin_errors.DispatcherError) { if d.errorHandler != nil { dispatcher.TrackGoRoutine(func() { - d.errorHandler.Exec(*err) + d.errorHandler.Exec(ctx, *err) }) } } @@ -157,20 +159,20 @@ func (d *Dispatcher) getDocumentStore(dest models.Destination, documentStoreMapp } // inject document and sanitize fields in event based on destination -func (d *Dispatcher) customizeEvent(e models.IncomingEvent, destination models.Destination, documentStore interfaces.DocumentStoreInterface) interfaces.IncomingEventInterface { +func (d *Dispatcher) customizeEvent(ctx context.Context, e models.IncomingEvent, destination models.Destination, documentStore interfaces.DocumentStoreInterface) interfaces.IncomingEventInterface { customized := e - customized.TargetDocument = d.customizeDocument(&customized, destination, documentStore) - customized.Payload = d.customizePayload(customized, destination) + customized.TargetDocument = d.customizeDocument(ctx, &customized, destination, documentStore) + customized.Payload = d.customizePayload(ctx, customized, destination) return customized } // inject throttled payloads from store if keep_throttled_payloads is true -func (d *Dispatcher) injectThrottledPayloads(e models.IncomingEvent, destination models.Destination, store interfaces.StoreInterface) interfaces.IncomingEventInterface { +func (d *Dispatcher) injectThrottledPayloads(ctx context.Context, e models.IncomingEvent, destination models.Destination, store interfaces.StoreInterface) interfaces.IncomingEventInterface { if destination.Config.GetKeepThrottledPayloads() { - queueKey := getEventThrottledPayloadsKey(store, e, destination) - payloadStrings, _, _, _ := store.GetQueue(queueKey) - store.Remove(queueKey) + queueKey := getEventThrottledPayloadsKey(ctx, store, e, destination) + payloadStrings, _, _, _ := store.GetQueue(ctx, queueKey) + store.Remove(ctx, queueKey) for _, payloadStr := range payloadStrings { payload := map[string]interface{}{} json.Unmarshal([]byte(payloadStr), &payload) @@ -181,11 +183,11 @@ func (d *Dispatcher) injectThrottledPayloads(e models.IncomingEvent, destination } // inject throttled documents from store if include_document and keep_throttled_documents is true -func (d *Dispatcher) injectThrottledDocuments(e models.IncomingEvent, destination models.Destination, store interfaces.StoreInterface) interfaces.IncomingEventInterface { +func (d *Dispatcher) injectThrottledDocuments(ctx context.Context, e models.IncomingEvent, destination models.Destination, store interfaces.StoreInterface) interfaces.IncomingEventInterface { if destination.Config.GetIncludeDocument() && destination.Config.GetKeepThrottledDocuments() { - queueKey := getEventThrottledDocumentsKey(store, e, destination) - documentStrings, _, _, _ := store.GetQueue(queueKey) - store.Remove(queueKey) + queueKey := getEventThrottledDocumentsKey(ctx, store, e, destination) + documentStrings, _, _, _ := store.GetQueue(ctx, queueKey) + store.Remove(ctx, queueKey) for _, documentStr := range documentStrings { document := map[string]interface{}{} json.Unmarshal([]byte(documentStr), &document) @@ -195,7 +197,7 @@ func (d *Dispatcher) injectThrottledDocuments(e models.IncomingEvent, destinatio return e } -func (d *Dispatcher) customizeDocument(e *models.IncomingEvent, destination models.Destination, documentStore interfaces.DocumentStoreInterface) map[string]interface{} { +func (d *Dispatcher) customizeDocument(ctx context.Context, e *models.IncomingEvent, destination models.Destination, documentStore interfaces.DocumentStoreInterface) map[string]interface{} { config := destination.Config if config.GetIncludeDocument() == false { return e.TargetDocument @@ -206,7 +208,7 @@ func (d *Dispatcher) customizeDocument(e *models.IncomingEvent, destination mode // memoize document to be used across events for diff. destinations if d.targetDocument == nil { - d.targetDocument = documentStore.GetDocument(*e) + d.targetDocument = documentStore.GetDocument(ctx, *e) } if len(config.GetIncludeDocumentAttrs()) >= 1 { @@ -218,7 +220,7 @@ func (d *Dispatcher) customizeDocument(e *models.IncomingEvent, destination mode } } -func (d *Dispatcher) customizePayload(e models.IncomingEvent, destination interfaces.DestinationInterface) map[string]interface{} { +func (d *Dispatcher) customizePayload(ctx context.Context, e models.IncomingEvent, destination interfaces.DestinationInterface) map[string]interface{} { config := destination.(models.Destination).Config if len(config.GetIncludePayloadAttrs()) >= 1 { return helpers.IncludeFields(e.Payload, config.GetIncludePayloadAttrs()).(map[string]interface{}) @@ -229,20 +231,21 @@ func (d *Dispatcher) customizePayload(e models.IncomingEvent, destination interf return e.Payload } -func (d *Dispatcher) processDelayedEvent(e models.IncomingEvent, timeRemain time.Duration, dest models.Destination, store interfaces.StoreInterface, documentStore interfaces.DocumentStoreInterface) { +func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingEvent, timeRemain time.Duration, dest models.Destination, store interfaces.StoreInterface, documentStore interfaces.DocumentStoreInterface) { defer func() { if err := recover(); err != nil { - d.OnError(e, &captin_errors.DispatcherError{ + err := &captin_errors.DispatcherError{ Msg: err.(error).Error(), Destination: dest, Event: e, - }) + } + d.OnError(ctx, e, err) } }() // Check if store have payload - dataKey := getEventDataKey(store, e, dest) - storedData, dataExists, _, storeErr := store.Get(dataKey) + dataKey := getEventDataKey(ctx, store, e, dest) + storedData, dataExists, _, storeErr := store.Get(ctx, dataKey) if storeErr != nil { panic(storeErr) } @@ -264,33 +267,37 @@ func (d *Dispatcher) processDelayedEvent(e models.IncomingEvent, timeRemain time } if dest.Config.GetKeepThrottledPayloads() { - customizedPayload := d.customizePayload(e, dest) - queueKey := getEventThrottledPayloadsKey(store, e, dest) + customizedPayload := d.customizePayload(ctx, e, dest) + queueKey := getEventThrottledPayloadsKey(ctx, store, e, dest) jsonString, jsonErr := json.Marshal(customizedPayload) if jsonErr != nil { panic(jsonErr) } + ttl := dest.Config.GetThrottleValue() * 2 dLogger.WithFields(log.Fields{ "queueKey": queueKey, "event": e, "enqueuePayload": jsonString, + "ttl": ttl, }).Debug("Storing throttled payload") - store.Enqueue(queueKey, string(jsonString), dest.Config.GetThrottleValue()*2) + store.Enqueue(ctx, queueKey, string(jsonString), ttl) } if dest.Config.GetIncludeDocument() && dest.Config.GetKeepThrottledDocuments() { - customizedDocument := d.customizeDocument(&e, dest, documentStore) - queueKey := getEventThrottledDocumentsKey(store, e, dest) + customizedDocument := d.customizeDocument(ctx, &e, dest, documentStore) + queueKey := getEventThrottledDocumentsKey(ctx, store, e, dest) jsonString, jsonErr := json.Marshal(customizedDocument) if jsonErr != nil { panic(jsonErr) } + ttl := dest.Config.GetThrottleValue() * 2 dLogger.WithFields(log.Fields{ "queueKey": queueKey, "event": e, "enqueueDocument": jsonString, + "ttl": ttl, }).Debug("Storing throttled document") - store.Enqueue(queueKey, string(jsonString), dest.Config.GetThrottleValue()*2) + store.Enqueue(ctx, queueKey, string(jsonString), ttl) } jsonString, jsonErr := e.ToJson() @@ -300,14 +307,14 @@ func (d *Dispatcher) processDelayedEvent(e models.IncomingEvent, timeRemain time if dataExists { // Update Value - _, updateErr := store.Update(dataKey, string(jsonString)) + _, updateErr := store.Update(ctx, dataKey, string(jsonString)) if updateErr != nil { panic(updateErr) } } else { // Create Value config := dest.Config - _, saveErr := store.Set(dataKey, string(jsonString), config.GetThrottleValue()*2) + _, saveErr := store.Set(ctx, dataKey, string(jsonString), config.GetThrottleValue()*2) if saveErr != nil { panic(saveErr) } @@ -319,13 +326,14 @@ func (d *Dispatcher) processDelayedEvent(e models.IncomingEvent, timeRemain time // Key might be deleted by another worker, resulting in data not found if !exists { dLogger.WithFields(log.Fields{"key": dataKey}).Debug("Event data not found") - d.OnError(models.IncomingEvent{}, &captin_errors.UnretryableError{Msg: "Event data not found", Event: e, Destination: dest}) + err := &captin_errors.UnretryableError{Msg: "Event data not found", Event: e, Destination: dest} + d.OnError(ctx, models.IncomingEvent{}, err) return } event := models.IncomingEvent{} json.Unmarshal([]byte(payload), &event) - d.sendEvent(event, dest, store, documentStore) - store.Remove(dataKey) + d.sendEvent(ctx, event, dest, store, documentStore) + store.Remove(ctx, dataKey) }) } } @@ -355,23 +363,23 @@ func getControlTimestamp(e models.IncomingEvent, defaultValue uint64) uint64 { return value.(uint64) } -func getEventKey(s interfaces.StoreInterface, e interfaces.IncomingEventInterface, d interfaces.DestinationInterface) string { - return s.DataKey(e, d, "", "") +func getEventKey(ctx context.Context, s interfaces.StoreInterface, e interfaces.IncomingEventInterface, d interfaces.DestinationInterface) string { + return s.DataKey(ctx, e, d, "", "") } -func getEventDataKey(s interfaces.StoreInterface, e interfaces.IncomingEventInterface, d interfaces.DestinationInterface) string { - return s.DataKey(e, d, "", "-data") +func getEventDataKey(ctx context.Context, s interfaces.StoreInterface, e interfaces.IncomingEventInterface, d interfaces.DestinationInterface) string { + return s.DataKey(ctx, e, d, "", "-data") } -func getEventThrottledPayloadsKey(s interfaces.StoreInterface, e models.IncomingEvent, d models.Destination) string { - return s.DataKey(e, d, "", "-throttled_payloads") +func getEventThrottledPayloadsKey(ctx context.Context, s interfaces.StoreInterface, e models.IncomingEvent, d models.Destination) string { + return s.DataKey(ctx, e, d, "", "-throttled_payloads") } -func getEventThrottledDocumentsKey(s interfaces.StoreInterface, e models.IncomingEvent, d models.Destination) string { - return s.DataKey(e, d, "", "-throttled_documents") +func getEventThrottledDocumentsKey(ctx context.Context, s interfaces.StoreInterface, e models.IncomingEvent, d models.Destination) string { + return s.DataKey(ctx, e, d, "", "-throttled_documents") } -func (d *Dispatcher) sendEvent(evt models.IncomingEvent, destination models.Destination, store interfaces.StoreInterface, documentStore interfaces.DocumentStoreInterface) { +func (d *Dispatcher) sendEvent(ctx context.Context, evt models.IncomingEvent, destination models.Destination, store interfaces.StoreInterface, documentStore interfaces.DocumentStoreInterface) { config := destination.Config callbackLogger := dLogger.WithFields(log.Fields{ "action": evt.Key, @@ -383,8 +391,9 @@ func (d *Dispatcher) sendEvent(evt models.IncomingEvent, destination models.Dest defer func() { if err := recover(); err != nil { - callbackLogger.Info(fmt.Sprintf("Event failed sending to %s [%s]", config.GetName(), destination.GetCallbackURL())) - d.OnError(evt, &captin_errors.DispatcherError{ + errMsg := fmt.Sprintf("Event failed sending to %s [%s]", config.GetName(), destination.GetCallbackURL()) + callbackLogger.Info(errMsg) + d.OnError(ctx, evt, &captin_errors.DispatcherError{ Msg: err.(error).Error(), Destination: destination, Event: evt, @@ -395,15 +404,15 @@ func (d *Dispatcher) sendEvent(evt models.IncomingEvent, destination models.Dest callbackLogger.Debug("Preprocess payload and document") - evt = d.customizeEvent(evt, destination, documentStore).(models.IncomingEvent) + evt = d.customizeEvent(ctx, evt, destination, documentStore).(models.IncomingEvent) - evt = d.injectThrottledPayloads(evt, destination, store).(models.IncomingEvent) + evt = d.injectThrottledPayloads(ctx, evt, destination, store).(models.IncomingEvent) - evt = d.injectThrottledDocuments(evt, destination, store).(models.IncomingEvent) + evt = d.injectThrottledDocuments(ctx, evt, destination, store).(models.IncomingEvent) callbackLogger.Debug("Final sift on dispatcher") - sifted := Custom{}.Sift(&evt, []models.Destination{destination}, d.filters, d.middlewares) + sifted := Custom{}.Sift(ctx, &evt, []models.Destination{destination}, d.filters, d.middlewares) if len(sifted) == 0 { callbackLogger.Info("Event interrupted by dispatcher filters") return @@ -440,12 +449,12 @@ func (d *Dispatcher) sendEvent(evt models.IncomingEvent, destination models.Dest Event: evt, } } - d.OnError(evt, newErr) + d.OnError(ctx, evt, newErr) } }() // Deep clone a new instance to prevent concurrent iteration and write on json.Marshal event := deepcopy.Copy(evt).(models.IncomingEvent) - err := sender.SendEvent(event, destination) + err := sender.SendEvent(ctx, event, destination) if err != nil { panic(err) } @@ -458,7 +467,7 @@ func (d *Dispatcher) sendEvent(evt models.IncomingEvent, destination models.Dest if d.delayer != nil { // Delayer will usually modify event.Control for delay info, deep clone to prevent concurrent write event := deepcopy.Copy(evt).(models.IncomingEvent) - d.delayer.Execute(event, destination, _sendEvent) + d.delayer.Execute(ctx, event, destination, _sendEvent) return } else { callbackLogger.Warn(fmt.Sprintf("Delayer not found, send event immediately")) diff --git a/internal/stores/memstore.go b/internal/stores/memstore.go index 724e670..705faaa 100644 --- a/internal/stores/memstore.go +++ b/internal/stores/memstore.go @@ -1,12 +1,13 @@ package stores import ( + "context" "fmt" "sync" "time" - interfaces "github.com/shoplineapp/captin/interfaces" - "github.com/shoplineapp/captin/models" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" ) @@ -48,7 +49,7 @@ func NewMemoryStore() *MemoryStore { } // Get - Get value from store, return with remaining time -func (ms *MemoryStore) Get(key string) (string, bool, time.Duration, error) { +func (ms *MemoryStore) Get(_ context.Context, key string) (string, bool, time.Duration, error) { ms.lock.Lock() defer ms.lock.Unlock() @@ -60,7 +61,7 @@ func (ms *MemoryStore) Get(key string) (string, bool, time.Duration, error) { } // Set - Set value into store with ttl -func (ms *MemoryStore) Set(key string, value string, ttl time.Duration) (bool, error) { +func (ms *MemoryStore) Set(_ context.Context, key string, value string, ttl time.Duration) (bool, error) { ms.lock.Lock() it, ok := ms.m[key] if !ok { @@ -74,7 +75,7 @@ func (ms *MemoryStore) Set(key string, value string, ttl time.Duration) (bool, e } // Update - Update value into store -func (ms *MemoryStore) Update(key string, value string) (bool, error) { +func (ms *MemoryStore) Update(_ context.Context, key string, value string) (bool, error) { ms.lock.Lock() defer ms.lock.Unlock() @@ -88,7 +89,7 @@ func (ms *MemoryStore) Update(key string, value string) (bool, error) { } // Remove - Remove value in store -func (ms *MemoryStore) Remove(key string) (bool, error) { +func (ms *MemoryStore) Remove(_ context.Context, key string) (bool, error) { ms.lock.Lock() delete(ms.m, key) ms.lock.Unlock() @@ -96,7 +97,7 @@ func (ms *MemoryStore) Remove(key string) (bool, error) { } // Enqueue - ttl: optional params for setting the ttl of queue when first element is enqueued -func (ms *MemoryStore) Enqueue(key string, value string, ttl time.Duration) (bool, error) { +func (ms *MemoryStore) Enqueue(_ context.Context, key string, value string, ttl time.Duration) (bool, error) { ms.lock.Lock() _, ok := ms.m[key] if !ok { @@ -113,7 +114,7 @@ func (ms *MemoryStore) Enqueue(key string, value string, ttl time.Duration) (boo return true, nil } -func (ms *MemoryStore) GetQueue(key string) ([]string, bool, time.Duration, error) { +func (ms *MemoryStore) GetQueue(_ context.Context, key string) ([]string, bool, time.Duration, error) { ms.lock.Lock() defer ms.lock.Unlock() @@ -133,7 +134,7 @@ func (ms *MemoryStore) Len() int { } // DataKey - Generate DataKey with events and destination -func (ms *MemoryStore) DataKey(ev interfaces.IncomingEventInterface, dest interfaces.DestinationInterface, prefix string, suffix string) string { +func (ms *MemoryStore) DataKey(_ context.Context, ev interfaces.IncomingEventInterface, dest interfaces.DestinationInterface, prefix string, suffix string) string { e := ev.(models.IncomingEvent) config := dest.(models.Destination).Config return fmt.Sprintf("%s%s.%s.%s%s", prefix, e.Key, config.GetName(), e.TargetId, suffix) diff --git a/internal/throttles/throttler.go b/internal/throttles/throttler.go index 3079182..6c511bc 100644 --- a/internal/throttles/throttler.go +++ b/internal/throttles/throttler.go @@ -1,9 +1,10 @@ package throttles import ( + "context" "time" - interfaces "github.com/shoplineapp/captin/interfaces" + interfaces "github.com/shoplineapp/captin/v2/interfaces" log "github.com/sirupsen/logrus" ) @@ -22,14 +23,13 @@ func NewThrottler(store interfaces.StoreInterface) *Throttler { } } -// CanTrigger - Check if can trigger -func (t *Throttler) CanTrigger(id string, period time.Duration) (bool, time.Duration, error) { +func (t *Throttler) CanTrigger(ctx context.Context, id string, period time.Duration) (bool, time.Duration, error) { // ignore throttle if no period is given if period == time.Duration(0) { return true, time.Duration(0), nil } - val, ok, duration, err := t.store.Get(id) + val, ok, duration, err := t.store.Get(ctx, id) if err != nil { return true, time.Duration(0), err @@ -37,7 +37,7 @@ func (t *Throttler) CanTrigger(id string, period time.Duration) (bool, time.Dura tLogger.WithFields(log.Fields{"value": val}).Debug("Check throttle value on CanTrigger") if !ok { tLogger.WithFields(log.Fields{"period": period}).Debug("Throttle value not set, creating...") - t.store.Set(id, "1", period) + t.store.Set(ctx, id, "1", period) return true, time.Duration(0), nil } diff --git a/models/config.go b/models/config.go index 399bcab..1945eed 100644 --- a/models/config.go +++ b/models/config.go @@ -7,7 +7,8 @@ import ( "strconv" "strings" "time" - "github.com/shoplineapp/captin/interfaces" + + "github.com/shoplineapp/captin/v2/interfaces" ) // Configuration - Webhook Configuration Model diff --git a/models/config_mapper.go b/models/config_mapper.go index 888cb84..0c55486 100644 --- a/models/config_mapper.go +++ b/models/config_mapper.go @@ -5,7 +5,7 @@ import ( "io/ioutil" log "github.com/sirupsen/logrus" - interfaces "github.com/shoplineapp/captin/interfaces" + interfaces "github.com/shoplineapp/captin/v2/interfaces" ) var cmLogger = log.WithFields(log.Fields{"class": "ConfigurationMapper"}) diff --git a/models/destination.go b/models/destination.go index c140d8d..883d527 100644 --- a/models/destination.go +++ b/models/destination.go @@ -1,19 +1,20 @@ package models import ( - interfaces "github.com/shoplineapp/captin/interfaces" - "os" "fmt" - "time" - "strings" + "os" "strconv" + "strings" + "time" + + interfaces "github.com/shoplineapp/captin/v2/interfaces" ) // Destination - Event dispatch destination type Destination struct { interfaces.DestinationInterface - Config interfaces.ConfigurationInterface + Config interfaces.ConfigurationInterface callbackUrl string } @@ -55,8 +56,8 @@ func (d Destination) GetDocumentStore() string { } func (d Destination) RequireDelay(evt interfaces.IncomingEventInterface) bool { - if (d.Config.GetDelayValue() <= time.Duration(0) || - evt.GetOutstandingDelaySeconds() == time.Duration(0)) { + if d.Config.GetDelayValue() <= time.Duration(0) || + evt.GetOutstandingDelaySeconds() == time.Duration(0) { return false } @@ -76,7 +77,7 @@ func (d Destination) GetRetryBackoffSeconds(evt interfaces.IncomingEventInterfac control := evt.GetControl() retryCount, _ := control["retry_count"].(float64) - if (len(backoffConfig) > 0 && len(backoffConfig) <= int(retryCount)) { + if len(backoffConfig) > 0 && len(backoffConfig) <= int(retryCount) { lastConfig := backoffConfig[(len(backoffConfig) - 1):] seconds, pErr := strconv.ParseInt(lastConfig[0], 10, 64) if pErr != nil { @@ -92,7 +93,6 @@ func (d Destination) GetRetryBackoffSeconds(evt interfaces.IncomingEventInterfac return seconds } - func trimArray(arr []string) []string { var r []string for _, str := range arr { diff --git a/models/incoming_event.go b/models/incoming_event.go index 81670d6..df1425b 100644 --- a/models/incoming_event.go +++ b/models/incoming_event.go @@ -9,7 +9,7 @@ import ( "github.com/google/uuid" - interfaces "github.com/shoplineapp/captin/interfaces" + interfaces "github.com/shoplineapp/captin/v2/interfaces" ) type IncomingEvent struct { diff --git a/senders/beanstlakd_sender.go b/senders/beanstlakd_sender.go index 8b90ec9..5f424d1 100644 --- a/senders/beanstlakd_sender.go +++ b/senders/beanstlakd_sender.go @@ -1,6 +1,7 @@ package senders import ( + "context" "encoding/json" "fmt" "net" @@ -10,9 +11,9 @@ import ( beanstalk "github.com/beanstalkd/go-beanstalk" statsd "github.com/joeycumines/statsd" - captin_errors "github.com/shoplineapp/captin/errors" - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + captin_errors "github.com/shoplineapp/captin/v2/errors" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" ) @@ -29,7 +30,7 @@ type BeanstalkdSender struct { } // SendEvent - #BeanstalkdSender SendEvent -func (c *BeanstalkdSender) SendEvent(ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) error { +func (c *BeanstalkdSender) SendEvent(ctx context.Context, ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) (err error) { e := ev.(models.IncomingEvent) d := dv.(models.Destination) diff --git a/senders/console_sender.go b/senders/console_sender.go index 0b904d5..4eafc6e 100644 --- a/senders/console_sender.go +++ b/senders/console_sender.go @@ -1,8 +1,10 @@ package senders import ( - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + "context" + + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" ) @@ -12,14 +14,14 @@ var cLogger = log.WithFields(log.Fields{"class": "ConsoleEventSender"}) type ConsoleEventSender struct{} // SendEvent - #ConsoleEventSender SendEvent -func (c *ConsoleEventSender) SendEvent(ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) error { +func (c *ConsoleEventSender) SendEvent(ctx context.Context, ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) error { e := ev.(models.IncomingEvent) d := dv.(models.Destination) cLogger.WithFields(log.Fields{ - "config_name": d.Config.GetName(), - "target_id": e.TargetId, - "target_type": e.TargetType, + "config_name": d.Config.GetName(), + "target_id": e.TargetId, + "target_type": e.TargetType, "target_document": e.TargetDocument, }).Debug("Event sent") return nil diff --git a/senders/http_proxy_sender.go b/senders/http_proxy_sender.go index 8db9e7c..ae01fa3 100644 --- a/senders/http_proxy_sender.go +++ b/senders/http_proxy_sender.go @@ -2,13 +2,14 @@ package senders import ( "bytes" + "context" "crypto/tls" "encoding/json" "io/ioutil" "net/http" - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" ) @@ -26,12 +27,7 @@ type HTTPProxyResponse struct { // in order to pass event meta data to destinations, // HTTPProxyEventSender only parses payload for general usage of // third party API calls. -type HTTPProxyEventSender struct { - interfaces.EventSenderInterface -} - -// SendEvent - #HTTPProxyEventSender SendEvent -func (c *HTTPProxyEventSender) SendEvent(ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) error { +func (c *HTTPProxyEventSender) SendEvent(ctx context.Context, ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) (err error) { e := ev.(models.IncomingEvent) d := dv.(models.Destination) diff --git a/senders/http_sender.go b/senders/http_sender.go index a3fdc32..bdfb23a 100644 --- a/senders/http_sender.go +++ b/senders/http_sender.go @@ -2,12 +2,13 @@ package senders import ( "bytes" + "context" "crypto/tls" "io/ioutil" "net/http" - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" ) @@ -25,8 +26,7 @@ type HTTPEventSender struct { interfaces.EventSenderInterface } -// SendEvent - #HttpEventSender SendEvent -func (c *HTTPEventSender) SendEvent(ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) error { +func (c *HTTPEventSender) SendEvent(ctx context.Context, ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) (err error) { e := ev.(models.IncomingEvent) d := dv.(models.Destination) diff --git a/senders/sqs_sender.go b/senders/sqs_sender.go index 7de2618..51efcf6 100644 --- a/senders/sqs_sender.go +++ b/senders/sqs_sender.go @@ -1,8 +1,10 @@ package senders import ( - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + "context" + + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" aws "github.com/aws/aws-sdk-go/aws" @@ -30,7 +32,7 @@ func NewSqsSender(defaultAwsConfig aws.Config) *SqsSender { } // SendEvent - Send incoming event into SQS queue -func (s *SqsSender) SendEvent(ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) error { +func (s *SqsSender) SendEvent(ctx context.Context, ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) (err error) { e := ev.(models.IncomingEvent) d := dv.(models.Destination) diff --git a/test/core/captin_test.go b/test/core/captin_test.go index 4d6686f..5764d53 100644 --- a/test/core/captin_test.go +++ b/test/core/captin_test.go @@ -1,13 +1,15 @@ package models_test import ( - "github.com/stretchr/testify/assert" + "context" "testing" - . "github.com/shoplineapp/captin/core" - captin_errors "github.com/shoplineapp/captin/errors" - interfaces "github.com/shoplineapp/captin/interfaces" - models "github.com/shoplineapp/captin/models" + "github.com/stretchr/testify/assert" + + . "github.com/shoplineapp/captin/v2/core" + captin_errors "github.com/shoplineapp/captin/v2/errors" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + models "github.com/shoplineapp/captin/v2/models" "github.com/stretchr/testify/mock" ) @@ -38,7 +40,7 @@ func TestExecute(t *testing.T) { var errors []interfaces.ErrorInterface captin := Captin{} - _, errors = captin.Execute(models.IncomingEvent{}) + _, errors = captin.Execute(context.Background(), models.IncomingEvent{}) if assert.Error(t, errors[0], "invalid incoming event") { assert.IsType(t, errors[0], &captin_errors.ExecutionError{}) diff --git a/test/desination/filters/desired_hook_test.go b/test/desination/filters/desired_hook_test.go index 45a500a..0fc43de 100644 --- a/test/desination/filters/desired_hook_test.go +++ b/test/desination/filters/desired_hook_test.go @@ -1,12 +1,14 @@ package destination_filters_test import ( - "github.com/stretchr/testify/assert" + "context" "testing" - . "github.com/shoplineapp/captin/destinations/filters" - helpers "github.com/shoplineapp/captin/internal/helpers" - models "github.com/shoplineapp/captin/models" + "github.com/stretchr/testify/assert" + + . "github.com/shoplineapp/captin/v2/destinations/filters" + helpers "github.com/shoplineapp/captin/v2/internal/helpers" + models "github.com/shoplineapp/captin/v2/models" ) func TestDesiredHookFilterRunValidate(t *testing.T) { @@ -15,8 +17,8 @@ func TestDesiredHookFilterRunValidate(t *testing.T) { "desired_hooks": []string{"desired"}, }, } - assert.Equal(t, true, helpers.Tuples(DesiredHookFilter{}.Run(event, models.Destination{Config: models.Configuration{Name: "desired"}}))[0]) - assert.Equal(t, false, helpers.Tuples(DesiredHookFilter{}.Run(event, models.Destination{Config: models.Configuration{Name: "not_desired"}}))[0]) + assert.Equal(t, true, helpers.Tuples(DesiredHookFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Name: "desired"}}))[0]) + assert.Equal(t, false, helpers.Tuples(DesiredHookFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Name: "not_desired"}}))[0]) // When event is unmarshal from JSON, hooks will be type []interface{} event = models.IncomingEvent{ @@ -24,8 +26,8 @@ func TestDesiredHookFilterRunValidate(t *testing.T) { "desired_hooks": []interface{}{"desired"}, }, } - assert.Equal(t, true, helpers.Tuples(DesiredHookFilter{}.Run(event, models.Destination{Config: models.Configuration{Name: "desired"}}))[0]) - assert.Equal(t, false, helpers.Tuples(DesiredHookFilter{}.Run(event, models.Destination{Config: models.Configuration{Name: "not_desired"}}))[0]) + assert.Equal(t, true, helpers.Tuples(DesiredHookFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Name: "desired"}}))[0]) + assert.Equal(t, false, helpers.Tuples(DesiredHookFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Name: "not_desired"}}))[0]) // When event control contains multiple hooks event = models.IncomingEvent{ @@ -36,9 +38,9 @@ func TestDesiredHookFilterRunValidate(t *testing.T) { }, }, } - assert.Equal(t, true, helpers.Tuples(DesiredHookFilter{}.Run(event, models.Destination{Config: models.Configuration{Name: "hook-1"}}))[0]) - assert.Equal(t, true, helpers.Tuples(DesiredHookFilter{}.Run(event, models.Destination{Config: models.Configuration{Name: "hook-2"}}))[0]) - assert.Equal(t, false, helpers.Tuples(DesiredHookFilter{}.Run(event, models.Destination{Config: models.Configuration{Name: "not_desired"}}))[0]) + assert.Equal(t, true, helpers.Tuples(DesiredHookFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Name: "hook-1"}}))[0]) + assert.Equal(t, true, helpers.Tuples(DesiredHookFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Name: "hook-2"}}))[0]) + assert.Equal(t, false, helpers.Tuples(DesiredHookFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Name: "not_desired"}}))[0]) } func TestDesiredHookFilterApplicable(t *testing.T) { @@ -47,8 +49,8 @@ func TestDesiredHookFilterApplicable(t *testing.T) { event := models.IncomingEvent{Control: map[string]interface{}{ "desired_hooks": []interface{}{"desired"}, }} - assert.Equal(t, true, DesiredHookFilter{}.Applicable(event, destination)) + assert.Equal(t, true, DesiredHookFilter{}.Applicable(context.Background(), event, destination)) event = models.IncomingEvent{} - assert.Equal(t, false, DesiredHookFilter{}.Applicable(event, destination)) + assert.Equal(t, false, DesiredHookFilter{}.Applicable(context.Background(), event, destination)) } diff --git a/test/desination/filters/environment_test.go b/test/desination/filters/environment_test.go index b23daaa..2d4a2a3 100644 --- a/test/desination/filters/environment_test.go +++ b/test/desination/filters/environment_test.go @@ -1,26 +1,28 @@ package destination_filters_test import ( - "github.com/stretchr/testify/assert" - "testing" - "os" + "context" + "os" + "testing" - . "github.com/shoplineapp/captin/destinations/filters" - helpers "github.com/shoplineapp/captin/internal/helpers" - models "github.com/shoplineapp/captin/models" + "github.com/stretchr/testify/assert" + + . "github.com/shoplineapp/captin/v2/destinations/filters" + helpers "github.com/shoplineapp/captin/v2/internal/helpers" + models "github.com/shoplineapp/captin/v2/models" ) func TestEnvironmentFilterRunValidate(t *testing.T) { - defer os.Unsetenv("HOOK_SERVICE_1_ENABLED") - event := models.IncomingEvent{} - assert.Equal(t, true, helpers.Tuples(EnvironmentFilter{}.Run(event, models.Destination{Config: models.Configuration{Name: "service_1"}}))[0]) - os.Setenv("HOOK_SERVICE_1_ENABLED", "false") - assert.Equal(t, false, helpers.Tuples(EnvironmentFilter{}.Run(event, models.Destination{Config: models.Configuration{Name: "service_1"}}))[0]) - os.Setenv("HOOK_SERVICE_1_ENABLED", "true") - assert.Equal(t, true, helpers.Tuples(EnvironmentFilter{}.Run(event, models.Destination{Config: models.Configuration{Name: "service_1"}}))[0]) + defer os.Unsetenv("HOOK_SERVICE_1_ENABLED") + event := models.IncomingEvent{} + assert.Equal(t, true, helpers.Tuples(EnvironmentFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Name: "service_1"}}))[0]) + os.Setenv("HOOK_SERVICE_1_ENABLED", "false") + assert.Equal(t, false, helpers.Tuples(EnvironmentFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Name: "service_1"}}))[0]) + os.Setenv("HOOK_SERVICE_1_ENABLED", "true") + assert.Equal(t, true, helpers.Tuples(EnvironmentFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Name: "service_1"}}))[0]) } func TestEnvironmentFilterApplicable(t *testing.T) { - event := models.IncomingEvent{} - assert.Equal(t, true, EnvironmentFilter{}.Applicable(event, models.Destination{Config: models.Configuration{}})) + event := models.IncomingEvent{} + assert.Equal(t, true, EnvironmentFilter{}.Applicable(context.Background(), event, models.Destination{Config: models.Configuration{}})) } diff --git a/test/desination/filters/source_test.go b/test/desination/filters/source_test.go index c5e8c2d..e8a33ec 100644 --- a/test/desination/filters/source_test.go +++ b/test/desination/filters/source_test.go @@ -1,22 +1,24 @@ package destination_filters_test import ( - "github.com/stretchr/testify/assert" + "context" "testing" - . "github.com/shoplineapp/captin/destinations/filters" - helpers "github.com/shoplineapp/captin/internal/helpers" - models "github.com/shoplineapp/captin/models" + "github.com/stretchr/testify/assert" + + . "github.com/shoplineapp/captin/v2/destinations/filters" + helpers "github.com/shoplineapp/captin/v2/internal/helpers" + models "github.com/shoplineapp/captin/v2/models" ) func TestSourceFilterRunValidate(t *testing.T) { event := models.IncomingEvent{Source: "service_1"} - assert.Equal(t, true, helpers.Tuples(SourceFilter{}.Run(event, models.Destination{Config: models.Configuration{Source: "service_2"}}))[0]) - assert.Equal(t, false, helpers.Tuples(SourceFilter{}.Run(event, models.Destination{Config: models.Configuration{Source: "service_1"}}))[0]) + assert.Equal(t, true, helpers.Tuples(SourceFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Source: "service_2"}}))[0]) + assert.Equal(t, false, helpers.Tuples(SourceFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Source: "service_1"}}))[0]) } func TestSourceFilterApplicable(t *testing.T) { event := models.IncomingEvent{} - assert.Equal(t, true, SourceFilter{}.Applicable(event, models.Destination{Config: models.Configuration{AllowLoopback: false}})) - assert.Equal(t, false, SourceFilter{}.Applicable(event, models.Destination{Config: models.Configuration{AllowLoopback: true}})) + assert.Equal(t, true, SourceFilter{}.Applicable(context.Background(), event, models.Destination{Config: models.Configuration{AllowLoopback: false}})) + assert.Equal(t, false, SourceFilter{}.Applicable(context.Background(), event, models.Destination{Config: models.Configuration{AllowLoopback: true}})) } diff --git a/test/desination/filters/validate_test.go b/test/desination/filters/validate_test.go index 83f79dd..1227edf 100644 --- a/test/desination/filters/validate_test.go +++ b/test/desination/filters/validate_test.go @@ -1,12 +1,14 @@ package destination_filters_test import ( - "github.com/stretchr/testify/assert" + "context" "testing" - . "github.com/shoplineapp/captin/destinations/filters" - helpers "github.com/shoplineapp/captin/internal/helpers" - models "github.com/shoplineapp/captin/models" + "github.com/stretchr/testify/assert" + + . "github.com/shoplineapp/captin/v2/destinations/filters" + helpers "github.com/shoplineapp/captin/v2/internal/helpers" + models "github.com/shoplineapp/captin/v2/models" ) func TestValidateFilterRunValidate(t *testing.T) { @@ -15,22 +17,22 @@ func TestValidateFilterRunValidate(t *testing.T) { "type": "line", } event := models.IncomingEvent{Payload: payload} - assert.Equal(t, true, helpers.Tuples(ValidateFilter{}.Run(event, models.Destination{Config: models.Configuration{Validate: "document.type == 'line'"}}))[0]) - assert.Equal(t, false, helpers.Tuples(ValidateFilter{}.Run(event, models.Destination{Config: models.Configuration{Validate: "document.type != 'line'"}}))[0]) + assert.Equal(t, true, helpers.Tuples(ValidateFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Validate: "document.type == 'line'"}}))[0]) + assert.Equal(t, false, helpers.Tuples(ValidateFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Validate: "document.type != 'line'"}}))[0]) // Error validate check - assert.Equal(t, false, helpers.Tuples(ValidateFilter{}.Run(event, models.Destination{Config: models.Configuration{Validate: "document.noneExist"}}))[0]) - assert.Equal(t, false, helpers.Tuples(ValidateFilter{}.Run(event, models.Destination{Config: models.Configuration{Validate: "invalidCall()"}}))[0]) - assert.Equal(t, false, helpers.Tuples(ValidateFilter{}.Run(event, models.Destination{Config: models.Configuration{Validate: "asd"}}))[0]) + assert.Equal(t, false, helpers.Tuples(ValidateFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Validate: "document.noneExist"}}))[0]) + assert.Equal(t, false, helpers.Tuples(ValidateFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Validate: "invalidCall()"}}))[0]) + assert.Equal(t, false, helpers.Tuples(ValidateFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Validate: "asd"}}))[0]) // Calling with nil payload should not affect test result event = models.IncomingEvent{Payload: nil} - assert.Equal(t, true, helpers.Tuples(ValidateFilter{}.Run(event, models.Destination{Config: models.Configuration{Validate: "true"}}))[0]) - assert.Equal(t, false, helpers.Tuples(ValidateFilter{}.Run(event, models.Destination{Config: models.Configuration{Validate: "false"}}))[0]) + assert.Equal(t, true, helpers.Tuples(ValidateFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Validate: "true"}}))[0]) + assert.Equal(t, false, helpers.Tuples(ValidateFilter{}.Run(context.Background(), event, models.Destination{Config: models.Configuration{Validate: "false"}}))[0]) } func TestValidateFilterApplicable(t *testing.T) { event := models.IncomingEvent{} - assert.Equal(t, true, ValidateFilter{}.Applicable(event, models.Destination{Config: models.Configuration{Validate: "true"}})) - assert.Equal(t, false, ValidateFilter{}.Applicable(event, models.Destination{Config: models.Configuration{}})) + assert.Equal(t, true, ValidateFilter{}.Applicable(context.Background(), event, models.Destination{Config: models.Configuration{Validate: "true"}})) + assert.Equal(t, false, ValidateFilter{}.Applicable(context.Background(), event, models.Destination{Config: models.Configuration{}})) } diff --git a/test/dispatcher/delayers/goroutine_test.go b/test/dispatcher/delayers/goroutine_test.go index 5e4c2cc..d39799b 100644 --- a/test/dispatcher/delayers/goroutine_test.go +++ b/test/dispatcher/delayers/goroutine_test.go @@ -1,11 +1,13 @@ package dispatcher_delayers_test import ( - "github.com/stretchr/testify/assert" + "context" "testing" - . "github.com/shoplineapp/captin/dispatcher/delayers" - models "github.com/shoplineapp/captin/models" + "github.com/stretchr/testify/assert" + + . "github.com/shoplineapp/captin/v2/dispatcher/delayers" + models "github.com/shoplineapp/captin/v2/models" ) func TestGoroutineDelayer_Execute(t *testing.T) { @@ -21,6 +23,6 @@ func TestGoroutineDelayer_Execute(t *testing.T) { }, } delayer := GoroutineDelayer{} - delayer.Execute(evt, dest, callback) + delayer.Execute(context.Background(), evt, dest, callback) assert.Equal(t, true, isCallbackCalled) } diff --git a/test/dispatcher/tracking_test.go b/test/dispatcher/tracking_test.go index fa7a9d6..237908b 100644 --- a/test/dispatcher/tracking_test.go +++ b/test/dispatcher/tracking_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/shoplineapp/captin/dispatcher" + "github.com/shoplineapp/captin/v2/dispatcher" "github.com/stretchr/testify/assert" ) diff --git a/test/internal/document_stores/null_store_test.go b/test/internal/document_stores/null_store_test.go index d3e140f..129f77e 100644 --- a/test/internal/document_stores/null_store_test.go +++ b/test/internal/document_stores/null_store_test.go @@ -1,16 +1,16 @@ package document_stores_test import ( - "testing" - - document_stores "github.com/shoplineapp/captin/internal/document_stores" - models "github.com/shoplineapp/captin/models" - "github.com/stretchr/testify/assert" + "context" + "testing" + document_stores "github.com/shoplineapp/captin/v2/internal/document_stores" + models "github.com/shoplineapp/captin/v2/models" + "github.com/stretchr/testify/assert" ) func TestGetDocument(t *testing.T) { - ns := document_stores.NewNullDocumentStore() - e := models.IncomingEvent{} - assert.Equal(t, ns.GetDocument(e), map[string]interface{}{}) + ns := document_stores.NewNullDocumentStore() + e := models.IncomingEvent{} + assert.Equal(t, ns.GetDocument(context.Background(), e), map[string]interface{}{}) } diff --git a/test/internal/helpers/select_field_test.go b/test/internal/helpers/select_field_test.go index afd8d50..d359db1 100644 --- a/test/internal/helpers/select_field_test.go +++ b/test/internal/helpers/select_field_test.go @@ -3,7 +3,7 @@ package helpers_test import ( "testing" - helpers "github.com/shoplineapp/captin/internal/helpers" + helpers "github.com/shoplineapp/captin/v2/internal/helpers" "github.com/stretchr/testify/assert" ) diff --git a/test/internal/outgoing/custom_test.go b/test/internal/outgoing/custom_test.go index 8733b55..8498425 100644 --- a/test/internal/outgoing/custom_test.go +++ b/test/internal/outgoing/custom_test.go @@ -1,11 +1,12 @@ package outgoing_test import ( + "context" "testing" - destination_filters "github.com/shoplineapp/captin/destinations/filters" - . "github.com/shoplineapp/captin/internal/outgoing" - models "github.com/shoplineapp/captin/models" + destination_filters "github.com/shoplineapp/captin/v2/destinations/filters" + . "github.com/shoplineapp/captin/v2/internal/outgoing" + models "github.com/shoplineapp/captin/v2/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -14,13 +15,13 @@ type FilterMock struct { mock.Mock } -func (f *FilterMock) Run(e models.IncomingEvent, d models.Destination) (bool, error) { - args := f.Called(e, d) +func (f *FilterMock) Run(ctx context.Context, e models.IncomingEvent, d models.Destination) (bool, error) { + args := f.Called(ctx, e, d) return args.Bool(0), args.Error(1) } -func (f *FilterMock) Applicable(e models.IncomingEvent, d models.Destination) bool { - args := f.Called(e, d) +func (f *FilterMock) Applicable(ctx context.Context, e models.IncomingEvent, d models.Destination) bool { + args := f.Called(ctx, e, d) return args.Bool(0) } @@ -28,8 +29,8 @@ type MiddlewareMock struct { mock.Mock } -func (m *MiddlewareMock) Apply(e *models.IncomingEvent, d []models.Destination) []models.Destination { - m.Called(e, d) +func (m *MiddlewareMock) Apply(ctx context.Context, e *models.IncomingEvent, d []models.Destination) []models.Destination { + m.Called(ctx, e, d) return d } @@ -38,15 +39,15 @@ func TestCustom_Sift(t *testing.T) { event := models.IncomingEvent{} filter := new(FilterMock) - filter.On("Applicable", mock.Anything, mock.Anything).Return(true) - filter.On("Run", mock.Anything, mock.Anything).Return(true, nil) + filter.On("Applicable", mock.Anything, mock.Anything, mock.Anything).Return(true) + filter.On("Run", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) middleware := new(MiddlewareMock) - middleware.On("Apply", mock.Anything, mock.Anything) + middleware.On("Apply", mock.Anything, mock.Anything, mock.Anything) destinations := []models.Destination{ models.Destination{Config: models.Configuration{Source: "service_1"}}, models.Destination{Config: models.Configuration{Source: "service_2"}}, } - sifted := Custom{}.Sift(&event, destinations, []destination_filters.DestinationFilterInterface{filter}, []destination_filters.DestinationMiddlewareInterface{middleware}) + sifted := Custom{}.Sift(context.Background(), &event, destinations, []destination_filters.DestinationFilterInterface{filter}, []destination_filters.DestinationMiddlewareInterface{middleware}) filter.AssertNumberOfCalls(t, "Applicable", 2) filter.AssertNumberOfCalls(t, "Run", 2) middleware.AssertNumberOfCalls(t, "Apply", 1) @@ -54,15 +55,15 @@ func TestCustom_Sift(t *testing.T) { // Test if filter run is skipped when filter is not applicable filter = new(FilterMock) - filter.On("Applicable", mock.Anything, mock.Anything).Return(false) - filter.On("Run", mock.Anything, mock.Anything).Return(true, nil) + filter.On("Applicable", mock.Anything, mock.Anything, mock.Anything).Return(false) + filter.On("Run", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) middleware = new(MiddlewareMock) - middleware.On("Apply", mock.Anything, mock.Anything) + middleware.On("Apply", mock.Anything, mock.Anything, mock.Anything) destinations = []models.Destination{ models.Destination{Config: models.Configuration{Source: "service_1"}}, models.Destination{Config: models.Configuration{Source: "service_2"}}, } - sifted = Custom{}.Sift(&event, destinations, []destination_filters.DestinationFilterInterface{filter}, []destination_filters.DestinationMiddlewareInterface{middleware}) + sifted = Custom{}.Sift(context.Background(), &event, destinations, []destination_filters.DestinationFilterInterface{filter}, []destination_filters.DestinationMiddlewareInterface{middleware}) filter.AssertNumberOfCalls(t, "Applicable", 2) filter.AssertNumberOfCalls(t, "Run", 0) middleware.AssertNumberOfCalls(t, "Apply", 1) @@ -70,14 +71,14 @@ func TestCustom_Sift(t *testing.T) { // Test if destinations is filtered when filter run returns false filter = new(FilterMock) - filter.On("Applicable", mock.Anything, mock.Anything).Return(true) - filter.On("Run", mock.Anything, mock.Anything).Return(false, nil) + filter.On("Applicable", mock.Anything, mock.Anything, mock.Anything).Return(true) + filter.On("Run", mock.Anything, mock.Anything, mock.Anything).Return(false, nil) middleware = new(MiddlewareMock) - middleware.On("Apply", mock.Anything, mock.Anything) + middleware.On("Apply", mock.Anything, mock.Anything, mock.Anything) destinations = []models.Destination{ models.Destination{Config: models.Configuration{Source: "service_1"}}, } - sifted = Custom{}.Sift(&event, destinations, []destination_filters.DestinationFilterInterface{filter}, []destination_filters.DestinationMiddlewareInterface{middleware}) + sifted = Custom{}.Sift(context.Background(), &event, destinations, []destination_filters.DestinationFilterInterface{filter}, []destination_filters.DestinationMiddlewareInterface{middleware}) filter.AssertNumberOfCalls(t, "Applicable", 1) filter.AssertNumberOfCalls(t, "Run", 1) middleware.AssertNumberOfCalls(t, "Apply", 1) diff --git a/test/internal/outgoing/dispatcher_test.go b/test/internal/outgoing/dispatcher_test.go index 3be1235..89d523a 100644 --- a/test/internal/outgoing/dispatcher_test.go +++ b/test/internal/outgoing/dispatcher_test.go @@ -1,6 +1,7 @@ package outgoing_test import ( + "context" "encoding/json" "errors" "fmt" @@ -8,15 +9,14 @@ import ( "reflect" "testing" "time" - "unsafe" - - delayers "github.com/shoplineapp/captin/dispatcher/delayers" - captin_errors "github.com/shoplineapp/captin/errors" - interfaces "github.com/shoplineapp/captin/interfaces" - outgoing "github.com/shoplineapp/captin/internal/outgoing" - stores "github.com/shoplineapp/captin/internal/stores" - models "github.com/shoplineapp/captin/models" - mocks "github.com/shoplineapp/captin/test/mocks" + + delayers "github.com/shoplineapp/captin/v2/dispatcher/delayers" + captin_errors "github.com/shoplineapp/captin/v2/errors" + interfaces "github.com/shoplineapp/captin/v2/interfaces" + outgoing "github.com/shoplineapp/captin/v2/internal/outgoing" + stores "github.com/shoplineapp/captin/v2/internal/stores" + models "github.com/shoplineapp/captin/v2/models" + mocks "github.com/shoplineapp/captin/v2/test/mocks" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -59,12 +59,12 @@ func hasDocument(e models.IncomingEvent) bool { func TestDispatchEvents_Error(t *testing.T) { store, documentStores, sender, dispatcher, throttler := setup("fixtures/config.json") - sender.On("SendEvent", mock.Anything, mock.Anything).Return(errors.New("Mock Error")) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("Mock Error")) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -72,7 +72,7 @@ func TestDispatchEvents_Error(t *testing.T) { TargetId: "product_id", }, store, throttler, documentStores) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 2}, @@ -87,12 +87,12 @@ func TestDispatchEvents_Error(t *testing.T) { func TestDispatchEvents_SendEvent_WithNotDispatcherError(t *testing.T) { store, documentStores, sender, dispatcher, throttler := setup("fixtures/config.json") - sender.On("SendEvent", mock.Anything, mock.Anything).Panic("Non-dispatcher panic") - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Panic("Non-dispatcher panic") + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -100,7 +100,7 @@ func TestDispatchEvents_SendEvent_WithNotDispatcherError(t *testing.T) { TargetId: "product_id", }, store, throttler, documentStores) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 2}, @@ -115,12 +115,12 @@ func TestDispatchEvents_SendEvent_WithNotDispatcherError(t *testing.T) { func TestDispatchEvents(t *testing.T) { store, documentStores, sender, dispatcher, throttler := setup("fixtures/config.json") - sender.On("SendEvent", mock.Anything, mock.Anything).Return(nil) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(nil) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -128,7 +128,7 @@ func TestDispatchEvents(t *testing.T) { TargetId: "product_id", }, store, throttler, documentStores) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 2}, @@ -142,14 +142,14 @@ func TestDispatchEvents(t *testing.T) { func TestDispatchEvents_Throttled_DelaySend(t *testing.T) { store, documentStores, sender, dispatcher, throttler := setup("fixtures/config.single.json") - sender.On("SendEvent", mock.Anything, mock.Anything).Return(nil) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil).Once() - store.On("Get", mock.Anything).Return("", true, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - store.On("Remove", mock.Anything).Return(true, nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(nil) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil).Once() + store.On("Get", mock.Anything, mock.Anything).Return("", true, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + store.On("Remove", mock.Anything, mock.Anything).Return(true, nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -161,8 +161,8 @@ func TestDispatchEvents_Throttled_DelaySend(t *testing.T) { throttleID := "product.update.service_one.product_id-data" throttlePeriod := time.Millisecond * 500 * 2 - store.AssertCalled(t, "Get", throttleID) - store.AssertCalled(t, "Set", throttleID, `{"control":null,"event_key":"product.update","payload":{"field1":1},"source":"core","target_id":"product_id","target_type":"Product","trace_id":""}`, throttlePeriod) + store.AssertCalled(t, "Get", mock.Anything, throttleID) + store.AssertCalled(t, "Set", mock.Anything, throttleID, `{"control":null,"distributed_tracing_info":{},"event_key":"product.update","payload":{"field1":1},"source":"core","target_id":"product_id","target_type":"Product","trace_id":""}`, throttlePeriod) time.Sleep(600 * time.Millisecond) @@ -172,14 +172,14 @@ func TestDispatchEvents_Throttled_DelaySend(t *testing.T) { func TestDispatchEvents_Delayer_Send(t *testing.T) { store, documentStores, sender, dispatcher, throttler := setup("fixtures/config.delay.json") dispatcher.SetDelayer(&delayers.GoroutineDelayer{}) - sender.On("SendEvent", mock.Anything, mock.Anything).Return(nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(false, 1*time.Millisecond, nil) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil).Twice() - store.On("Get", mock.Anything).Return("", true, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - store.On("Remove", mock.Anything).Return(true, nil) - - dispatcher.Dispatch(models.IncomingEvent{ + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(false, 1*time.Millisecond, nil) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil).Twice() + store.On("Get", mock.Anything, mock.Anything).Return("", true, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + store.On("Remove", mock.Anything, mock.Anything).Return(true, nil) + + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -198,13 +198,13 @@ func TestDispatchEvents_Throttled_SkipUpdatingValue(t *testing.T) { throttleID := "product.update.service_one.product_id-data" throttlePeriod := time.Millisecond * 700 * 2 - sender.On("SendEvent", mock.Anything, mock.Anything).Return(nil) - store.On("Get", throttleID).Return(`{"event_key":"product.update","source":"core","payload":{"field1":1},"control":{"ts":"99999999999999"},"target_type":"Product","target_id":"product_id"}`, true, throttlePeriod, nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - store.On("Update", mock.Anything, mock.Anything).Return(true, nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(false, time.Millisecond*700, nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(nil) + store.On("Get", mock.Anything, throttleID).Return(`{"event_key":"product.update","source":"core","payload":{"field1":1},"control":{"ts":"99999999999999"},"target_type":"Product","target_id":"product_id"}`, true, throttlePeriod, nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + store.On("Update", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(false, time.Millisecond*700, nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 2}, @@ -216,7 +216,7 @@ func TestDispatchEvents_Throttled_SkipUpdatingValue(t *testing.T) { // Expecting update of store is skipped due to older timestamp from new incoming event sender.AssertNumberOfCalls(t, "SendEvent", 0) store.AssertNumberOfCalls(t, "Update", 0) - store.AssertCalled(t, "Get", throttleID) + store.AssertCalled(t, "Get", mock.Anything, throttleID) } func TestDispatchEvents_Throttled_UpdatePayload(t *testing.T) { @@ -224,14 +224,14 @@ func TestDispatchEvents_Throttled_UpdatePayload(t *testing.T) { throttleID := "product.update.service_one.product_id-data" throttlePeriod := time.Millisecond * 700 * 2 - sender.On("SendEvent", mock.Anything, mock.Anything).Return(nil) - store.On("Get", throttleID).Return(`{"event_key":"product.update","source":"core","payload":{"field1":1},"control":null,"target_type":"Product","target_id":"product_id"}`, true, throttlePeriod, nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - store.On("Update", mock.Anything, mock.Anything).Return(true, nil) - store.On("Remove", mock.Anything).Return(true, nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(false, time.Millisecond*700, nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(nil) + store.On("Get", mock.Anything, throttleID).Return(`{"event_key":"product.update","source":"core","payload":{"field1":1},"control":null,"target_type":"Product","target_id":"product_id"}`, true, throttlePeriod, nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + store.On("Update", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + store.On("Remove", mock.Anything, mock.Anything).Return(true, nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(false, time.Millisecond*700, nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 2}, @@ -240,21 +240,21 @@ func TestDispatchEvents_Throttled_UpdatePayload(t *testing.T) { }, store, throttler, documentStores) sender.AssertNumberOfCalls(t, "SendEvent", 0) - store.AssertCalled(t, "Get", throttleID) - store.AssertCalled(t, "Update", throttleID, `{"control":null,"event_key":"product.update","payload":{"field1":2},"source":"core","target_id":"product_id","target_type":"Product","trace_id":""}`) + store.AssertCalled(t, "Get", mock.Anything, throttleID) + store.AssertCalled(t, "Update", mock.Anything, throttleID, `{"control":null,"distributed_tracing_info":{},"event_key":"product.update","payload":{"field1":2},"source":"core","target_id":"product_id","target_type":"Product","trace_id":""}`) } func TestDispatchEvents_Throttled_KeepThrottledPayloads(t *testing.T) { _, documentStores, sender, dispatcher, throttler := setup("fixtures/config.keep_throttled_payloads.json") store := stores.NewMemoryStore() - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil) - sender.On("SendEvent", mock.Anything, mock.Anything).Return(nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(nil) throttleID := "product.update.service_one.product_id-data" throttlePayloadsID := "product.update.service_one.product_id-throttled_payloads" - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -262,21 +262,21 @@ func TestDispatchEvents_Throttled_KeepThrottledPayloads(t *testing.T) { TargetId: "product_id", }, store, throttler, documentStores) - throttledPayloads, _, _, _ := store.GetQueue(throttlePayloadsID) + throttledPayloads, _, _, _ := store.GetQueue(context.Background(), throttlePayloadsID) assert.Equal(t, len(throttledPayloads), 1) sender.AssertNumberOfCalls(t, "SendEvent", 0) time.Sleep(600 * time.Millisecond) - sender.AssertCalled(t, "SendEvent", mock.MatchedBy(func(e models.IncomingEvent) bool { + sender.AssertCalled(t, "SendEvent", mock.Anything, mock.MatchedBy(func(e models.IncomingEvent) bool { return fmt.Sprint(e.ThrottledPayloads) == fmt.Sprint([]map[string]interface{}{ {"field1": 1}, }) }), mock.Anything) - _, storedEventExists, _, _ := store.Get(throttleID) - throttledPayloads, _, _, _ = store.GetQueue(throttlePayloadsID) + _, storedEventExists, _, _ := store.Get(context.Background(), throttleID) + throttledPayloads, _, _, _ = store.GetQueue(context.Background(), throttlePayloadsID) assert.Equal(t, false, storedEventExists) assert.Equal(t, 0, len(throttledPayloads)) } @@ -285,13 +285,15 @@ func TestDispatchEvents_Throttled_KeepThrottledPayloads_Multiple(t *testing.T) { _, documentStores, sender, dispatcher, throttler := setup("fixtures/config.keep_throttled_payloads.json") store := stores.NewMemoryStore() - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil) - sender.On("SendEvent", mock.Anything, mock.Anything).Return(nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(nil) throttleID := "product.update.service_one.product_id-data" throttlePayloadsID := "product.update.service_one.product_id-throttled_payloads" - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher2 := *dispatcher + + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -299,7 +301,7 @@ func TestDispatchEvents_Throttled_KeepThrottledPayloads_Multiple(t *testing.T) { TargetId: "product_id", }, store, throttler, documentStores) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher2.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 2}, @@ -307,7 +309,7 @@ func TestDispatchEvents_Throttled_KeepThrottledPayloads_Multiple(t *testing.T) { TargetId: "product_id", }, store, throttler, documentStores) - throttledPayloads, _, _, _ := store.GetQueue(throttlePayloadsID) + throttledPayloads, _, _, _ := store.GetQueue(context.Background(), throttlePayloadsID) assert.Equal(t, 2, len(throttledPayloads)) sender.AssertNumberOfCalls(t, "SendEvent", 0) @@ -315,7 +317,7 @@ func TestDispatchEvents_Throttled_KeepThrottledPayloads_Multiple(t *testing.T) { time.Sleep(600 * time.Millisecond) // it should SendEvent with 2 throttled payloads - sender.AssertCalled(t, "SendEvent", mock.MatchedBy(func(e models.IncomingEvent) bool { + sender.AssertCalled(t, "SendEvent", mock.Anything, mock.MatchedBy(func(e models.IncomingEvent) bool { return fmt.Sprint(e.ThrottledPayloads) == fmt.Sprint([]map[string]interface{}{ {"field1": 1}, {"field1": 2}, @@ -323,13 +325,13 @@ func TestDispatchEvents_Throttled_KeepThrottledPayloads_Multiple(t *testing.T) { }), mock.Anything) // it should SendEvent with last payload - sender.AssertCalled(t, "SendEvent", mock.MatchedBy(func(e models.IncomingEvent) bool { + sender.AssertCalled(t, "SendEvent", mock.Anything, mock.MatchedBy(func(e models.IncomingEvent) bool { return fmt.Sprint(e.Payload) == fmt.Sprint(map[string]interface{}{"field1": 2}) }), mock.Anything) // it should clean up after sendEvent - _, storedEventExists, _, _ := store.Get(throttleID) - throttledPayloads, _, _, _ = store.GetQueue(throttlePayloadsID) + _, storedEventExists, _, _ := store.Get(context.Background(), throttleID) + throttledPayloads, _, _, _ = store.GetQueue(context.Background(), throttlePayloadsID) assert.Equal(t, false, storedEventExists) assert.Equal(t, 0, len(throttledPayloads)) } @@ -340,37 +342,37 @@ func TestDispatchEvents_Throttled_KeepThrottledDocuments(t *testing.T) { mockDocumentStore := new(mocks.DocumentStoreMock) documentStores["default"] = mockDocumentStore - mockDocumentStore.On("GetDocument", mock.Anything).Return(map[string]interface{}{"foo": "bar"}) + mockDocumentStore.On("GetDocument", mock.Anything, mock.Anything).Return(map[string]interface{}{"foo": "bar"}) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil) - sender.On("SendEvent", mock.Anything, mock.Anything).Return(nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(nil) throttleID := "product.update.service_one.product_id-data" throttleDocumentsID := "product.update.service_one.product_id-throttled_documents" - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", TargetType: "Product", TargetId: "product_id", }, store, throttler, documentStores) - throttledDocuments, _, _, _ := store.GetQueue(throttleDocumentsID) + throttledDocuments, _, _, _ := store.GetQueue(context.Background(), throttleDocumentsID) assert.Equal(t, 1, len(throttledDocuments)) sender.AssertNumberOfCalls(t, "SendEvent", 0) time.Sleep(600 * time.Millisecond) - sender.AssertCalled(t, "SendEvent", mock.MatchedBy(func(e models.IncomingEvent) bool { + sender.AssertCalled(t, "SendEvent", mock.Anything, mock.MatchedBy(func(e models.IncomingEvent) bool { return fmt.Sprint(e.ThrottledDocuments) == fmt.Sprint([]map[string]interface{}{ {"foo": "bar"}, }) }), mock.Anything) // it should clean up after sendEvent - _, storedEventExists, _, _ := store.Get(throttleID) - throttledDocuments, _, _, _ = store.GetQueue(throttleDocumentsID) + _, storedEventExists, _, _ := store.Get(context.Background(), throttleID) + throttledDocuments, _, _, _ = store.GetQueue(context.Background(), throttleDocumentsID) assert.Equal(t, false, storedEventExists) assert.Equal(t, 0, len(throttledDocuments)) } @@ -381,42 +383,43 @@ func TestDispatchEvents_Throttled_KeepThrottledDocuments_Multiple(t *testing.T) mockDocumentStore := new(mocks.DocumentStoreMock) documentStores["default"] = mockDocumentStore - mockDocumentStore.On("GetDocument", mock.Anything).Return(map[string]interface{}{"foo": "bar1"}).Once() - mockDocumentStore.On("GetDocument", mock.Anything).Return(map[string]interface{}{"foo": "bar2"}) + mockDocumentStore.On("GetDocument", mock.Anything, mock.Anything).Return(map[string]interface{}{"foo": "bar1"}).Once() + mockDocumentStore.On("GetDocument", mock.Anything, mock.Anything).Return(map[string]interface{}{"foo": "bar2"}) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil) - sender.On("SendEvent", mock.Anything, mock.Anything).Return(nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(nil) throttleID := "product.update.service_one.product_id-data" throttleDocumentsID := "product.update.service_one.product_id-throttled_documents" - dispatcher.Dispatch(models.IncomingEvent{ + // In Captain we are not reusing dispatcher for dispatcher.Dispatch + // (it should not be re-used anyway as there are states like Errors and targetDocument) + // If we reuse the same dispatcher in test case, it fails with data race, which won't happen in production + // So we copy the dispatcher (with the same stores) + dispatcher2 := *dispatcher + + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", TargetType: "Product", TargetId: "product_id", }, store, throttler, documentStores) - // clear cache in dispatcher.targetDocument private field - // src: https://gist.github.com/CyberLight/1da35b4e0093bc12302f - ptrToTargetDocument := (*interface{})(unsafe.Pointer(reflect.Indirect(reflect.ValueOf(dispatcher)).FieldByName("targetDocument").UnsafeAddr())) - *ptrToTargetDocument = nil - - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher2.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", TargetType: "Product", TargetId: "product_id", }, store, throttler, documentStores) - throttledDocuments, _, _, _ := store.GetQueue(throttleDocumentsID) + throttledDocuments, _, _, _ := store.GetQueue(context.Background(), throttleDocumentsID) assert.Equal(t, 2, len(throttledDocuments)) sender.AssertNumberOfCalls(t, "SendEvent", 0) time.Sleep(600 * time.Millisecond) - sender.AssertCalled(t, "SendEvent", mock.MatchedBy(func(e models.IncomingEvent) bool { + sender.AssertCalled(t, "SendEvent", mock.Anything, mock.MatchedBy(func(e models.IncomingEvent) bool { return fmt.Sprint(e.ThrottledDocuments) == fmt.Sprint([]map[string]interface{}{ {"foo": "bar1"}, {"foo": "bar2"}, @@ -424,8 +427,8 @@ func TestDispatchEvents_Throttled_KeepThrottledDocuments_Multiple(t *testing.T) }), mock.Anything) // it should clean up after sendEvent - _, storedEventExists, _, _ := store.Get(throttleID) - throttledDocuments, _, _, _ = store.GetQueue(throttleDocumentsID) + _, storedEventExists, _, _ := store.Get(context.Background(), throttleID) + throttledDocuments, _, _, _ = store.GetQueue(context.Background(), throttleDocumentsID) assert.Equal(t, false, storedEventExists) assert.Equal(t, 0, len(throttledDocuments)) } @@ -435,15 +438,15 @@ func TestDispatchEvents_With_Document(t *testing.T) { mockDocumentStore := new(mocks.DocumentStoreMock) documentStores["default"] = mockDocumentStore - sender.On("SendEvent", mock.MatchedBy(hasDocument), mock.Anything).Return(nil) - sender.On("SendEvent", mock.MatchedBy(hasNoDocument), mock.Anything).Return(nil) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - mockDocumentStore.On("GetDocument", mock.Anything).Return(map[string]interface{}{"foo": "bar"}) + sender.On("SendEvent", mock.Anything, mock.MatchedBy(hasDocument), mock.Anything).Return(nil) + sender.On("SendEvent", mock.Anything, mock.MatchedBy(hasNoDocument), mock.Anything).Return(nil) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + mockDocumentStore.On("GetDocument", mock.Anything, mock.Anything).Return(map[string]interface{}{"foo": "bar"}) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -459,16 +462,17 @@ func TestDispatchEvents_With_Include_Document_Attrs(t *testing.T) { mockDocumentStore := new(mocks.DocumentStoreMock) documentStores["default"] = mockDocumentStore - sender.On("SendEvent", mock.MatchedBy(func(e models.IncomingEvent) bool { + sender.On("SendEvent", mock.Anything, mock.MatchedBy(func(e models.IncomingEvent) bool { return reflect.DeepEqual(e.TargetDocument, map[string]interface{}{"foo": map[string]interface{}{"bar": "yo"}}) }), mock.Anything).Return(nil) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + throttler.On("CanTriggerWithCtx", mock.Anything, mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - mockDocumentStore.On("GetDocument", mock.Anything).Return(map[string]interface{}{"foo": map[string]interface{}{"bar": "yo", "a": "b"}, "foo2": "bar2"}) + mockDocumentStore.On("GetDocument", mock.Anything, mock.Anything).Return(map[string]interface{}{"foo": map[string]interface{}{"bar": "yo", "a": "b"}, "foo2": "bar2"}) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -484,16 +488,16 @@ func TestDispatchEvents_With_Exclude_Document_Attrs(t *testing.T) { mockDocumentStore := new(mocks.DocumentStoreMock) documentStores["default"] = mockDocumentStore - sender.On("SendEvent", mock.MatchedBy(func(e models.IncomingEvent) bool { + sender.On("SendEvent", mock.Anything, mock.MatchedBy(func(e models.IncomingEvent) bool { return reflect.DeepEqual(e.TargetDocument, map[string]interface{}{"foo": map[string]interface{}{"a": "b"}, "foo2": "bar2"}) }), mock.Anything).Return(nil) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - mockDocumentStore.On("GetDocument", mock.Anything).Return(map[string]interface{}{"foo": map[string]interface{}{"bar": "yo", "a": "b"}, "foo2": "bar2"}) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + mockDocumentStore.On("GetDocument", mock.Anything, mock.Anything).Return(map[string]interface{}{"foo": map[string]interface{}{"bar": "yo", "a": "b"}, "foo2": "bar2"}) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -507,15 +511,15 @@ func TestDispatchEvents_With_Exclude_Document_Attrs(t *testing.T) { func TestDispatchEvents_With_Include_Payload_Attrs(t *testing.T) { store, documentStores, sender, dispatcher, throttler := setup("fixtures/config.include_payload_attrs.json") - sender.On("SendEvent", mock.MatchedBy(func(e models.IncomingEvent) bool { + sender.On("SendEvent", mock.Anything, mock.MatchedBy(func(e models.IncomingEvent) bool { return reflect.DeepEqual(e.Payload, map[string]interface{}{"field1": 1}) }), mock.Anything).Return(nil) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1, "field2": 2}, @@ -529,15 +533,15 @@ func TestDispatchEvents_With_Include_Payload_Attrs(t *testing.T) { func TestDispatchEvents_With_Exclude_Payload_Attrs(t *testing.T) { store, documentStores, sender, dispatcher, throttler := setup("fixtures/config.exclude_payload_attrs.json") - sender.On("SendEvent", mock.MatchedBy(func(e models.IncomingEvent) bool { + sender.On("SendEvent", mock.Anything, mock.MatchedBy(func(e models.IncomingEvent) bool { return reflect.DeepEqual(e.Payload, map[string]interface{}{"field2": 2}) }), mock.Anything).Return(nil) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1, "field2": 2}, @@ -555,16 +559,16 @@ func TestDispatchEvents_WithSpecificDocumentStore(t *testing.T) { documentStores["specific"] = anotherDocumentStore documentStores["default"] = defaultDocumentStore - sender.On("SendEvent", mock.MatchedBy(hasDocument), mock.Anything).Return(nil) + sender.On("SendEvent", mock.Anything, mock.MatchedBy(hasDocument), mock.Anything).Return(nil) // sender.On("SendEvent", mock.MatchedBy(hasNoDocument), mock.Anything).Return(nil) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - defaultDocumentStore.On("GetDocument", mock.Anything).Return(map[string]interface{}{"foo": "bar"}) - anotherDocumentStore.On("GetDocument", mock.Anything).Return(map[string]interface{}{"foo": "bar"}) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + defaultDocumentStore.On("GetDocument", mock.Anything, mock.Anything).Return(map[string]interface{}{"foo": "bar"}) + anotherDocumentStore.On("GetDocument", mock.Anything, mock.Anything).Return(map[string]interface{}{"foo": "bar"}) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -578,15 +582,15 @@ func TestDispatchEvents_WithSpecificDocumentStore(t *testing.T) { func TestDispatchEvents_Throttled_Without_TrailingEdge(t *testing.T) { store, documentStores, sender, dispatcher, throttler := setup("fixtures/config.throttle.disable_trailing.json") - sender.On("SendEvent", mock.Anything, mock.Anything).Return(nil) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - store.On("Remove", mock.Anything).Return(true, nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(nil) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + store.On("Remove", mock.Anything, mock.Anything).Return(true, nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(true, 500*time.Millisecond, nil).Once() - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil).Twice() + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(true, 500*time.Millisecond, nil).Once() + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(false, 500*time.Millisecond, nil).Twice() - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -594,7 +598,7 @@ func TestDispatchEvents_Throttled_Without_TrailingEdge(t *testing.T) { TargetId: "product_id", }, store, throttler, documentStores) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -602,7 +606,7 @@ func TestDispatchEvents_Throttled_Without_TrailingEdge(t *testing.T) { TargetId: "product_id", }, store, throttler, documentStores) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -619,6 +623,7 @@ func TestDispatchEvents_OnError(t *testing.T) { _, _, _, dispatcher, _ := setup("fixtures/config.single.json") dispatcher.OnError( + context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", @@ -635,12 +640,12 @@ func TestDispatchEvents_DispatchErrorTriggerOnError(t *testing.T) { store, documentStores, sender, dispatcher, throttler := setup("fixtures/config.single.json") - sender.On("SendEvent", mock.Anything, mock.Anything).Return(errors.New("Mock Error")) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("Mock Error")) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(true, time.Duration(0), nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core", Payload: map[string]interface{}{"field1": 1}, @@ -658,12 +663,12 @@ func TestDispatchEvents_DispatchErrorTriggerOnError(t *testing.T) { func TestDispatchEvents_DelaySend_NotExist(t *testing.T) { store, documentStores, sender, dispatcher, throttler := setup("fixtures/config.single.json") - sender.On("SendEvent", mock.Anything, mock.Anything).Return(nil) - store.On("Get", mock.Anything).Return("", false, time.Duration(0), nil) - store.On("Set", mock.Anything, mock.Anything, mock.Anything).Return(true, nil) - throttler.On("CanTrigger", mock.Anything, mock.Anything).Return(false, time.Duration(0), nil) + sender.On("SendEvent", mock.Anything, mock.Anything, mock.Anything).Return(nil) + store.On("Get", mock.Anything, mock.Anything).Return("", false, time.Duration(0), nil) + store.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(true, nil) + throttler.On("CanTrigger", mock.Anything, mock.Anything, mock.Anything).Return(false, time.Duration(0), nil) - dispatcher.Dispatch(models.IncomingEvent{ + dispatcher.Dispatch(context.Background(), models.IncomingEvent{ Key: "product.update", Source: "core-api", Payload: map[string]interface{}{"field1": 1}, diff --git a/test/internal/stores/memstore_test.go b/test/internal/stores/memstore_test.go index f336507..3522599 100644 --- a/test/internal/stores/memstore_test.go +++ b/test/internal/stores/memstore_test.go @@ -1,11 +1,12 @@ package stores_test import ( + "context" "fmt" "testing" "time" - stores "github.com/shoplineapp/captin/internal/stores" + stores "github.com/shoplineapp/captin/v2/internal/stores" "github.com/stretchr/testify/assert" ) @@ -13,7 +14,7 @@ func TestStoreWithTTL(t *testing.T) { ms := stores.NewMemoryStore() for i := 0; i < 10000; i++ { k, v := fmt.Sprint("key", i), fmt.Sprint("value", i) - result, err := ms.Set(k, v, 200*time.Millisecond) + result, err := ms.Set(context.Background(), k, v, 200*time.Millisecond) assert.Nil(t, err) assert.True(t, result) } diff --git a/test/internal/throttles/throttler_test.go b/test/internal/throttles/throttler_test.go index 79f6803..417b0ea 100644 --- a/test/internal/throttles/throttler_test.go +++ b/test/internal/throttles/throttler_test.go @@ -1,15 +1,17 @@ package throttles_test import ( + "context" "errors" "testing" "time" - throttles "github.com/shoplineapp/captin/internal/throttles" + throttles "github.com/shoplineapp/captin/v2/internal/throttles" - mocks "github.com/shoplineapp/captin/test/mocks" + mocks "github.com/shoplineapp/captin/v2/test/mocks" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) func setup() (string, time.Duration, *mocks.StoreMock) { @@ -22,17 +24,17 @@ func setup() (string, time.Duration, *mocks.StoreMock) { func TestThrottler_CreateStoreRecord(t *testing.T) { throttleID, throttlePeriod, store := setup() - store.On("Get", throttleID).Return("", false, time.Millisecond*10, nil) - store.On("Set", throttleID, "1", throttlePeriod).Return(true, nil) + store.On("Get", mock.Anything, throttleID).Return("", false, time.Millisecond*10, nil) + store.On("Set", mock.Anything, throttleID, "1", throttlePeriod).Return(true, nil) subject := throttles.NewThrottler(store) - result, duration, err := subject.CanTrigger(throttleID, throttlePeriod) + result, duration, err := subject.CanTrigger(context.Background(), throttleID, throttlePeriod) assert.True(t, result) assert.Equal(t, time.Duration(0), duration) assert.Nil(t, err) - store.AssertCalled(t, "Get", throttleID) - store.AssertCalled(t, "Set", throttleID, "1", throttlePeriod) + store.AssertCalled(t, "Get", mock.Anything, throttleID) + store.AssertCalled(t, "Set", mock.Anything, throttleID, "1", throttlePeriod) } func TestThrottler_NoThrottleSet(t *testing.T) { @@ -40,39 +42,39 @@ func TestThrottler_NoThrottleSet(t *testing.T) { throttlePeriod := time.Duration(0) subject := throttles.NewThrottler(store) - result, duration, err := subject.CanTrigger(throttleID, throttlePeriod) + result, duration, err := subject.CanTrigger(context.Background(), throttleID, throttlePeriod) assert.True(t, result) assert.Equal(t, time.Duration(0), duration) assert.Nil(t, err) - store.AssertNotCalled(t, "Get", throttleID) - store.AssertNotCalled(t, "Set", throttleID, "1", throttlePeriod) + store.AssertNotCalled(t, "Get", mock.Anything, throttleID) + store.AssertNotCalled(t, "Set", mock.Anything, throttleID, "1", throttlePeriod) } func TestThrottler_Reject(t *testing.T) { throttleID, throttlePeriod, store := setup() - store.On("Get", throttleID).Return("1", true, time.Millisecond*10, nil) + store.On("Get", mock.Anything, throttleID).Return("1", true, time.Millisecond*10, nil) subject := throttles.NewThrottler(store) - result, duration, err := subject.CanTrigger(throttleID, throttlePeriod) + result, duration, err := subject.CanTrigger(context.Background(), throttleID, throttlePeriod) assert.False(t, result) assert.Equal(t, time.Millisecond*10, duration) assert.Nil(t, err) - store.AssertCalled(t, "Get", throttleID) + store.AssertCalled(t, "Get", mock.Anything, throttleID) } func TestThrottler_Error(t *testing.T) { throttleID, throttlePeriod, store := setup() - store.On("Get", throttleID).Return("", false, time.Duration(0), errors.New("some error")) + store.On("Get", mock.Anything, throttleID).Return("", false, time.Duration(0), errors.New("some error")) subject := throttles.NewThrottler(store) - result, duration, err := subject.CanTrigger(throttleID, throttlePeriod) + result, duration, err := subject.CanTrigger(context.Background(), throttleID, throttlePeriod) assert.True(t, result) assert.Equal(t, time.Duration(0), duration) assert.EqualError(t, err, "some error") - store.AssertCalled(t, "Get", throttleID) + store.AssertCalled(t, "Get", mock.Anything, throttleID) } diff --git a/test/mocks/document_store_mock.go b/test/mocks/document_store_mock.go index a9c6728..8d4aafd 100644 --- a/test/mocks/document_store_mock.go +++ b/test/mocks/document_store_mock.go @@ -1,8 +1,10 @@ package mocks import ( - "github.com/shoplineapp/captin/interfaces" - "github.com/shoplineapp/captin/models" + "context" + + "github.com/shoplineapp/captin/v2/interfaces" + "github.com/shoplineapp/captin/v2/models" "github.com/stretchr/testify/mock" ) @@ -13,9 +15,8 @@ type DocumentStoreMock struct { } // Get - Get value from store, return with remaining time -func (ds *DocumentStoreMock) GetDocument(ie interfaces.IncomingEventInterface) (map[string]interface{}) { +func (ds *DocumentStoreMock) GetDocument(ctx context.Context, ie interfaces.IncomingEventInterface) map[string]interface{} { e := ie.(models.IncomingEvent) - args := ds.Called(e) + args := ds.Called(ctx, e) return args.Get(0).(map[string]interface{}) } - diff --git a/test/mocks/sender_mock.go b/test/mocks/sender_mock.go index 37f9beb..3adbf2b 100644 --- a/test/mocks/sender_mock.go +++ b/test/mocks/sender_mock.go @@ -1,8 +1,10 @@ package mocks import ( - "github.com/shoplineapp/captin/interfaces" - "github.com/shoplineapp/captin/models" + "context" + + "github.com/shoplineapp/captin/v2/interfaces" + "github.com/shoplineapp/captin/v2/models" "github.com/stretchr/testify/mock" ) @@ -13,9 +15,9 @@ type SenderMock struct { } // SendEvent - Send an event -func (s *SenderMock) SendEvent(ie interfaces.IncomingEventInterface, id interfaces.DestinationInterface) error { +func (s *SenderMock) SendEvent(ctx context.Context, ie interfaces.IncomingEventInterface, id interfaces.DestinationInterface) error { e := ie.(models.IncomingEvent) d := id.(models.Destination) - args := s.Called(e, d) + args := s.Called(ctx, e, d) return args.Error(0) } diff --git a/test/mocks/store_mock.go b/test/mocks/store_mock.go index 55cedda..4412d1f 100644 --- a/test/mocks/store_mock.go +++ b/test/mocks/store_mock.go @@ -1,6 +1,7 @@ package mocks import ( + "context" "fmt" "time" @@ -16,41 +17,41 @@ type StoreMock struct { } // Get - Get value from store, return with remaining time -func (s *StoreMock) Get(key string) (string, bool, time.Duration, error) { - args := s.Called(key) +func (s *StoreMock) Get(ctx context.Context, key string) (string, bool, time.Duration, error) { + args := s.Called(ctx, key) return args.String(0), args.Bool(1), args.Get(2).(time.Duration), args.Error(3) } // Set - Set value into store with ttl -func (s *StoreMock) Set(key string, value string, ttl time.Duration) (bool, error) { - args := s.Called(key, value, ttl) +func (s *StoreMock) Set(ctx context.Context, key string, value string, ttl time.Duration) (bool, error) { + args := s.Called(ctx, key, value, ttl) return args.Bool(0), args.Error(1) } // Update - Update value for key -func (s *StoreMock) Update(key string, value string) (bool, error) { - args := s.Called(key, value) +func (s *StoreMock) Update(ctx context.Context, key string, value string) (bool, error) { + args := s.Called(ctx, key, value) return args.Bool(0), args.Error(1) } -func (s *StoreMock) Enqueue(key string, value string, ttl time.Duration) (bool, error) { - args := s.Called(key, value, ttl) +func (s *StoreMock) Enqueue(ctx context.Context, key string, value string, ttl time.Duration) (bool, error) { + args := s.Called(ctx, key, value, ttl) return args.Bool(0), args.Error(1) } -func (s *StoreMock) GetQueue(key string) ([]string, bool, time.Duration, error) { - args := s.Called(key) +func (s *StoreMock) GetQueue(ctx context.Context, key string) ([]string, bool, time.Duration, error) { + args := s.Called(ctx, key) return args.Get(0).([]string), args.Bool(1), args.Get(2).(time.Duration), args.Error(3) } // Remove - Remove value for key -func (s *StoreMock) Remove(key string) (bool, error) { - args := s.Called(key) +func (s *StoreMock) Remove(ctx context.Context, key string) (bool, error) { + args := s.Called(ctx, key) return args.Bool(0), args.Error(1) } // DataKey - Generate DataKey with events and destination (Won't Mock) -func (s *StoreMock) DataKey(ie interfaces.IncomingEventInterface, idest interfaces.DestinationInterface, prefix string, suffix string) string { +func (s *StoreMock) DataKey(ctx context.Context, ie interfaces.IncomingEventInterface, idest interfaces.DestinationInterface, prefix string, suffix string) string { e := ie.(models.IncomingEvent) dest := idest.(models.Destination) return fmt.Sprintf("%s%s.%s.%s%s", prefix, e.Key, dest.Config.GetName(), e.TargetId, suffix) diff --git a/test/mocks/throttle_mock.go b/test/mocks/throttle_mock.go index 2c62b26..077de21 100644 --- a/test/mocks/throttle_mock.go +++ b/test/mocks/throttle_mock.go @@ -1,6 +1,7 @@ package mocks import ( + "context" "time" "github.com/shoplineapp/captin/interfaces" @@ -13,8 +14,7 @@ type ThrottleMock struct { mock.Mock } -// CanTrigger - Check if can trigger -func (t *ThrottleMock) CanTrigger(id string, period time.Duration) (bool, time.Duration, error) { - args := t.Called(id, period) +func (t *ThrottleMock) CanTrigger(ctx context.Context, id string, period time.Duration) (bool, time.Duration, error) { + args := t.Called(ctx, id, period) return args.Bool(0), args.Get(1).(time.Duration), args.Error(2) } diff --git a/test/models/config_mapper_test.go b/test/models/config_mapper_test.go index d3bd5a3..846c0c7 100644 --- a/test/models/config_mapper_test.go +++ b/test/models/config_mapper_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/assert" - . "github.com/shoplineapp/captin/models" - interfaces "github.com/shoplineapp/captin/interfaces" + . "github.com/shoplineapp/captin/v2/models" + interfaces "github.com/shoplineapp/captin/v2/interfaces" ) func setup() []interfaces.ConfigurationInterface { diff --git a/test/models/config_test.go b/test/models/config_test.go index f7baf70..c8a7707 100644 --- a/test/models/config_test.go +++ b/test/models/config_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" - . "github.com/shoplineapp/captin/models" + . "github.com/shoplineapp/captin/v2/models" ) func TestDecodeConfigurationJson(t *testing.T) { diff --git a/test/models/destination_test.go b/test/models/destination_test.go index 638a902..5affed5 100644 --- a/test/models/destination_test.go +++ b/test/models/destination_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" - . "github.com/shoplineapp/captin/models" + . "github.com/shoplineapp/captin/v2/models" ) func TestDestination_GetCallbackURL(t *testing.T) { diff --git a/test/models/incoming_event_test.go b/test/models/incoming_event_test.go index 9ad102a..bb6d372 100644 --- a/test/models/incoming_event_test.go +++ b/test/models/incoming_event_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" - . "github.com/shoplineapp/captin/models" + . "github.com/shoplineapp/captin/v2/models" ) func TestNewIncomingEvent(t *testing.T) { diff --git a/test/senders/beanstlakd_sender_test.go b/test/senders/beanstlakd_sender_test.go index d97a6db..ab9946a 100644 --- a/test/senders/beanstlakd_sender_test.go +++ b/test/senders/beanstlakd_sender_test.go @@ -1,12 +1,13 @@ package senders_test import ( + "context" "strings" "testing" - models "github.com/shoplineapp/captin/models" + models "github.com/shoplineapp/captin/v2/models" - "github.com/shoplineapp/captin/senders" + "github.com/shoplineapp/captin/v2/senders" "github.com/stretchr/testify/assert" ) @@ -38,6 +39,7 @@ func TestBeanstalkdSender_SendEvent_BeanstalkdHost(t *testing.T) { } got := sender.SendEvent( + context.Background(), models.IncomingEvent{ Control: map[string]interface{}{ "beanstalkd_host": beanstalkdHost, @@ -83,6 +85,7 @@ func TestBeanstalkdSender_SendEvent_QueueName(t *testing.T) { } got := sender.SendEvent( + context.Background(), models.IncomingEvent{ Control: map[string]interface{}{ "beanstalkd_host": "127.0.0.1:11300", diff --git a/test/senders/sqs_sender_test.go b/test/senders/sqs_sender_test.go index 014a119..b037cf2 100644 --- a/test/senders/sqs_sender_test.go +++ b/test/senders/sqs_sender_test.go @@ -1,16 +1,17 @@ package senders_test import ( - "os" + "context" "encoding/json" "errors" + "os" "testing" aws "github.com/aws/aws-sdk-go/aws" aws_sqs "github.com/aws/aws-sdk-go/service/sqs" aws_sqsiface "github.com/aws/aws-sdk-go/service/sqs/sqsiface" - models "github.com/shoplineapp/captin/models" - . "github.com/shoplineapp/captin/senders" + models "github.com/shoplineapp/captin/v2/models" + . "github.com/shoplineapp/captin/v2/senders" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -48,6 +49,7 @@ func TestSqsSender_SendEvent_Success(t *testing.T) { sender.DefaultClient = sqs result := sender.SendEvent( + context.Background(), models.IncomingEvent{}, models.Destination{ Config: models.Configuration{}, @@ -67,6 +69,7 @@ func TestSqsSender_SendEvent_Failed(t *testing.T) { sender.DefaultClient = sqs result := sender.SendEvent( + context.Background(), models.IncomingEvent{Control: map[string]interface{}{"result": "failed"}}, models.Destination{ Config: models.Configuration{Name: "failed"}, @@ -121,6 +124,7 @@ func TestSqsSender_SendEvent_UseAccessKey_Success(t *testing.T) { sender.DestinationClientMap["test_destination"] = sqs result := sender.SendEvent( + context.Background(), models.IncomingEvent{}, models.Destination{ Config: models.Configuration{ From de508c9c97798b24aa1d34ba57daca269bf25695 Mon Sep 17 00:00:00 2001 From: Chung Wu Date: Thu, 11 Jan 2024 15:03:46 +0800 Subject: [PATCH 2/8] chore: add comment and warning for Dispatcher double dispatch --- internal/outgoing/dispatcher.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/outgoing/dispatcher.go b/internal/outgoing/dispatcher.go index 9774d13..be56128 100644 --- a/internal/outgoing/dispatcher.go +++ b/internal/outgoing/dispatcher.go @@ -6,6 +6,7 @@ import ( "fmt" "strconv" "sync" + "sync/atomic" "time" "github.com/mohae/deepcopy" @@ -35,6 +36,8 @@ type Dispatcher struct { muTargetDocument sync.Mutex muErrors sync.Mutex + + dispatchCalled atomic.Bool } // NewDispatcherWithDestinations - Create Outgoing event dispatcher with destinations @@ -94,6 +97,9 @@ func (d *Dispatcher) OnError(ctx context.Context, evt interfaces.IncomingEventIn } // Dispatch - Dispatch an event to outgoing webhook +// NOTE: +// This function should not be called more than once, as it alters the object state (e.g. errors, caches) +// Subsequence calls should construct a new instance of Dispatcher. func (d *Dispatcher) Dispatch( ctx context.Context, event interfaces.IncomingEventInterface, @@ -101,8 +107,13 @@ func (d *Dispatcher) Dispatch( throttler interfaces.ThrottleInterface, documentStoreMappings map[string]interfaces.DocumentStoreInterface, ) interfaces.ErrorInterface { - responses := make(chan int, len(d.destinations)) e := event.(models.IncomingEvent) + alreadyCalled := d.dispatchCalled.CompareAndSwap(false, true) + if alreadyCalled { + dLogger.WithField("event", e).Warn("Triggering dispatch more than once") + } + d.dispatchCalled.Store(true) + responses := make(chan int, len(d.destinations)) for _, destination := range d.destinations { config := destination.Config canTrigger, timeRemain, err := throttler.CanTrigger(ctx, getEventKey(ctx, store, e, destination), config.GetThrottleValue()) From b10b480d959925537142a7895ba0cc4d5aa380ee Mon Sep 17 00:00:00 2001 From: Chung Wu Date: Wed, 10 Jan 2024 21:23:31 +0800 Subject: [PATCH 3/8] add tracing and propagation --- internal/helpers/tracing.go | 10 ++++ internal/outgoing/custom.go | 10 +++- internal/outgoing/dispatcher.go | 52 +++++++++++++++++- models/distributed_tracing_info.go | 87 ++++++++++++++++++++++++++++++ models/incoming_event.go | 22 +++++--- senders/beanstlakd_sender.go | 17 +++++- senders/http_proxy_sender.go | 40 +++++++++----- senders/http_sender.go | 36 +++++++------ senders/sqs_sender.go | 19 ++++++- 9 files changed, 250 insertions(+), 43 deletions(-) create mode 100644 internal/helpers/tracing.go create mode 100644 models/distributed_tracing_info.go diff --git a/internal/helpers/tracing.go b/internal/helpers/tracing.go new file mode 100644 index 0000000..c7f5b6a --- /dev/null +++ b/internal/helpers/tracing.go @@ -0,0 +1,10 @@ +package helpers + +import ( + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" +) + +func Tracer() trace.Tracer { + return otel.GetTracerProvider().Tracer("captin") +} diff --git a/internal/outgoing/custom.go b/internal/outgoing/custom.go index e9ab362..e456e0d 100644 --- a/internal/outgoing/custom.go +++ b/internal/outgoing/custom.go @@ -4,7 +4,11 @@ import ( "context" destination_filters "github.com/shoplineapp/captin/v2/destinations/filters" + "github.com/shoplineapp/captin/v2/internal/helpers" + models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) var cLogger = log.WithFields(log.Fields{"class": "Custom"}) @@ -13,6 +17,8 @@ type Custom struct{} // Sift - Custom check will filter ineligible destination func (c Custom) Sift(ctx context.Context, e *models.IncomingEvent, destinations []models.Destination, filters []destination_filters.DestinationFilterInterface, middlewares []destination_filters.DestinationMiddlewareInterface) []models.Destination { + ctx, span := helpers.Tracer().Start(ctx, "captin.Custom.Sift") + defer span.End() cLogger.WithFields(log.Fields{ "event": e, "destinations": destinations, @@ -34,10 +40,12 @@ func (c Custom) Sift(ctx context.Context, e *models.IncomingEvent, destinations } if eligible { sifted = append(sifted, destination) + } else { + span.AddEvent("destination removed", trace.WithAttributes(attribute.String("destination", destination.Config.GetName()))) } } for _, m := range middlewares { - sifted = m.Apply(e, sifted) + sifted = m.Apply(ctx, e, sifted) } return sifted diff --git a/internal/outgoing/dispatcher.go b/internal/outgoing/dispatcher.go index be56128..f595ba4 100644 --- a/internal/outgoing/dispatcher.go +++ b/internal/outgoing/dispatcher.go @@ -18,6 +18,9 @@ import ( "github.com/shoplineapp/captin/v2/internal/helpers" models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) var dLogger = log.WithFields(log.Fields{"class": "Dispatcher"}) @@ -114,6 +117,15 @@ func (d *Dispatcher) Dispatch( } d.dispatchCalled.Store(true) responses := make(chan int, len(d.destinations)) + ctx = e.DistributedTracingInfo.PropagateIntoContext(ctx) + ctx, span := helpers.Tracer().Start(ctx, "captin.Dispatch", trace.WithAttributes( + attribute.String("event_key", e.Key), + attribute.Bool("dispatch_already_called", alreadyCalled), + attribute.Int("raw_destination_count", len(d.destinations)), + )) + defer span.End() + // propagate the trace context into the event, for both downstream receivers, and also later workers if the event is delayed or throttled + e.DistributedTracingInfo.InjectContext(ctx) for _, destination := range d.destinations { config := destination.Config canTrigger, timeRemain, err := throttler.CanTrigger(ctx, getEventKey(ctx, store, e, destination), config.GetThrottleValue()) @@ -186,6 +198,7 @@ func (d *Dispatcher) injectThrottledPayloads(ctx context.Context, e models.Incom store.Remove(ctx, queueKey) for _, payloadStr := range payloadStrings { payload := map[string]interface{}{} + // TODO: handle error json.Unmarshal([]byte(payloadStr), &payload) e.ThrottledPayloads = append(e.ThrottledPayloads, payload) } @@ -201,6 +214,7 @@ func (d *Dispatcher) injectThrottledDocuments(ctx context.Context, e models.Inco store.Remove(ctx, queueKey) for _, documentStr := range documentStrings { document := map[string]interface{}{} + // TODO: handle error json.Unmarshal([]byte(documentStr), &document) e.ThrottledDocuments = append(e.ThrottledDocuments, document) } @@ -243,6 +257,7 @@ func (d *Dispatcher) customizePayload(ctx context.Context, e models.IncomingEven } func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingEvent, timeRemain time.Duration, dest models.Destination, store interfaces.StoreInterface, documentStore interfaces.DocumentStoreInterface) { + ctx, span := helpers.Tracer().Start(ctx, "captin.processDelayedEvent") defer func() { if err := recover(); err != nil { err := &captin_errors.DispatcherError{ @@ -250,7 +265,10 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE Destination: dest, Event: e, } + span.RecordError(err) + span.SetStatus(codes.Error, err.Msg) d.OnError(ctx, e, err) + span.End() } }() @@ -274,6 +292,8 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE "event": e, "eventDataKey": "dataKey", }).Debug("Skipping update on event data") + span.AddEvent("Skipping update on event data") + span.End() return } @@ -291,6 +311,7 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE "enqueuePayload": jsonString, "ttl": ttl, }).Debug("Storing throttled payload") + span.AddEvent("Storing throttled payload", trace.WithAttributes(attribute.String("queueKey", queueKey), attribute.String("throttleValue", ttl.String()))) store.Enqueue(ctx, queueKey, string(jsonString), ttl) } @@ -308,6 +329,7 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE "enqueueDocument": jsonString, "ttl": ttl, }).Debug("Storing throttled document") + span.AddEvent("Storing throttled document", trace.WithAttributes(attribute.String("queueKey", queueKey), attribute.String("throttleValue", ttl.String()))) store.Enqueue(ctx, queueKey, string(jsonString), ttl) } @@ -332,16 +354,20 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE // Schedule send event later dispatcher.TrackAfterFuncJob(timeRemain, func() { + defer span.End() dLogger.WithFields(log.Fields{"key": dataKey}).Debug("After event callback") - payload, exists, _, _ := store.Get(dataKey) + payload, exists, _, _ := store.Get(ctx, dataKey) // Key might be deleted by another worker, resulting in data not found if !exists { dLogger.WithFields(log.Fields{"key": dataKey}).Debug("Event data not found") err := &captin_errors.UnretryableError{Msg: "Event data not found", Event: e, Destination: dest} + span.RecordError(err) + span.SetStatus(codes.Error, err.Msg) d.OnError(ctx, models.IncomingEvent{}, err) return } event := models.IncomingEvent{} + // TODO: check error json.Unmarshal([]byte(payload), &event) d.sendEvent(ctx, event, dest, store, documentStore) store.Remove(ctx, dataKey) @@ -390,6 +416,11 @@ func getEventThrottledDocumentsKey(ctx context.Context, s interfaces.StoreInterf return s.DataKey(ctx, e, d, "", "-throttled_documents") } +// TODO: +// This method and subsequential calls should be extracted as a "sub-Dispatcher" dedicated for one destination, so that +// (1) the logic are clearer +// (2) we can keep destination-specific state in the sub-Dispatcher instead of passing everything as arguments again and again +// (3) we can keep context and span for distributed tracing func (d *Dispatcher) sendEvent(ctx context.Context, evt models.IncomingEvent, destination models.Destination, store interfaces.StoreInterface, documentStore interfaces.DocumentStoreInterface) { config := destination.Config callbackLogger := dLogger.WithFields(log.Fields{ @@ -399,6 +430,13 @@ func (d *Dispatcher) sendEvent(ctx context.Context, evt models.IncomingEvent, de "callback_url": destination.GetCallbackURL(), "document_store": destination.GetDocumentStore(), }) + // derive a new context and new span for this destination + ctx, span := helpers.Tracer().Start(ctx, "captin.sendEvent", trace.WithAttributes( + attribute.String("destination", config.GetName()), + attribute.String("action", evt.Key), + attribute.String("callback_url", destination.GetCallbackURL()), + attribute.String("document_store", destination.GetDocumentStore()), + )) defer func() { if err := recover(); err != nil { @@ -409,8 +447,10 @@ func (d *Dispatcher) sendEvent(ctx context.Context, evt models.IncomingEvent, de Destination: destination, Event: evt, }) + span.RecordError(err.(error)) + span.SetStatus(codes.Error, errMsg) + span.End() } - return }() callbackLogger.Debug("Preprocess payload and document") @@ -435,6 +475,7 @@ func (d *Dispatcher) sendEvent(ctx context.Context, evt models.IncomingEvent, de if senderKey == "" { senderKey = "http" } + span.SetAttributes(attribute.String("sender", senderKey)) sender, senderExists := d.senderMapping[senderKey] if senderExists == false { panic(&captin_errors.DispatcherError{ @@ -446,6 +487,7 @@ func (d *Dispatcher) sendEvent(ctx context.Context, evt models.IncomingEvent, de // Wrap event sender and error handling as closure for reusing in delayer _sendEvent := func() { + defer span.End() defer func() { if err := recover(); err != nil { var newErr error @@ -461,10 +503,15 @@ func (d *Dispatcher) sendEvent(ctx context.Context, evt models.IncomingEvent, de } } d.OnError(ctx, evt, newErr) + span.RecordError(newErr) + span.SetStatus(codes.Error, newErr.Error()) } }() + span.AddEvent("_sendEvent") // Deep clone a new instance to prevent concurrent iteration and write on json.Marshal event := deepcopy.Copy(evt).(models.IncomingEvent) + // propagate the new trace context into the event + event.DistributedTracingInfo.InjectContext(ctx) err := sender.SendEvent(ctx, event, destination) if err != nil { panic(err) @@ -473,6 +520,7 @@ func (d *Dispatcher) sendEvent(ctx context.Context, evt models.IncomingEvent, de } if destination.RequireDelay(evt) { + span.SetAttributes(attribute.Bool("RequireDelay", true)) // Sending message with delay in goroutine, no error will be caught callbackLogger.Info(fmt.Sprintf("Event requires delay")) if d.delayer != nil { diff --git a/models/distributed_tracing_info.go b/models/distributed_tracing_info.go new file mode 100644 index 0000000..75d4e90 --- /dev/null +++ b/models/distributed_tracing_info.go @@ -0,0 +1,87 @@ +package models + +import ( + "context" + "encoding/json" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +// DistributedTracingInfo is basically the same as propagation.MapCarrier +// But we reimplement it for easier interop (e.g. serialization and deserialization without caring map being nil) +type DistributedTracingInfo struct { + Carrier propagation.MapCarrier +} + +var _ propagation.TextMapCarrier = (*DistributedTracingInfo)(nil) +var _ json.Marshaler = (*DistributedTracingInfo)(nil) +var _ json.Unmarshaler = (*DistributedTracingInfo)(nil) + +// To prevent nil pointer dereference, we need to initialize the MapCarrier before any action +func (d *DistributedTracingInfo) setup() { + if d.Carrier == nil { + d.Carrier = make(propagation.MapCarrier) + } +} + +func (d *DistributedTracingInfo) Get(s string) string { + d.setup() + return d.Carrier.Get(s) +} + +func (d *DistributedTracingInfo) Set(key, value string) { + d.setup() + d.Carrier.Set(key, value) +} + +func (d *DistributedTracingInfo) Keys() []string { + d.setup() + return d.Carrier.Keys() +} + +func (d DistributedTracingInfo) MarshalJSON() ([]byte, error) { + d.setup() + return json.Marshal(d.Carrier) +} + +func (d *DistributedTracingInfo) UnmarshalJSON(bz []byte) error { + return json.Unmarshal(bz, &d.Carrier) +} + +// If it's not set, then the default value of otel.GetTextMapPropagator() will be a no-op propagator. +// But we want to make sure that it works, so we prepend our propagator by a composite propagator. +// If someone use cases don't want propagation, ClearContext can be used. +func getPropagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, otel.GetTextMapPropagator()) +} + +// PropagateIntoContext takes a context, extract the tracing info from the DistributedTracingInfo, +// then return a new context with the tracing info injected based on the original context +func (d *DistributedTracingInfo) PropagateIntoContext(ctx context.Context) context.Context { + // extract the tracing info from the carrier d into ctx + return getPropagator().Extract(ctx, d) +} + +func NewDistributedTracingInfoFromContext(ctx context.Context) DistributedTracingInfo { + var d DistributedTracingInfo + // inject the tracing info from ctx into the carrier d + getPropagator().Inject(ctx, &d) + return d +} + +func (d *DistributedTracingInfo) InjectContext(ctx context.Context) *DistributedTracingInfo { + getPropagator().Inject(ctx, d) + return d +} + +func (d *DistributedTracingInfo) ClearContext() *DistributedTracingInfo { + d.Carrier = nil + d.setup() + return d +} + +// GetTraceParent returns the traceparent header value, which provides a convenient way to get a representation of the current trace context +func (d *DistributedTracingInfo) GetTraceParent() string { + return d.Get("traceparent") +} diff --git a/models/incoming_event.go b/models/incoming_event.go index df1425b..5eed915 100644 --- a/models/incoming_event.go +++ b/models/incoming_event.go @@ -12,15 +12,21 @@ import ( interfaces "github.com/shoplineapp/captin/v2/interfaces" ) +var _ interfaces.IncomingEventInterface = IncomingEvent{} + type IncomingEvent struct { - interfaces.IncomingEventInterface - - TraceId string `json:"trace_id"` - Key string `json:"event_key"` // Required, The identifier of an event, usually form as PREFIX.MODEL.ACTION - Source string `json:"source"` // Required, Event source from - Payload map[string]interface{} `json:"payload"` // Optional, custom payload / document from caller - ThrottledPayloads []map[string]interface{} `json:"throttled_payloads,omitempty"` // for response only - Control map[string]interface{} `json:"control"` // Optional, custom control values from caller + // Note: + // this `TraceID` is NOT the OpenTelemetry trace ID, but more like a message ID which persist across message retries + // We keep this for backward compatibility + // For OpenTelemetry trace ID, see DistributedTracingInfo + TraceId string `json:"trace_id"` + + Key string `json:"event_key"` // Required, The identifier of an event, usually form as PREFIX.MODEL.ACTION + Source string `json:"source"` // Required, Event source from + Payload map[string]interface{} `json:"payload"` // Optional, custom payload / document from caller + ThrottledPayloads []map[string]interface{} `json:"throttled_payloads,omitempty"` // for response only + Control map[string]interface{} `json:"control"` // Optional, custom control values from caller + DistributedTracingInfo DistributedTracingInfo `json:"distributed_tracing_info"` // Optional, for distributed tracing (e.g. OpenTelemetry) // Optional with payload, Captin will try to fetch the document from the default database TargetType string `json:"target_type"` diff --git a/senders/beanstlakd_sender.go b/senders/beanstlakd_sender.go index 5f424d1..b1902c5 100644 --- a/senders/beanstlakd_sender.go +++ b/senders/beanstlakd_sender.go @@ -13,8 +13,11 @@ import ( statsd "github.com/joeycumines/statsd" captin_errors "github.com/shoplineapp/captin/v2/errors" interfaces "github.com/shoplineapp/captin/v2/interfaces" + "github.com/shoplineapp/captin/v2/internal/helpers" models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" ) var bLogger = log.WithFields(log.Fields{"class": "BeanstalkdSender"}) @@ -31,6 +34,15 @@ type BeanstalkdSender struct { // SendEvent - #BeanstalkdSender SendEvent func (c *BeanstalkdSender) SendEvent(ctx context.Context, ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) (err error) { + ctx, span := helpers.Tracer().Start(ctx, "captin.BeanstalkdSender.SendEvent") + defer func() { + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + span.End() + }() + e := ev.(models.IncomingEvent) d := dv.(models.Destination) @@ -69,6 +81,7 @@ func (c *BeanstalkdSender) SendEvent(ctx context.Context, ev interfaces.Incoming //return &captin_errors.UnretryableError{Msg: "beanstalkd_host is invalid", Event: e} } + span.SetAttributes(attribute.String("beanstalkdHost", beanstalkdHostStr)) conn, err := beanstalk.Dial("tcp", beanstalkdHostStr) if err != nil { @@ -80,6 +93,7 @@ func (c *BeanstalkdSender) SendEvent(ctx context.Context, ev interfaces.Incoming } return err } + defer conn.Close() beanstalkdQueueName := e.Control["queue_name"] if beanstalkdQueueName == nil || beanstalkdQueueName == "" { @@ -95,6 +109,7 @@ func (c *BeanstalkdSender) SendEvent(ctx context.Context, ev interfaces.Incoming } beanstalkdQueueNameStr := beanstalkdQueueName.(string) + span.SetAttributes(attribute.String("beanstalkdQueueName", beanstalkdQueueNameStr)) isValidBeanstalkdQueueNameStr, err := regexp.MatchString(allowedCharacters, beanstalkdQueueNameStr) if err != nil || !isValidBeanstalkdQueueNameStr { bLogger.WithFields(log.Fields{ @@ -110,6 +125,7 @@ func (c *BeanstalkdSender) SendEvent(ctx context.Context, ev interfaces.Incoming conn.Tube = beanstalk.Tube{Conn: conn, Name: beanstalkdQueueNameStr} + e.DistributedTracingInfo.InjectContext(ctx) jobBody, err := json.Marshal(e.Payload) if err != nil { bLogger.WithFields(log.Fields{ @@ -153,7 +169,6 @@ func (c *BeanstalkdSender) SendEvent(ctx context.Context, ev interfaces.Incoming "jobBody": string(jobBody), }).Info("Enqueue job.") - defer conn.Close() return nil } diff --git a/senders/http_proxy_sender.go b/senders/http_proxy_sender.go index ae01fa3..e0c4593 100644 --- a/senders/http_proxy_sender.go +++ b/senders/http_proxy_sender.go @@ -9,47 +9,61 @@ import ( "net/http" interfaces "github.com/shoplineapp/captin/v2/interfaces" + "github.com/shoplineapp/captin/v2/internal/helpers" models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" ) var hpLogger = log.WithFields(log.Fields{"class": "HTTPProxyEventSender"}) -// HTTPProxyResponse - HTTP Response -type HTTPProxyResponse struct { - url string - response *http.Response - err error -} +var _ interfaces.EventSenderInterface = &HTTPProxyEventSender{} // HTTPProxyEventSender - Send Event through HTTP with payload only // Different from HTTPEventSender, which parses the whole event body // in order to pass event meta data to destinations, -// HTTPProxyEventSender only parses payload for general usage of +// HTTPProxyEventSender only parses payload for general usage of // third party API calls. +type HTTPProxyEventSender struct{} + func (c *HTTPProxyEventSender) SendEvent(ctx context.Context, ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) (err error) { + ctx, span := helpers.Tracer().Start(ctx, "captin.HTTPProxyEventSender.SendEvent") + defer func() { + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + span.End() + }() + e := ev.(models.IncomingEvent) d := dv.(models.Destination) url := d.GetCallbackURL() + // clear the distributed tracing context before sending the request to external receivers + e.DistributedTracingInfo.ClearContext() payload, err := json.Marshal(e.Payload) if err != nil { return err } - req, reqErr := http.NewRequest("POST", url, bytes.NewBuffer(payload)) + req, reqErr := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(payload)) if reqErr != nil { return reqErr } req.Header.Set("Content-Type", "application/json") - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - + // For tracing the event of sending the request + // For external receivers, we don't want to propagate the trace context, while we still want to trace the event of sending the request + // Therefore we use a CompositeTextMapPropagator with no base propagators as a no-op propagator + noopPropagator := propagation.NewCompositeTextMapPropagator() client := &http.Client{ - Transport: tr, + Transport: otelhttp.NewTransport(&http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, otelhttp.WithPropagators(noopPropagator)), } res, resErr := client.Do(req) diff --git a/senders/http_sender.go b/senders/http_sender.go index bdfb23a..4519014 100644 --- a/senders/http_sender.go +++ b/senders/http_sender.go @@ -8,48 +8,52 @@ import ( "net/http" interfaces "github.com/shoplineapp/captin/v2/interfaces" + "github.com/shoplineapp/captin/v2/internal/helpers" models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/codes" ) var hLogger = log.WithFields(log.Fields{"class": "HttpEventSender"}) -// HTTPResponse - HTTP Response -type HTTPResponse struct { - url string - response *http.Response - err error -} +var _ interfaces.EventSenderInterface = &HTTPEventSender{} // HTTPEventSender - Send Event through HTTP -type HTTPEventSender struct { - interfaces.EventSenderInterface -} +type HTTPEventSender struct{} func (c *HTTPEventSender) SendEvent(ctx context.Context, ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) (err error) { + ctx, span := helpers.Tracer().Start(ctx, "captin.HTTPEventSender.SendEvent") + defer func() { + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + span.End() + }() + e := ev.(models.IncomingEvent) d := dv.(models.Destination) url := d.GetCallbackURL() + e.DistributedTracingInfo.InjectContext(ctx) payload, err := e.ToJson() if err != nil { return err } - // TODO: Read from config - req, reqErr := http.NewRequest("POST", url, bytes.NewBuffer(payload)) + req, reqErr := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(payload)) if reqErr != nil { return reqErr } req.Header.Set("Content-Type", "application/json") - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{ - Transport: tr, + // for tracing + Transport: otelhttp.NewTransport(&http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }), } res, resErr := client.Do(req) diff --git a/senders/sqs_sender.go b/senders/sqs_sender.go index 51efcf6..cee6a8e 100644 --- a/senders/sqs_sender.go +++ b/senders/sqs_sender.go @@ -4,8 +4,11 @@ import ( "context" interfaces "github.com/shoplineapp/captin/v2/interfaces" + "github.com/shoplineapp/captin/v2/internal/helpers" models "github.com/shoplineapp/captin/v2/models" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" aws "github.com/aws/aws-sdk-go/aws" aws_credentials "github.com/aws/aws-sdk-go/aws/credentials" @@ -16,9 +19,10 @@ import ( var sLogger = log.WithFields(log.Fields{"class": "SqsSender"}) +var _ interfaces.EventSenderInterface = &SqsSender{} + // SqsSender - Send Event to AWS SQS type SqsSender struct { - interfaces.EventSenderInterface DefaultClient aws_sqsiface.SQSAPI DestinationClientMap map[string]aws_sqsiface.SQSAPI } @@ -33,19 +37,30 @@ func NewSqsSender(defaultAwsConfig aws.Config) *SqsSender { // SendEvent - Send incoming event into SQS queue func (s *SqsSender) SendEvent(ctx context.Context, ev interfaces.IncomingEventInterface, dv interfaces.DestinationInterface) (err error) { + ctx, span := helpers.Tracer().Start(ctx, "captin.SqsSender.SendEvent") + defer func() { + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + } + span.End() + }() + e := ev.(models.IncomingEvent) d := dv.(models.Destination) queueURL := d.GetCallbackURL() + span.SetAttributes(attribute.String("queueURL", queueURL)) sLogger.WithFields(log.Fields{"queueURL": queueURL}).Debug("Send sqs event") + e.DistributedTracingInfo.InjectContext(ctx) payload, jsonErr := e.ToJson() if jsonErr != nil { sLogger.WithFields(log.Fields{"error": jsonErr}).Error("Failed to convert incoming event to json payload") return jsonErr } - _, err := s.GetClient(dv).SendMessage(&aws_sqs.SendMessageInput{ + _, err = s.GetClient(dv).SendMessageWithContext(ctx, &aws_sqs.SendMessageInput{ MessageBody: aws.String(string(payload)), QueueUrl: &queueURL, }) From 92b243befe72e39c0d1f184b6d0cbc84c88a3b89 Mon Sep 17 00:00:00 2001 From: Chung Wu Date: Thu, 11 Jan 2024 15:22:53 +0800 Subject: [PATCH 4/8] chore: split span for delayed event AfterFunc --- internal/outgoing/dispatcher.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/outgoing/dispatcher.go b/internal/outgoing/dispatcher.go index f595ba4..f2e0a47 100644 --- a/internal/outgoing/dispatcher.go +++ b/internal/outgoing/dispatcher.go @@ -268,8 +268,8 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE span.RecordError(err) span.SetStatus(codes.Error, err.Msg) d.OnError(ctx, e, err) - span.End() } + span.End() }() // Check if store have payload @@ -293,7 +293,6 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE "eventDataKey": "dataKey", }).Debug("Skipping update on event data") span.AddEvent("Skipping update on event data") - span.End() return } @@ -353,7 +352,9 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE } // Schedule send event later + span.SetAttributes(attribute.Int("delay_milliseconds", int(timeRemain.Milliseconds()))) dispatcher.TrackAfterFuncJob(timeRemain, func() { + ctx, span := helpers.Tracer().Start(ctx, "captin.processDelayedEvent.AfterFunc") defer span.End() dLogger.WithFields(log.Fields{"key": dataKey}).Debug("After event callback") payload, exists, _, _ := store.Get(ctx, dataKey) From 0a0ca786bb35589eaacaaf542e9337ab8a70a127 Mon Sep 17 00:00:00 2001 From: Chung Wu Date: Thu, 11 Jan 2024 15:53:30 +0800 Subject: [PATCH 5/8] chore: warn for unhandled error --- internal/outgoing/dispatcher.go | 81 +++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 15 deletions(-) diff --git a/internal/outgoing/dispatcher.go b/internal/outgoing/dispatcher.go index f2e0a47..dff4f8e 100644 --- a/internal/outgoing/dispatcher.go +++ b/internal/outgoing/dispatcher.go @@ -194,12 +194,23 @@ func (d *Dispatcher) customizeEvent(ctx context.Context, e models.IncomingEvent, func (d *Dispatcher) injectThrottledPayloads(ctx context.Context, e models.IncomingEvent, destination models.Destination, store interfaces.StoreInterface) interfaces.IncomingEventInterface { if destination.Config.GetKeepThrottledPayloads() { queueKey := getEventThrottledPayloadsKey(ctx, store, e, destination) - payloadStrings, _, _, _ := store.GetQueue(ctx, queueKey) - store.Remove(ctx, queueKey) + payloadStrings, _, _, err := store.GetQueue(ctx, queueKey) + if err != nil { + dLogger.WithFields(log.Fields{"event": e, "error": err}).Error("Error occurred when getting queue for throttled payloads") + // TODO: handle the error properly + } + _, err = store.Remove(ctx, queueKey) + if err != nil { + dLogger.WithFields(log.Fields{"event": e, "error": err}).Error("Error occurred when removing queue key for throttled payloads") + // TODO: handle the error properly + } for _, payloadStr := range payloadStrings { payload := map[string]interface{}{} - // TODO: handle error - json.Unmarshal([]byte(payloadStr), &payload) + err := json.Unmarshal([]byte(payloadStr), &payload) + if err != nil { + dLogger.WithFields(log.Fields{"event": e, "error": err}).Error("Error occurred when unmarshalling throttled payload") + // TODO: handle the error properly + } e.ThrottledPayloads = append(e.ThrottledPayloads, payload) } } @@ -210,12 +221,23 @@ func (d *Dispatcher) injectThrottledPayloads(ctx context.Context, e models.Incom func (d *Dispatcher) injectThrottledDocuments(ctx context.Context, e models.IncomingEvent, destination models.Destination, store interfaces.StoreInterface) interfaces.IncomingEventInterface { if destination.Config.GetIncludeDocument() && destination.Config.GetKeepThrottledDocuments() { queueKey := getEventThrottledDocumentsKey(ctx, store, e, destination) - documentStrings, _, _, _ := store.GetQueue(ctx, queueKey) - store.Remove(ctx, queueKey) + documentStrings, _, _, err := store.GetQueue(ctx, queueKey) + if err != nil { + dLogger.WithFields(log.Fields{"event": e, "error": err}).Warn("Error occurred when getting queue for throttled documents") + // TODO: handle the error properly + } + _, err = store.Remove(ctx, queueKey) + if err != nil { + dLogger.WithFields(log.Fields{"event": e, "error": err}).Warn("Error occurred when removing queue key for throttled documents") + // TODO: handle the error properly + } for _, documentStr := range documentStrings { document := map[string]interface{}{} - // TODO: handle error - json.Unmarshal([]byte(documentStr), &document) + err := json.Unmarshal([]byte(documentStr), &document) + if err != nil { + dLogger.WithFields(log.Fields{"event": e, "error": err}).Warn("Error occurred when unmarshalling throttled document") + // TODO: handle the error properly + } e.ThrottledDocuments = append(e.ThrottledDocuments, document) } } @@ -282,7 +304,12 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE var storedEvent models.IncomingEvent if dataExists { storedEvent = models.IncomingEvent{} - json.Unmarshal([]byte(storedData), &storedEvent) + err := json.Unmarshal([]byte(storedData), &storedEvent) + if err != nil { + dLogger.WithFields(log.Fields{"event": e, "error": err}).Warn("Error occurred when unmarshalling stored event when processing delayed event") + span.RecordError(err) + // TODO: handle the error properly + } } if dataExists && getControlTimestamp(storedEvent, 0) > getControlTimestamp(e, uint64(time.Now().UnixNano())) { @@ -311,7 +338,12 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE "ttl": ttl, }).Debug("Storing throttled payload") span.AddEvent("Storing throttled payload", trace.WithAttributes(attribute.String("queueKey", queueKey), attribute.String("throttleValue", ttl.String()))) - store.Enqueue(ctx, queueKey, string(jsonString), ttl) + _, err := store.Enqueue(ctx, queueKey, string(jsonString), ttl) + if err != nil { + dLogger.WithFields(log.Fields{"event": e, "error": err, "queueKey": queueKey}).Warn("Error occurred when enqueuing customized payload when processing delayed event") + span.RecordError(err) + // TODO: handle the error properly + } } if dest.Config.GetIncludeDocument() && dest.Config.GetKeepThrottledDocuments() { @@ -329,7 +361,12 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE "ttl": ttl, }).Debug("Storing throttled document") span.AddEvent("Storing throttled document", trace.WithAttributes(attribute.String("queueKey", queueKey), attribute.String("throttleValue", ttl.String()))) - store.Enqueue(ctx, queueKey, string(jsonString), ttl) + _, err := store.Enqueue(ctx, queueKey, string(jsonString), ttl) + if err != nil { + dLogger.WithFields(log.Fields{"event": e, "error": err, "queueKey": queueKey}).Warn("Error occurred when enqueuing customized document when processing delayed event") + span.RecordError(err) + // TODO: handle the error properly + } } jsonString, jsonErr := e.ToJson() @@ -357,7 +394,12 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE ctx, span := helpers.Tracer().Start(ctx, "captin.processDelayedEvent.AfterFunc") defer span.End() dLogger.WithFields(log.Fields{"key": dataKey}).Debug("After event callback") - payload, exists, _, _ := store.Get(ctx, dataKey) + payload, exists, _, err := store.Get(ctx, dataKey) + if err != nil { + dLogger.WithFields(log.Fields{"event": e, "error": err, "dataKey": dataKey}).Warn("Error occurred when getting payload for delayed event") + span.RecordError(err) + // TODO: handle the error properly + } // Key might be deleted by another worker, resulting in data not found if !exists { dLogger.WithFields(log.Fields{"key": dataKey}).Debug("Event data not found") @@ -368,10 +410,19 @@ func (d *Dispatcher) processDelayedEvent(ctx context.Context, e models.IncomingE return } event := models.IncomingEvent{} - // TODO: check error - json.Unmarshal([]byte(payload), &event) + err = json.Unmarshal([]byte(payload), &event) + if err != nil { + dLogger.WithFields(log.Fields{"key": dataKey}).Warn("Error occurred when unmarshalling event data for delayed event") + span.RecordError(err) + // TODO: handle the error properly + } d.sendEvent(ctx, event, dest, store, documentStore) - store.Remove(ctx, dataKey) + _, err = store.Remove(ctx, dataKey) + if err != nil { + dLogger.WithFields(log.Fields{"key": dataKey}).Warn("Error occurred when removing event data from store for delayed event") + span.RecordError(err) + // TODO: handle the error properly + } }) } } From 5664e81ef2c5cbc3f9c23a60fddb5c047daf170a Mon Sep 17 00:00:00 2001 From: Chung Wu Date: Wed, 10 Jan 2024 21:24:09 +0800 Subject: [PATCH 6/8] testing and other minor stuff --- destinations/filters/desired_hook.go | 6 +- destinations/filters/environment.go | 6 +- destinations/filters/source.go | 6 +- destinations/filters/validate.go | 3 +- dispatcher/delayers/goroutine.go | 6 +- errors/dispatcher_error.go | 4 +- interfaces/errors.go | 2 +- internal/document_stores/null_store.go | 6 +- internal/stores/memstore.go | 3 +- internal/throttles/throttler.go | 3 +- models/config.go | 4 +- senders/beanstlakd_sender.go | 3 +- senders/console_sender.go | 2 + test/mocks/document_store_mock.go | 3 +- test/mocks/sender_mock.go | 3 +- test/mocks/store_mock.go | 7 +- test/mocks/throttle_mock.go | 5 +- test/models/distributed_tracing_info_test.go | 76 ++++++++++++++ test/models/incoming_event_test.go | 101 ++++++++++++++----- test/senders/sqs_sender_test.go | 18 ++-- 20 files changed, 202 insertions(+), 65 deletions(-) create mode 100644 test/models/distributed_tracing_info_test.go diff --git a/destinations/filters/desired_hook.go b/destinations/filters/desired_hook.go index 3a84dd0..8fbc000 100644 --- a/destinations/filters/desired_hook.go +++ b/destinations/filters/desired_hook.go @@ -23,10 +23,10 @@ func stringList(list []interface{}) []string { return sList } +var _ DestinationFilterInterface = DesiredHookFilter{} + // DesiredHookFilter - Filter destination if given event has desired destination -type DesiredHookFilter struct { - DestinationFilterInterface -} +type DesiredHookFilter struct{} // Run - Get desired hooks in control and filter out exclusion func (f DesiredHookFilter) Run(ctx context.Context, e models.IncomingEvent, d models.Destination) (bool, error) { diff --git a/destinations/filters/environment.go b/destinations/filters/environment.go index c231011..ceb5df2 100644 --- a/destinations/filters/environment.go +++ b/destinations/filters/environment.go @@ -9,9 +9,9 @@ import ( var eLogger = log.WithFields(log.Fields{"class": "EnvironmentFilter"}) -type EnvironmentFilter struct { - DestinationFilterInterface -} +var _ DestinationFilterInterface = EnvironmentFilter{} + +type EnvironmentFilter struct{} // Destination needs to be enabled by ENV Variable {Config Name}_ENABLED, e.g, WAPOS_SYNC_ENABLED func (f EnvironmentFilter) Run(ctx context.Context, e models.IncomingEvent, d models.Destination) (bool, error) { diff --git a/destinations/filters/source.go b/destinations/filters/source.go index ff713ce..e4c474a 100644 --- a/destinations/filters/source.go +++ b/destinations/filters/source.go @@ -6,9 +6,9 @@ import ( models "github.com/shoplineapp/captin/v2/models" ) -type SourceFilter struct { - DestinationFilterInterface -} +var _ DestinationFilterInterface = SourceFilter{} + +type SourceFilter struct{} func (f SourceFilter) Run(ctx context.Context, e models.IncomingEvent, d models.Destination) (bool, error) { return e.Source != d.Config.GetSource(), nil diff --git a/destinations/filters/validate.go b/destinations/filters/validate.go index 2326bf5..5c7c13b 100644 --- a/destinations/filters/validate.go +++ b/destinations/filters/validate.go @@ -12,8 +12,9 @@ import ( var vLogger = log.WithFields(log.Fields{"class": "ValidateFilter"}) +var _ DestinationFilterInterface = ValidateFilter{} + type ValidateFilter struct { - DestinationFilterInterface } func (f ValidateFilter) Run(ctx context.Context, e models.IncomingEvent, d models.Destination) (bool, error) { diff --git a/dispatcher/delayers/goroutine.go b/dispatcher/delayers/goroutine.go index 64ae28a..747016d 100644 --- a/dispatcher/delayers/goroutine.go +++ b/dispatcher/delayers/goroutine.go @@ -12,9 +12,9 @@ import ( log "github.com/sirupsen/logrus" ) -type GoroutineDelayer struct { - interfaces.DispatchDelayerInterface -} +var _ interfaces.DispatchDelayerInterface = GoroutineDelayer{} + +type GoroutineDelayer struct{} var dLogger = log.WithFields(log.Fields{"class": "Goroutine"}) diff --git a/errors/dispatcher_error.go b/errors/dispatcher_error.go index fe0b725..964bdc2 100644 --- a/errors/dispatcher_error.go +++ b/errors/dispatcher_error.go @@ -7,10 +7,10 @@ import ( models "github.com/shoplineapp/captin/v2/models" ) +var _ interfaces.ErrorInterface = &DispatcherError{} + // DispatcherError - Error when send events type DispatcherError struct { - interfaces.ErrorInterface - Msg string Event models.IncomingEvent Destination models.Destination diff --git a/interfaces/errors.go b/interfaces/errors.go index 4922ea4..7158089 100644 --- a/interfaces/errors.go +++ b/interfaces/errors.go @@ -2,4 +2,4 @@ package interfaces type ErrorInterface interface { Error() string -} \ No newline at end of file +} diff --git a/internal/document_stores/null_store.go b/internal/document_stores/null_store.go index 3c55ca2..47a1c18 100644 --- a/internal/document_stores/null_store.go +++ b/internal/document_stores/null_store.go @@ -6,10 +6,10 @@ import ( interfaces "github.com/shoplineapp/captin/v2/interfaces" ) +var _ interfaces.DocumentStoreInterface = NullDocumentStore{} + // NullDocumentStore - Null data store -type NullDocumentStore struct { - interfaces.DocumentStoreInterface -} +type NullDocumentStore struct{} // NewNullDocumentStore - Create new NullDocumentStore func NewNullDocumentStore() *NullDocumentStore { diff --git a/internal/stores/memstore.go b/internal/stores/memstore.go index 705faaa..9ebdafa 100644 --- a/internal/stores/memstore.go +++ b/internal/stores/memstore.go @@ -19,9 +19,10 @@ type item struct { ttl time.Duration } +var _ interfaces.StoreInterface = &MemoryStore{} + // MemoryStore - In-app memory storage type MemoryStore struct { - interfaces.StoreInterface m map[string]*item lock sync.Mutex } diff --git a/internal/throttles/throttler.go b/internal/throttles/throttler.go index 6c511bc..771cd71 100644 --- a/internal/throttles/throttler.go +++ b/internal/throttles/throttler.go @@ -10,9 +10,10 @@ import ( var tLogger = log.WithFields(log.Fields{"class": "Throttler"}) +var _ interfaces.ThrottleInterface = &Throttler{} + // Throttler - Event Throttler type Throttler struct { - interfaces.ThrottleInterface store interfaces.StoreInterface } diff --git a/models/config.go b/models/config.go index 1945eed..af57851 100644 --- a/models/config.go +++ b/models/config.go @@ -11,10 +11,10 @@ import ( "github.com/shoplineapp/captin/v2/interfaces" ) +var _ interfaces.ConfigurationInterface = &Configuration{} + // Configuration - Webhook Configuration Model type Configuration struct { - interfaces.ConfigurationInterface - ConfigID string `json:"id"` CallbackURL string `json:"callback_url"` Validate string `json:"validate"` diff --git a/senders/beanstlakd_sender.go b/senders/beanstlakd_sender.go index b1902c5..cf9feae 100644 --- a/senders/beanstlakd_sender.go +++ b/senders/beanstlakd_sender.go @@ -26,9 +26,10 @@ var bLogger = log.WithFields(log.Fields{"class": "BeanstalkdSender"}) // Source: https://github.com/beanstalkd/go-beanstalk/blob/master/name.go const allowedCharacters = `^[A-Za-z0-9\\\-\+\/\;\.\$\_\(\)]{1,199}$` +var _ interfaces.EventSenderInterface = &BeanstalkdSender{} + // BeanstalkdSender - Send Event to beanstalkd type BeanstalkdSender struct { - interfaces.EventSenderInterface StatsdClient *statsd.Client } diff --git a/senders/console_sender.go b/senders/console_sender.go index 4eafc6e..432ca2a 100644 --- a/senders/console_sender.go +++ b/senders/console_sender.go @@ -10,6 +10,8 @@ import ( var cLogger = log.WithFields(log.Fields{"class": "ConsoleEventSender"}) +var _ interfaces.EventSenderInterface = &ConsoleEventSender{} + // ConsoleEventSender - Present Event in console type ConsoleEventSender struct{} diff --git a/test/mocks/document_store_mock.go b/test/mocks/document_store_mock.go index 8d4aafd..c45bac1 100644 --- a/test/mocks/document_store_mock.go +++ b/test/mocks/document_store_mock.go @@ -8,9 +8,10 @@ import ( "github.com/stretchr/testify/mock" ) +var _ interfaces.DocumentStoreInterface = &DocumentStoreMock{} + // DocumentStoreMock - Mock of DocumentStoreInterface type DocumentStoreMock struct { - interfaces.DocumentStoreInterface mock.Mock } diff --git a/test/mocks/sender_mock.go b/test/mocks/sender_mock.go index 3adbf2b..5f882d5 100644 --- a/test/mocks/sender_mock.go +++ b/test/mocks/sender_mock.go @@ -8,10 +8,11 @@ import ( "github.com/stretchr/testify/mock" ) +var _ interfaces.EventSenderInterface = &SenderMock{} + // SenderMock - Mock of SenderInterface type SenderMock struct { mock.Mock - interfaces.EventSenderInterface } // SendEvent - Send an event diff --git a/test/mocks/store_mock.go b/test/mocks/store_mock.go index 4412d1f..2d6e4e9 100644 --- a/test/mocks/store_mock.go +++ b/test/mocks/store_mock.go @@ -5,14 +5,15 @@ import ( "fmt" "time" - "github.com/shoplineapp/captin/interfaces" - "github.com/shoplineapp/captin/models" + "github.com/shoplineapp/captin/v2/interfaces" + "github.com/shoplineapp/captin/v2/models" "github.com/stretchr/testify/mock" ) +var _ interfaces.StoreInterface = &StoreMock{} + // StoreMock - Mock of StoreInterface type StoreMock struct { - interfaces.StoreInterface mock.Mock } diff --git a/test/mocks/throttle_mock.go b/test/mocks/throttle_mock.go index 077de21..38a4b90 100644 --- a/test/mocks/throttle_mock.go +++ b/test/mocks/throttle_mock.go @@ -4,13 +4,14 @@ import ( "context" "time" - "github.com/shoplineapp/captin/interfaces" + "github.com/shoplineapp/captin/v2/interfaces" "github.com/stretchr/testify/mock" ) +var _ interfaces.ThrottleInterface = &ThrottleMock{} + // ThrottleMock - Mock ThrottleInterface type ThrottleMock struct { - interfaces.ThrottleInterface mock.Mock } diff --git a/test/models/distributed_tracing_info_test.go b/test/models/distributed_tracing_info_test.go new file mode 100644 index 0000000..e4ff0eb --- /dev/null +++ b/test/models/distributed_tracing_info_test.go @@ -0,0 +1,76 @@ +package models_test + +import ( + "context" + "encoding/json" + "testing" + + . "github.com/shoplineapp/captin/v2/models" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" +) + +func TestDistributedTracingInfoNormalFlow(t *testing.T) { + spanCtxConfig := trace.SpanContextConfig{ + TraceID: trace.TraceID([16]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}), + SpanID: trace.SpanID([8]byte{0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22}), + TraceFlags: trace.TraceFlags(0x01), + } + var err error + spanCtxConfig.TraceState, err = trace.ParseTraceState("a=b, c=d") + require.NoError(t, err) + + spanContext := trace.NewSpanContext(spanCtxConfig) + require.True(t, spanContext.IsValid()) + d := NewDistributedTracingInfoFromContext(trace.ContextWithSpanContext(context.Background(), spanContext)) + + // serialize to JSON + bz, err := json.Marshal(d) + require.NoError(t, err) + + jsonStructure := make(map[string]string) + err = json.Unmarshal(bz, &jsonStructure) + require.NoError(t, err) + require.Equal(t, map[string]string{ + "traceparent": "00-11111111111111111111111111111111-2222222222222222-01", + "tracestate": "a=b,c=d", + }, jsonStructure) + + // recover from JSON + var d2 DistributedTracingInfo + err = json.Unmarshal(bz, &d2) + require.NoError(t, err) + require.Equal(t, d, d2) + + // recover the trace context + ctx := d2.PropagateIntoContext(context.Background()) + spanContext2 := trace.SpanContextFromContext(ctx) + require.Equal(t, spanContext.TraceID(), spanContext2.TraceID()) + require.Equal(t, spanContext.SpanID(), spanContext2.SpanID()) + require.Equal(t, spanContext.TraceFlags(), spanContext2.TraceFlags()) + require.Equal(t, spanContext.TraceState().String(), spanContext2.TraceState().String()) + // since the recovered span context is from propagator, it will be a remote context + require.True(t, spanContext2.IsRemote()) +} + +func TestDistributedTracingInfoNilFlow(t *testing.T) { + // default DistributedTracingInfo should work like a no-op propagator without nil pointer dereference + d := DistributedTracingInfo{} + + // serialize to JSON should work without nil pointer dereference + bz, err := json.Marshal(d) + require.NoError(t, err) + require.Equal(t, "{}", string(bz)) + + // deserialize from null should work + var d2 DistributedTracingInfo + err = json.Unmarshal([]byte("null"), &d2) + require.NoError(t, err) + require.Equal(t, d, d2) + + // PropagateIntoContext should work without nil pointer dereference + ctx := d.PropagateIntoContext(context.Background()) + spanContext2 := trace.SpanContextFromContext(ctx) + require.False(t, spanContext2.IsValid()) +} diff --git a/test/models/incoming_event_test.go b/test/models/incoming_event_test.go index bb6d372..4038885 100644 --- a/test/models/incoming_event_test.go +++ b/test/models/incoming_event_test.go @@ -1,15 +1,62 @@ package models_test import ( + "context" "encoding/json" "reflect" "testing" + "github.com/google/uuid" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" . "github.com/shoplineapp/captin/v2/models" ) +func mustParseSpanContextConfig(traceIDHex string, spanIDHex string, traceFlags byte, traceStateStr string) trace.SpanContextConfig { + traceID, err := trace.TraceIDFromHex(traceIDHex) + if err != nil { + panic(err) + } + spanID, err := trace.SpanIDFromHex(spanIDHex) + if err != nil { + panic(err) + } + traceState, err := trace.ParseTraceState(traceStateStr) + if err != nil { + panic(err) + } + return trace.SpanContextConfig{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: trace.TraceFlags(traceFlags), + TraceState: traceState, + } +} + +var sampleDistributedTracingInfo = NewDistributedTracingInfoFromContext(trace.ContextWithSpanContext( + context.Background(), + trace.NewSpanContext(mustParseSpanContextConfig("11111111111111111111111111111111", "2222222222222222", 0x01, "a=b, c=d")), +)) + +var sampleIncomingEvent = IncomingEvent{ + Key: "product.update", + Source: "core", + Payload: map[string]interface{}{"payload": "data"}, + TargetType: "product", + TargetId: "xxxxx", + Control: map[string]interface{}{ + "extra": "extra", + "ts": 99999999999999, + "host": "host", + "ip_addresses": "ip_addresses", + }, + TargetDocument: map[string]interface{}{"payload": "data"}, + DistributedTracingInfo: sampleDistributedTracingInfo, + TraceId: "11111111-2222-3333-4444-555555555555", +} + func TestNewIncomingEvent(t *testing.T) { event_key := "product.update" source := "core" @@ -22,14 +69,17 @@ func TestNewIncomingEvent(t *testing.T) { } target_type := "product" target_id := "xxxxx" + trace_id := uuid.New().String() data, _ := json.Marshal(map[string]interface{}{ - "event_key": event_key, - "source": source, - "payload": payload, - "control": control, - "target_type": target_type, - "target_id": target_id, + "event_key": event_key, + "source": source, + "payload": payload, + "control": control, + "target_type": target_type, + "target_id": target_id, + "trace_id": trace_id, + "distributed_tracing_info": sampleDistributedTracingInfo, }) inc := NewIncomingEvent(data) assert.Equal(t, event_key, inc.Key) @@ -38,6 +88,8 @@ func TestNewIncomingEvent(t *testing.T) { assert.Equal(t, target_type, inc.TargetType) assert.Equal(t, target_id, inc.TargetId) assert.Equal(t, control, inc.Control) + assert.Equal(t, trace_id, inc.TraceId) + assert.Equal(t, sampleDistributedTracingInfo, inc.DistributedTracingInfo) } func TestIsValid(t *testing.T) { @@ -54,35 +106,34 @@ func TestIsValid(t *testing.T) { } func TestMarshalJSON(t *testing.T) { - e := IncomingEvent{Key: "product.update", Source: "core", Payload: map[string]interface{}{"payload": "data"}, TargetType: "product", TargetId: "xxxxx", Control: map[string]interface{}{"extra": "extra", "ts": 99999999999999, "host": "host", "ip_addresses": "ip_addresses"}, TargetDocument: map[string]interface{}{"payload": "data"}} - val, _ := e.MarshalJSON() - assert.Equal(t, `{"control":{"host":"host","ip_addresses":"ip_addresses","ts":99999999999999},"id":"xxxxx","key":"product.update","source":"core","trace_id":"","type":"product"}`, string(val)) + val, err := sampleIncomingEvent.MarshalJSON() + require.NoError(t, err) + assert.Equal(t, `{"control":{"host":"host","ip_addresses":"ip_addresses","ts":99999999999999},"id":"xxxxx","key":"product.update","source":"core","trace_id":"11111111-2222-3333-4444-555555555555","type":"product"}`, string(val)) } func TestString(t *testing.T) { - e := IncomingEvent{Key: "product.update", Source: "core", Payload: map[string]interface{}{"payload": "data"}, TargetType: "product", TargetId: "xxxxx", Control: map[string]interface{}{"extra": "extra", "ts": 99999999999999, "host": "host", "ip_addresses": "ip_addresses"}, TargetDocument: map[string]interface{}{"payload": "data"}} - val := e.String() - assert.Equal(t, `{"control":{"host":"host","ip_addresses":"ip_addresses","ts":99999999999999},"id":"xxxxx","key":"product.update","source":"core","trace_id":"","type":"product"}`, val) + val := sampleIncomingEvent.String() + assert.Equal(t, `{"control":{"host":"host","ip_addresses":"ip_addresses","ts":99999999999999},"id":"xxxxx","key":"product.update","source":"core","trace_id":"11111111-2222-3333-4444-555555555555","type":"product"}`, string(val)) } func TestToMap(t *testing.T) { - e := IncomingEvent{Key: "product.update", Source: "core", Payload: map[string]interface{}{"payload": "data"}, TargetType: "product", TargetId: "xxxxx", Control: map[string]interface{}{"extra": "extra", "ts": 99999999999999, "host": "host", "ip_addresses": "ip_addresses"}, TargetDocument: map[string]interface{}{"payload": "data"}} - val := e.ToMap() + val := sampleIncomingEvent.ToMap() rs := reflect.DeepEqual(val, map[string]interface{}{ - "control": map[string]interface{}{"host": "host", "ip_addresses": "ip_addresses", "extra": "extra", "ts": 99999999999999}, - "event_key": "product.update", - "payload": map[string]interface{}{"payload": "data"}, - "source": "core", - "target_document": map[string]interface{}{"payload": "data"}, - "target_id": "xxxxx", - "target_type": "product", - "trace_id": "", + "control": map[string]interface{}{"host": "host", "ip_addresses": "ip_addresses", "extra": "extra", "ts": 99999999999999}, + "event_key": "product.update", + "payload": map[string]interface{}{"payload": "data"}, + "source": "core", + "target_document": map[string]interface{}{"payload": "data"}, + "target_id": "xxxxx", + "target_type": "product", + "trace_id": "11111111-2222-3333-4444-555555555555", + "distributed_tracing_info": sampleDistributedTracingInfo, }) assert.Equal(t, true, rs) } func TestToJson(t *testing.T) { - e := IncomingEvent{Key: "product.update", Source: "core", Payload: map[string]interface{}{"payload": "data"}, TargetType: "product", TargetId: "xxxxx", Control: map[string]interface{}{"extra": "extra", "ts": 99999999999999, "host": "host", "ip_addresses": "ip_addresses"}, TargetDocument: map[string]interface{}{"payload": "data"}} - val, _ := e.ToJson() - assert.Equal(t, `{"control":{"extra":"extra","host":"host","ip_addresses":"ip_addresses","ts":99999999999999},"event_key":"product.update","payload":{"payload":"data"},"source":"core","target_document":{"payload":"data"},"target_id":"xxxxx","target_type":"product","trace_id":""}`, string(val)) + val, err := sampleIncomingEvent.ToJson() + require.NoError(t, err) + assert.Equal(t, `{"control":{"extra":"extra","host":"host","ip_addresses":"ip_addresses","ts":99999999999999},"distributed_tracing_info":{"traceparent":"00-11111111111111111111111111111111-2222222222222222-01","tracestate":"a=b,c=d"},"event_key":"product.update","payload":{"payload":"data"},"source":"core","target_document":{"payload":"data"},"target_id":"xxxxx","target_type":"product","trace_id":"11111111-2222-3333-4444-555555555555"}`, string(val)) } diff --git a/test/senders/sqs_sender_test.go b/test/senders/sqs_sender_test.go index b037cf2..c9e5af8 100644 --- a/test/senders/sqs_sender_test.go +++ b/test/senders/sqs_sender_test.go @@ -8,6 +8,7 @@ import ( "testing" aws "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" aws_sqs "github.com/aws/aws-sdk-go/service/sqs" aws_sqsiface "github.com/aws/aws-sdk-go/service/sqs/sqsiface" models "github.com/shoplineapp/captin/v2/models" @@ -23,7 +24,7 @@ type sqsMock struct { SentMessages []aws_sqs.SendMessageInput } -func (s *sqsMock) SendMessage(input *aws_sqs.SendMessageInput) (*aws_sqs.SendMessageOutput, error) { +func (s *sqsMock) SendMessageWithContext(ctx context.Context, input *aws_sqs.SendMessageInput, _ ...request.Option) (*aws_sqs.SendMessageOutput, error) { if s.SentMessages == nil { s.SentMessages = []aws_sqs.SendMessageInput{} } @@ -45,7 +46,7 @@ func TestSqsSender_SendEvent_Success(t *testing.T) { sender := NewSqsSender(awsConfig) sqs := new(sqsMock) - sqs.On("SendMessage", mock.Anything).Return(nil) + sqs.On("SendMessageWithContext", mock.Anything, mock.Anything).Return(nil) sender.DefaultClient = sqs result := sender.SendEvent( @@ -57,7 +58,7 @@ func TestSqsSender_SendEvent_Success(t *testing.T) { ) assert.Nil(t, result) - sqs.AssertNumberOfCalls(t, "SendMessage", 1) + sqs.AssertNumberOfCalls(t, "SendMessageWithContext", 1) } func TestSqsSender_SendEvent_Failed(t *testing.T) { @@ -65,7 +66,7 @@ func TestSqsSender_SendEvent_Failed(t *testing.T) { sender := NewSqsSender(awsConfig) sqs := new(sqsMock) - sqs.On("SendMessage", mock.Anything).Return(nil) + sqs.On("SendMessageWithContext", mock.Anything, mock.Anything).Return(nil) sender.DefaultClient = sqs result := sender.SendEvent( @@ -77,16 +78,15 @@ func TestSqsSender_SendEvent_Failed(t *testing.T) { ) assert.Error(t, result, "some error") - sqs.AssertNumberOfCalls(t, "SendMessage", 1) + sqs.AssertNumberOfCalls(t, "SendMessageWithContext", 1) } - func TestSqsSender_GetClient_UseAccessKey_WithCorrectAwsConfig(t *testing.T) { awsConfig := aws.Config{Region: aws.String("ap-southeast-1")} sender := NewSqsSender(awsConfig) sqs := new(sqsMock) - sqs.On("SendMessage", mock.Anything).Return(nil) + sqs.On("SendMessageWithContext", mock.Anything, mock.Anything).Return(nil) os.Setenv("HOOK_TEST_DESTINATION_CALLBACK_URL", "https://sqs.ap-southeast-1.amazonaws.com/000000000000/queue") os.Setenv("HOOK_TEST_DESTINATION_SQS_SENDER_USE_CUSTOM_CONFIG", "true") @@ -119,7 +119,7 @@ func TestSqsSender_SendEvent_UseAccessKey_Success(t *testing.T) { sender := NewSqsSender(awsConfig) sqs := new(sqsMock) - sqs.On("SendMessage", mock.Anything).Return(nil) + sqs.On("SendMessageWithContext", mock.Anything, mock.Anything).Return(nil) sender.DestinationClientMap["test_destination"] = sqs @@ -134,5 +134,5 @@ func TestSqsSender_SendEvent_UseAccessKey_Success(t *testing.T) { ) assert.Nil(t, result) - sqs.AssertNumberOfCalls(t, "SendMessage", 1) + sqs.AssertNumberOfCalls(t, "SendMessageWithContext", 1) } From 7fc808553f350b5b0ee241f1cf23b00554caed9b Mon Sep 17 00:00:00 2001 From: Chung Wu Date: Tue, 30 Apr 2024 14:23:25 +0800 Subject: [PATCH 7/8] chore: upgrade to AWS SDK v2 --- .semaphore/semaphore.yml | 6 ++--- go.mod | 26 ++++++++++++++++---- go.sum | 29 +++++++++------------- senders/sqs_sender.go | 34 +++++++++++++------------- test/senders/sqs_sender_test.go | 43 +++++++++++++++++---------------- 5 files changed, 75 insertions(+), 63 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index b24389d..6416e6e 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -12,7 +12,7 @@ blocks: value: "1" prologue: commands: - - sem-version go 1.15 + - sem-version go 1.20 # Go project boiler plate - export "SEMAPHORE_GIT_DIR=/tmp/${SEMAPHORE_PROJECT_NAME}" - export "PATH=$(go env GOPATH)/bin:${PATH}" @@ -29,7 +29,7 @@ blocks: task: prologue: commands: - - sem-version go 1.15 + - sem-version go 1.20 # Go project boiler plate - export "SEMAPHORE_GIT_DIR=/tmp/${SEMAPHORE_PROJECT_NAME}" - export "PATH=$(go env GOPATH)/bin:${PATH}" @@ -45,4 +45,4 @@ blocks: jobs: - name: Suite commands: - - make test \ No newline at end of file + - make test diff --git a/go.mod b/go.mod index 6576539..4a0b1b0 100644 --- a/go.mod +++ b/go.mod @@ -1,23 +1,39 @@ module github.com/shoplineapp/captin/v2 -go 1.15 +go 1.20 require ( - github.com/aws/aws-sdk-go v1.34.34 + github.com/aws/aws-sdk-go-v2 v1.26.1 + github.com/aws/aws-sdk-go-v2/credentials v1.17.11 + github.com/aws/aws-sdk-go-v2/service/sqs v1.31.4 github.com/beanstalkd/go-beanstalk v0.0.0-20190515041346-390b03b3064a github.com/google/uuid v1.2.0 github.com/joeycumines/statsd v1.0.1-0.20201117043332-bb35aa955658 - github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect - github.com/kr/pretty v0.1.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d github.com/sirupsen/logrus v1.4.2 github.com/stretchr/testify v1.8.4 github.com/thoas/go-funk v0.7.0 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.21.0 go.opentelemetry.io/otel/trace v1.21.0 +) + +require ( + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/smithy-go v1.20.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect + github.com/kr/pretty v0.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 82519ec..e39873e 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,15 @@ -github.com/aws/aws-sdk-go v1.34.34 h1:5dC0ZU0xy25+UavGNEkQ/5MOQwxXDA2YXtjCL1HfYKI= -github.com/aws/aws-sdk-go v1.34.34/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= +github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= +github.com/aws/aws-sdk-go-v2/service/sqs v1.31.4 h1:mE2ysZMEeQ3ulHWs4mmc4fZEhOfeY1o6QXAfDqjbSgw= +github.com/aws/aws-sdk-go-v2/service/sqs v1.31.4/go.mod h1:lCN2yKnj+Sp9F6UzpoPPTir+tSaC9Jwf6LcmTqnXFZw= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/beanstalkd/go-beanstalk v0.0.0-20190515041346-390b03b3064a h1:Q9n7/Y0jg/U18xjQz2l42we7XQAqwkBGWByBZ36BAHo= github.com/beanstalkd/go-beanstalk v0.0.0-20190515041346-390b03b3064a/go.mod h1:Q3f6RCbUHp8RHSfBiPUZBojK76rir8Rl+KINuz2/sYs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -12,15 +22,9 @@ github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joeycumines/statsd v1.0.1-0.20201117043332-bb35aa955658 h1:qg1swZu2+awU2o2Vq0HiIfbvyUBV0MnCeG/BKoXN+Dg= github.com/joeycumines/statsd v1.0.1-0.20201117043332-bb35aa955658/go.mod h1:SLKAkQ5CgPBRFFIv3JAjQjBWEOmJJxHn33bwAnFFVMU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -33,7 +37,6 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d h1:1VUlQbCfkoSGv7qP7Y+ro3ap1P1pPZxgdGVqiTVy5C4= @@ -61,23 +64,15 @@ go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ3 go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 h1:fqTvyMIIj+HRzMmnzr9NtpHP6uVpvB5fkHcgPDC4nu8= golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/senders/sqs_sender.go b/senders/sqs_sender.go index cee6a8e..c157622 100644 --- a/senders/sqs_sender.go +++ b/senders/sqs_sender.go @@ -10,28 +10,29 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" - aws "github.com/aws/aws-sdk-go/aws" - aws_credentials "github.com/aws/aws-sdk-go/aws/credentials" - aws_session "github.com/aws/aws-sdk-go/aws/session" - aws_sqs "github.com/aws/aws-sdk-go/service/sqs" - aws_sqsiface "github.com/aws/aws-sdk-go/service/sqs/sqsiface" + "github.com/aws/aws-sdk-go-v2/aws" + aws_credentials "github.com/aws/aws-sdk-go-v2/credentials" + aws_sqs "github.com/aws/aws-sdk-go-v2/service/sqs" ) var sLogger = log.WithFields(log.Fields{"class": "SqsSender"}) +type SqsApiInterface interface { + SendMessage(ctx context.Context, input *aws_sqs.SendMessageInput, _ ...func(*aws_sqs.Options)) (*aws_sqs.SendMessageOutput, error) +} + var _ interfaces.EventSenderInterface = &SqsSender{} // SqsSender - Send Event to AWS SQS type SqsSender struct { - DefaultClient aws_sqsiface.SQSAPI - DestinationClientMap map[string]aws_sqsiface.SQSAPI + DefaultClient SqsApiInterface + DestinationClientMap map[string]SqsApiInterface } func NewSqsSender(defaultAwsConfig aws.Config) *SqsSender { - defaultSession := aws_session.Must(aws_session.NewSession(&defaultAwsConfig)) return &SqsSender{ - DefaultClient: aws_sqs.New(defaultSession), - DestinationClientMap: map[string]aws_sqsiface.SQSAPI{}, + DefaultClient: aws_sqs.NewFromConfig(defaultAwsConfig), + DestinationClientMap: make(map[string]SqsApiInterface), } } @@ -60,7 +61,7 @@ func (s *SqsSender) SendEvent(ctx context.Context, ev interfaces.IncomingEventIn return jsonErr } - _, err = s.GetClient(dv).SendMessageWithContext(ctx, &aws_sqs.SendMessageInput{ + _, err = s.GetClient(dv).SendMessage(ctx, &aws_sqs.SendMessageInput{ MessageBody: aws.String(string(payload)), QueueUrl: &queueURL, }) @@ -72,7 +73,7 @@ func (s *SqsSender) SendEvent(ctx context.Context, ev interfaces.IncomingEventIn return err } -func (s *SqsSender) GetClient(dv interfaces.DestinationInterface) aws_sqsiface.SQSAPI { +func (s *SqsSender) GetClient(dv interfaces.DestinationInterface) SqsApiInterface { d := dv.(models.Destination) destName := d.Config.GetName() @@ -82,19 +83,18 @@ func (s *SqsSender) GetClient(dv interfaces.DestinationInterface) aws_sqsiface.S awsConfig := aws.Config{} if dv.GetSqsSenderConfig("AWS_ENDPOINT") != "" { - awsConfig.Endpoint = aws.String(dv.GetSqsSenderConfig("AWS_ENDPOINT")) + awsConfig.BaseEndpoint = aws.String(dv.GetSqsSenderConfig("AWS_ENDPOINT")) } if dv.GetSqsSenderConfig("AWS_REGION") != "" { - awsConfig.Region = aws.String(dv.GetSqsSenderConfig("AWS_REGION")) + awsConfig.Region = dv.GetSqsSenderConfig("AWS_REGION") } if dv.GetSqsSenderConfig("AWS_ACCESS_KEY_ID") != "" && dv.GetSqsSenderConfig("AWS_SECRET_ACCESS_KEY") != "" { - awsConfig.Credentials = aws_credentials.NewStaticCredentials(dv.GetSqsSenderConfig("AWS_ACCESS_KEY_ID"), dv.GetSqsSenderConfig("AWS_SECRET_ACCESS_KEY"), "") + awsConfig.Credentials = aws_credentials.NewStaticCredentialsProvider(dv.GetSqsSenderConfig("AWS_ACCESS_KEY_ID"), dv.GetSqsSenderConfig("AWS_SECRET_ACCESS_KEY"), "") } - session := aws_session.Must(aws_session.NewSession(&awsConfig)) - s.DestinationClientMap[destName] = aws_sqs.New(session) + s.DestinationClientMap[destName] = aws_sqs.NewFromConfig(awsConfig) } return s.DestinationClientMap[destName] diff --git a/test/senders/sqs_sender_test.go b/test/senders/sqs_sender_test.go index c9e5af8..8f70710 100644 --- a/test/senders/sqs_sender_test.go +++ b/test/senders/sqs_sender_test.go @@ -7,10 +7,8 @@ import ( "os" "testing" - aws "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - aws_sqs "github.com/aws/aws-sdk-go/service/sqs" - aws_sqsiface "github.com/aws/aws-sdk-go/service/sqs/sqsiface" + "github.com/aws/aws-sdk-go-v2/aws" + aws_sqs "github.com/aws/aws-sdk-go-v2/service/sqs" models "github.com/shoplineapp/captin/v2/models" . "github.com/shoplineapp/captin/v2/senders" "github.com/stretchr/testify/assert" @@ -18,13 +16,13 @@ import ( ) type sqsMock struct { - aws_sqsiface.SQSAPI mock.Mock SentMessages []aws_sqs.SendMessageInput } -func (s *sqsMock) SendMessageWithContext(ctx context.Context, input *aws_sqs.SendMessageInput, _ ...request.Option) (*aws_sqs.SendMessageOutput, error) { +func (s *sqsMock) SendMessage(ctx context.Context, input *aws_sqs.SendMessageInput, _ ...func(*aws_sqs.Options)) (*aws_sqs.SendMessageOutput, error) { + if s.SentMessages == nil { s.SentMessages = []aws_sqs.SendMessageInput{} } @@ -42,11 +40,11 @@ func (s *sqsMock) SendMessageWithContext(ctx context.Context, input *aws_sqs.Sen } func TestSqsSender_SendEvent_Success(t *testing.T) { - awsConfig := aws.Config{Region: aws.String("ap-southeast-1")} + awsConfig := aws.Config{Region: "ap-southeast-1"} sender := NewSqsSender(awsConfig) sqs := new(sqsMock) - sqs.On("SendMessageWithContext", mock.Anything, mock.Anything).Return(nil) + sqs.On("SendMessage", mock.Anything, mock.Anything).Return(nil) sender.DefaultClient = sqs result := sender.SendEvent( @@ -58,15 +56,15 @@ func TestSqsSender_SendEvent_Success(t *testing.T) { ) assert.Nil(t, result) - sqs.AssertNumberOfCalls(t, "SendMessageWithContext", 1) + sqs.AssertNumberOfCalls(t, "SendMessage", 1) } func TestSqsSender_SendEvent_Failed(t *testing.T) { - awsConfig := aws.Config{Region: aws.String("ap-southeast-1")} + awsConfig := aws.Config{Region: "ap-southeast-1"} sender := NewSqsSender(awsConfig) sqs := new(sqsMock) - sqs.On("SendMessageWithContext", mock.Anything, mock.Anything).Return(nil) + sqs.On("SendMessage", mock.Anything, mock.Anything).Return(nil) sender.DefaultClient = sqs result := sender.SendEvent( @@ -78,15 +76,15 @@ func TestSqsSender_SendEvent_Failed(t *testing.T) { ) assert.Error(t, result, "some error") - sqs.AssertNumberOfCalls(t, "SendMessageWithContext", 1) + sqs.AssertNumberOfCalls(t, "SendMessage", 1) } func TestSqsSender_GetClient_UseAccessKey_WithCorrectAwsConfig(t *testing.T) { - awsConfig := aws.Config{Region: aws.String("ap-southeast-1")} + awsConfig := aws.Config{Region: "ap-southeast-1"} sender := NewSqsSender(awsConfig) sqs := new(sqsMock) - sqs.On("SendMessageWithContext", mock.Anything, mock.Anything).Return(nil) + sqs.On("SendMessage", mock.Anything, mock.Anything).Return(nil) os.Setenv("HOOK_TEST_DESTINATION_CALLBACK_URL", "https://sqs.ap-southeast-1.amazonaws.com/000000000000/queue") os.Setenv("HOOK_TEST_DESTINATION_SQS_SENDER_USE_CUSTOM_CONFIG", "true") @@ -103,11 +101,14 @@ func TestSqsSender_GetClient_UseAccessKey_WithCorrectAwsConfig(t *testing.T) { }, ) - sqsClient, _ := (client).(*aws_sqs.SQS) - credentials, _ := sqsClient.Config.Credentials.Get() + sqsClient, _ := (client).(*aws_sqs.Client) + + options := sqsClient.Options() + credentials, err := options.Credentials.Retrieve(context.Background()) + assert.NoError(t, err) - assert.Equal(t, *sqsClient.Config.Region, "ap-southeast-1") - assert.Equal(t, *sqsClient.Config.Endpoint, "http://localhost:4566") + assert.Equal(t, options.Region, "ap-southeast-1") + assert.Equal(t, *options.BaseEndpoint, "http://localhost:4566") assert.Equal(t, credentials.AccessKeyID, "MY_ACCESS_KEY_ID") assert.Equal(t, credentials.SecretAccessKey, "MY_SECRET_ACCESS_KEY") } @@ -115,11 +116,11 @@ func TestSqsSender_GetClient_UseAccessKey_WithCorrectAwsConfig(t *testing.T) { func TestSqsSender_SendEvent_UseAccessKey_Success(t *testing.T) { os.Setenv("HOOK_TEST_DESTINATION_SQS_SENDER_USE_CUSTOM_CONFIG", "true") - awsConfig := aws.Config{Region: aws.String("ap-southeast-1")} + awsConfig := aws.Config{Region: "ap-southeast-1"} sender := NewSqsSender(awsConfig) sqs := new(sqsMock) - sqs.On("SendMessageWithContext", mock.Anything, mock.Anything).Return(nil) + sqs.On("SendMessage", mock.Anything, mock.Anything).Return(nil) sender.DestinationClientMap["test_destination"] = sqs @@ -134,5 +135,5 @@ func TestSqsSender_SendEvent_UseAccessKey_Success(t *testing.T) { ) assert.Nil(t, result) - sqs.AssertNumberOfCalls(t, "SendMessageWithContext", 1) + sqs.AssertNumberOfCalls(t, "SendMessage", 1) } From ebfd1cc18e52db9dea9a1267fb086ebaa2db7de8 Mon Sep 17 00:00:00 2001 From: Chung Wu Date: Thu, 2 May 2024 09:48:11 +0800 Subject: [PATCH 8/8] chore: upgrade Go version to 1.22 --- .semaphore/semaphore.yml | 4 ++-- go.mod | 21 ++++++++++++++------- go.sum | 24 ++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 6416e6e..b2a5ada 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -12,7 +12,7 @@ blocks: value: "1" prologue: commands: - - sem-version go 1.20 + - sem-version go 1.22 # Go project boiler plate - export "SEMAPHORE_GIT_DIR=/tmp/${SEMAPHORE_PROJECT_NAME}" - export "PATH=$(go env GOPATH)/bin:${PATH}" @@ -29,7 +29,7 @@ blocks: task: prologue: commands: - - sem-version go 1.20 + - sem-version go 1.22 # Go project boiler plate - export "SEMAPHORE_GIT_DIR=/tmp/${SEMAPHORE_PROJECT_NAME}" - export "PATH=$(go env GOPATH)/bin:${PATH}" diff --git a/go.mod b/go.mod index 4a0b1b0..6a476db 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/shoplineapp/captin/v2 -go 1.20 +go 1.21 + +toolchain go1.22.2 require ( github.com/aws/aws-sdk-go-v2 v1.26.1 @@ -12,26 +14,31 @@ require ( github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/robertkrimen/otto v0.0.0-20180617131154-15f95af6e78d github.com/sirupsen/logrus v1.4.2 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/thoas/go-funk v0.7.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 - go.opentelemetry.io/otel v1.21.0 - go.opentelemetry.io/otel/trace v1.21.0 + go.opentelemetry.io/otel v1.26.0 + go.opentelemetry.io/otel/trace v1.26.0 ) require ( github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.31.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6 // indirect github.com/aws/smithy-go v1.20.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/objx v0.5.0 // indirect - go.opentelemetry.io/otel/metric v1.21.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect + go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws v0.51.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect diff --git a/go.sum b/go.sum index e39873e..512368b 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,12 @@ github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGV github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.31.1 h1:dZXY07Dm59TxAjJcUfNMJHLDI/gLMxTRZefn2jFAVsw= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.31.1/go.mod h1:lVLqEtX+ezgtfalyJs7Peb0uv9dEpAQP5yuq2O26R44= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6 h1:6tayEze2Y+hiL3kdnEUxSPsP+pJsUfwLSFspFl1ru9Q= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.6/go.mod h1:qVNb/9IOVsLCZh0x2lnagrBwQ9fxajUpXS7OZfIsKn0= github.com/aws/aws-sdk-go-v2/service/sqs v1.31.4 h1:mE2ysZMEeQ3ulHWs4mmc4fZEhOfeY1o6QXAfDqjbSgw= github.com/aws/aws-sdk-go-v2/service/sqs v1.31.4/go.mod h1:lCN2yKnj+Sp9F6UzpoPPTir+tSaC9Jwf6LcmTqnXFZw= github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= @@ -20,11 +26,16 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joeycumines/statsd v1.0.1-0.20201117043332-bb35aa955658 h1:qg1swZu2+awU2o2Vq0HiIfbvyUBV0MnCeG/BKoXN+Dg= github.com/joeycumines/statsd v1.0.1-0.20201117043332-bb35aa955658/go.mod h1:SLKAkQ5CgPBRFFIv3JAjQjBWEOmJJxHn33bwAnFFVMU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -48,22 +59,34 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/thoas/go-funk v0.7.0 h1:GmirKrs6j6zJbhJIficOsz2aAI7700KsU/5YrdHRM1Y= github.com/thoas/go-funk v0.7.0/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws v0.51.0 h1:FGMfzzxfkNkw+gvKJOeT8dSmBjgrSFh+ClLl+OMKPno= +go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws v0.51.0/go.mod h1:hmHUXiKhyxbIhuNfG5ZTySq9HqqxJFNxaFOfXXvoMmQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 h1:fqTvyMIIj+HRzMmnzr9NtpHP6uVpvB5fkHcgPDC4nu8= golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -73,6 +96,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI= gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=